feat(pipeline): уведомления CI/CD — PipelineWatcher, CI загрузок на Gitea Actions

- app/pipeline: протокол CIProvider, клиент Gitea Actions (прогон по head_sha, упавший
  шаг, лог джоба, ретраи), PipelineWatcher — опрос только при незавершённых push,
  сообщение о push обновляется CI ⏳ → ✅/❌ → CD 🚀/⛔, автору — отдельные уведомления
- деплой сервисов — только после зелёного CI; старый push не откатывает сервис,
  который развернёт более новый; после рестарта бот продолжает с того же места
- CI_PROVIDER=none — прежнее поведение (деплой сразу после push)
- шаблон tgit-uploads: .gitea/workflows/check.yml; .gitea/ защищён от правок из бота
- миграция 0002: старые push помечены skipped, чтобы не слать им «таймаут CI»
- ADR-016, ARCHITECTURE §2.5/§5.2, README, DEPLOY, .env.example

Refs #11

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/pipeline/__init__.py

@@ -0,0 +1 @@
+"""CI/CD of the uploads repo after a push: wait for the CI run, then deploy, telling the author (#11)."""

app/pipeline/factory.py

@@ -0,0 +1,10 @@
+from app.config import CISettings
+from app.pipeline.gitea import GiteaCI
+from app.pipeline.provider import CIProvider
+
+
+def make_ci(settings: CISettings) -> CIProvider | None:
+    """None — no CI to wait for: services deploy right after push."""
+    if settings.provider == "gitea":
+        return GiteaCI(settings)
+    return None

app/pipeline/gitea.py

@@ -0,0 +1,118 @@
+"""Gitea Actions as the uploads CI (ADR-016). Unlike SourceCraft, a Gitea run carries `head_sha` — no meta cube."""
+
+import asyncio
+import logging
+import re
+from collections.abc import Awaitable, Callable
+from typing import Any
+
+import httpx
+
+from app.config import CISettings
+from app.db.models import PipelineStatus
+from app.pipeline.provider import CIRun
+
+log = logging.getLogger(__name__)
+
+_TIMESTAMP = re.compile(r"^\d{4}-\d\d-\d\dT[\d:.]+Z ?", re.MULTILINE)
+_FAILED = {"failure", "cancelled"}
+
+Sleep = Callable[[float], Awaitable[None]]
+
+
+class GiteaError(Exception):
+    pass
+
+
+def _status(status: str, conclusion: str | None) -> PipelineStatus:
+    if status != "completed":
+        return PipelineStatus.RUNNING if status == "in_progress" else PipelineStatus.PENDING  # queued · waiting
+    if conclusion == "success":
+        return PipelineStatus.SUCCESS
+    if conclusion == "skipped":
+        return PipelineStatus.SKIPPED
+    return PipelineStatus.FAILED  # failure · cancelled
+
+
+class GiteaCI:
+    def __init__(
+        self,
+        settings: CISettings,
+        transport: httpx.AsyncBaseTransport | None = None,
+        sleep: Sleep = asyncio.sleep,
+    ) -> None:
+        self._s = settings
+        self._sleep = sleep
+        self._base = f"/api/v1/repos/{settings.repo}/actions"
+        self._http = httpx.AsyncClient(
+            base_url=settings.api_url,
+            headers={"Authorization": f"token {settings.token.get_secret_value()}"},
+            timeout=settings.timeout,
+            transport=transport,
+        )
+
+    async def aclose(self) -> None:
+        await self._http.aclose()
+
+    async def find(self, sha: str) -> CIRun | None:
+        data = await self._get(f"{self._base}/runs", params={"head_sha": sha, "limit": 20})
+        # path is "<file>@<ref>"; other workflows of the repo (if any) don't gate the deploy
+        runs = [r for r in data.get("workflow_runs") or [] if str(r.get("path", "")).split("@")[0] == self._s.workflow]
+        if not runs:
+            return None
+        run = max(runs, key=lambda r: int(r["id"]))  # a re-run is a new id
+        status = _status(str(run.get("status")), run.get("conclusion"))
+        failed_step = await self._failed_step(run["id"]) if status is PipelineStatus.FAILED else None
+        return CIRun(str(run["id"]), status, str(run.get("html_url", "")), failed_step)
+
+    async def failure_log(self, run: CIRun) -> str:
+        job = await self._failed_job(run.id)
+        if job is None:
+            return ""
+        text = await self._get_text(f"{self._base}/jobs/{job['id']}/logs")
+        return _TIMESTAMP.sub("", text)
+
+    async def _failed_job(self, run_id: object) -> dict[str, Any] | None:
+        data = await self._get(f"{self._base}/runs/{run_id}/jobs")
+        jobs = data.get("jobs") or []
+        return next((j for j in jobs if j.get("conclusion") in _FAILED), None)
+
+    async def _failed_step(self, run_id: object) -> str | None:
+        job = await self._failed_job(run_id)
+        if job is None:
+            return None
+        step = next((s for s in job.get("steps") or [] if s.get("conclusion") in _FAILED), None)
+        if job.get("conclusion") == "cancelled" and step is None:
+            return f"{job.get('name')} / cancelled"
+        return f"{job.get('name')} / {step['name']}" if step else str(job.get("name"))
+
+    async def _request(self, path: str, params: dict[str, Any] | None = None) -> httpx.Response:
+        attempts = self._s.max_attempts
+        for attempt in range(1, attempts + 1):
+            try:
+                response = await self._http.get(path, params=params)
+            except httpx.TransportError as e:
+                if attempt == attempts:
+                    raise GiteaError(f"{path}: {type(e).__name__}") from e
+                log.warning("gitea %s: %s, retry %d/%d", path, type(e).__name__, attempt, attempts - 1)
+            else:
+                if response.status_code < 500 or attempt == attempts:
+                    if response.status_code >= 400:
+                        raise GiteaError(f"{path}: HTTP {response.status_code}")
+                    return response
+                log.warning("gitea %s: HTTP %d, retry %d/%d", path, response.status_code, attempt, attempts - 1)
+            await self._sleep(min(0.5 * 2 ** (attempt - 1), 8.0))
+        raise AssertionError("unreachable")
+
+    async def _get(self, path: str, params: dict[str, Any] | None = None) -> dict[str, Any]:
+        response = await self._request(path, params)
+        try:
+            body = response.json()
+        except ValueError as e:
+            raise GiteaError(f"{path}: not JSON") from e
+        if not isinstance(body, dict):
+            raise GiteaError(f"{path}: expected a JSON object")
+        return body
+
+    async def _get_text(self, path: str) -> str:
+        return (await self._request(path)).text

app/pipeline/provider.py

@@ -0,0 +1,26 @@
+"""What the watcher needs from a CI: the run of a commit and the log of what failed. Vendors live next to it."""
+
+from dataclasses import dataclass
+from typing import Protocol
+
+from app.db.models import PipelineStatus
+
+
+@dataclass(frozen=True)
+class CIRun:
+    id: str
+    status: PipelineStatus  # pending · running · success · failed · skipped
+    url: str
+    failed_step: str | None = None  # "job / step" of the first failure
+
+
+class CIProvider(Protocol):
+    async def find(self, sha: str) -> CIRun | None:
+        """The newest run of the uploads workflow for this commit; None — not started (yet)."""
+        ...
+
+    async def failure_log(self, run: CIRun) -> str:
+        """Log of the failed job, plain text without runner timestamps."""
+        ...
+
+    async def aclose(self) -> None: ...

app/pipeline/watcher.py

@@ -0,0 +1,361 @@
+"""PipelineWatcher: after a push — wait for the uploads CI, then deploy through tgit-runner (SPEC C-24, C-30…C-34).
+
+The truth is in the DB (`pushes.ci_status/cd_status`, `pipeline_runs`), so a restarted bot picks up where it stopped.
+The CI is polled only while some push waits for it (ADR-003); with nothing to wait for the loop sleeps until `wake()`.
+
+The push message is edited in place (CI ⏳ → ✅/❌ → CD ⏳ → 🚀/⛔); the author also gets separate messages for the
+outcomes — an edit doesn't notify. A failed deploy is an alert ⛔ for the whole team (raised by the Deployer).
+"""
+
+import asyncio
+import contextlib
+import html
+import logging
+from collections.abc import Awaitable, Callable
+from datetime import datetime, timedelta
+
+from aiogram import Bot
+from aiogram.enums import ParseMode
+from aiogram.exceptions import TelegramAPIError, TelegramBadRequest
+from aiogram.types import BufferedInputFile, ReplyParameters
+from sqlalchemy import or_, select, update
+
+from app.db.models import PipelineRun, PipelineStatus, Push, Stage, as_utc, utcnow
+from app.db.session import SessionMaker
+from app.pipeline.provider import CIProvider, CIRun
+from app.runner.deployer import Deployer, Outcome, changed_services
+from app.texts import Texts
+
+log = logging.getLogger(__name__)
+
+WAITING = (PipelineStatus.PENDING, PipelineStatus.RUNNING)
+PASSED = (PipelineStatus.SUCCESS, PipelineStatus.SKIPPED)
+RUNNER = "tgit-runner"  # run_slug of the CD stage: the deploy is the bot's own, not a CI run
+LOG_CHARS = 3000  # inline tail; the whole log goes as a file when it is longer
+MAX_LOG_BYTES = 5 * 1024 * 1024  # SPEC L-5
+
+
+def initial_status(ci: bool, deploys: bool, paths: list[str]) -> tuple[PipelineStatus, PipelineStatus]:
+    """(ci, cd) of a fresh push: SKIPPED where there is nothing to wait for."""
+    cd = PipelineStatus.PENDING if deploys and changed_services(paths) else PipelineStatus.SKIPPED
+    return (PipelineStatus.PENDING if ci else PipelineStatus.SKIPPED), cd
+
+
+def render(
+    texts: Texts,
+    push: Push,
+    ci_run: PipelineRun | None = None,
+    cd_run: PipelineRun | None = None,
+    commit_url_template: str = "",
+    timeout_minutes: float = 30,
+) -> str:
+    """The push message: what was pushed, then a line per stage that applies to it."""
+    t = texts.pipeline
+    first_line = push.message.splitlines()[0] if push.message else ""
+    lines = [texts.commit.pushed.format(sha=push.sha[:7], message=first_line)]
+    if commit_url_template:
+        lines.append(commit_url_template.format(sha=push.sha))
+    match push.ci_status:
+        case PipelineStatus.PENDING:
+            lines.append(t.ci_pending)
+        case PipelineStatus.RUNNING:
+            lines.append(t.ci_running)
+        case PipelineStatus.SUCCESS:
+            lines.append(t.ci_success)
+        case PipelineStatus.FAILED:
+            lines.append(t.ci_failed.format(step=(ci_run and ci_run.failed_cube) or "?"))
+        case PipelineStatus.TIMEOUT:
+            lines.append(t.ci_timeout.format(minutes=round(timeout_minutes)))
+    if ci_run is not None and ci_run.url:
+        lines.append(ci_run.url)
+
+    services = ", ".join(changed_services(push.paths))
+    failed = (cd_run.failed_cube or "") if cd_run else ""
+    match push.cd_status:
+        case PipelineStatus.PENDING if push.ci_status in WAITING:
+            lines.append(t.cd_waiting.format(services=services))
+        case PipelineStatus.PENDING | PipelineStatus.RUNNING:
+            lines.append(t.cd_running.format(services=services))
+        case PipelineStatus.SUCCESS:
+            lines.append(t.cd_success.format(services=services))
+        case PipelineStatus.FAILED:
+            lines.append(t.cd_failed.format(services=failed.replace(",", ", ")))
+        case PipelineStatus.SKIPPED if services and push.ci_status not in PASSED:
+            lines.append(t.cd_not_run)
+        case PipelineStatus.SKIPPED if cd_run is not None:
+            lines.append(t.cd_superseded)
+    return "\n".join(lines)
+
+
+class PipelineWatcher:
+    def __init__(
+        self,
+        sessions: SessionMaker,
+        bot: Bot,
+        texts: Texts,
+        ci: CIProvider | None,
+        deployer: Deployer | None,
+        *,
+        commit_url_template: str = "",
+        poll_interval: float = 10.0,
+        max_poll_interval: float = 60.0,
+        run_timeout: timedelta = timedelta(minutes=30),
+        clock: Callable[[], datetime] = utcnow,
+    ) -> None:
+        self._sessions = sessions
+        self._bot = bot
+        self._texts = texts
+        self._ci = ci
+        self._deployer = deployer
+        self._url = commit_url_template
+        self._interval = poll_interval
+        self._max_interval = max_poll_interval
+        self._timeout = run_timeout
+        self._clock = clock
+        self._wake = asyncio.Event()
+        self._deploying: dict[int, asyncio.Task[None]] = {}
+
+    def initial_status(self, paths: list[str]) -> tuple[PipelineStatus, PipelineStatus]:
+        return initial_status(self._ci is not None, self._deployer is not None, paths)
+
+    def wake(self) -> None:
+        """A new push: look at it now rather than after the current pause."""
+        self._wake.set()
+
+    async def run(self) -> None:
+        delay = self._interval
+        while True:
+            self._wake.clear()  # before reading the DB: a push recorded during the tick wakes the next wait
+            try:
+                polling = await self.tick()
+                delay = self._interval
+            except Exception:
+                log.exception("pipeline watcher tick")
+                polling = True
+                delay = min(delay * 2, self._max_interval)
+            if polling:
+                with contextlib.suppress(TimeoutError):
+                    await asyncio.wait_for(self._wake.wait(), timeout=delay)
+            else:
+                await self._wake.wait()
+
+    async def tick(self) -> bool:
+        """One pass over unfinished pushes. True — some push still waits for CI, poll again soon."""
+        async with self._sessions() as s:
+            pushes = list(
+                await s.scalars(
+                    select(Push)
+                    .where(
+                        or_(
+                            Push.ci_status.in_(WAITING),
+                            Push.ci_status.in_(PASSED) & Push.cd_status.in_(WAITING),
+                        )
+                    )
+                    .order_by(Push.id)
+                )
+            )
+        polling = False
+        for push in pushes:
+            if push.ci_status in WAITING:
+                await self._poll_ci(push)
+                polling |= push.ci_status in WAITING
+            elif push.id not in self._deploying:
+                await self._start_cd(push)
+        return polling
+
+    async def drain(self) -> None:
+        """Wait for the deploys in flight."""
+        while self._deploying:
+            await asyncio.gather(*self._deploying.values(), return_exceptions=True)
+
+    async def aclose(self) -> None:
+        """Stop deploys in flight: their push stays `cd=running` in the DB, and the next start deploys it again."""
+        tasks = list(self._deploying.values())
+        for task in tasks:
+            task.cancel()
+        await asyncio.gather(*tasks, return_exceptions=True)
+
+    # ---- CI ------------------------------------------------------------------------------------------------------
+
+    async def _poll_ci(self, push: Push) -> None:
+        if self._ci is None:  # CI switched off since the push: don't keep it waiting forever
+            await self._set(push, ci=PipelineStatus.SKIPPED)
+            await self._refresh(push)
+            return
+        run = await self._ci.find(push.sha)
+        if run is None or run.status in WAITING:
+            if self._clock() - as_utc(push.created_at) > self._timeout:
+                await self._finish_ci(push, PipelineStatus.TIMEOUT, run)
+            elif run is not None and run.status is not push.ci_status:
+                await self._save_run(push, Stage.CI, run.id, run.status, None, run.url)
+                await self._set(push, ci=run.status)
+                await self._refresh(push)
+            return
+        await self._finish_ci(push, run.status, run)
+
+    async def _finish_ci(self, push: Push, status: PipelineStatus, run: CIRun | None) -> None:
+        if run is not None:
+            await self._save_run(push, Stage.CI, run.id, status, run.failed_step, run.url)
+        passed = status in PASSED
+        cd = push.cd_status if passed or push.cd_status is not PipelineStatus.PENDING else PipelineStatus.SKIPPED
+        await self._set(push, ci=status, cd=cd)
+        await self._refresh(push)
+        t = self._texts.pipeline
+        sha = push.sha[:7]
+        url = html.escape(run.url) if run else ""
+        if passed:
+            await self._notify(push, t.notify_ci_ok.format(sha=sha, url=url))
+            if push.cd_status is PipelineStatus.PENDING:
+                await self._start_cd(push)
+        elif status is PipelineStatus.TIMEOUT:
+            minutes = round(self._timeout.total_seconds() / 60)
+            await self._notify(push, t.notify_ci_timeout.format(sha=sha, minutes=minutes))
+        else:
+            assert run is not None
+            await self._notify_failure(push, run)
+
+    async def _notify_failure(self, push: Push, run: CIRun) -> None:
+        t = self._texts.pipeline
+        step = html.escape(run.failed_step or "?")
+        text = t.notify_ci_failed.format(step=step, sha=push.sha[:7], url=html.escape(run.url))
+        assert self._ci is not None
+        try:
+            full = await self._ci.failure_log(run)
+        except Exception as e:  # the verdict matters more than its log
+            log.warning("ci log for %s: %s", push.sha[:7], e)
+            full = ""
+        tail = full.strip()[-LOG_CHARS:]
+        if tail:
+            text += f"\n<pre>{html.escape(tail)}</pre>"
+        await self._notify(push, text)
+        if len(full.strip()) > LOG_CHARS:
+            data = full.encode()[-MAX_LOG_BYTES:]
+            await self._send(
+                lambda: self._bot.send_document(
+                    push.chat_id,
+                    BufferedInputFile(data, f"ci-{push.sha[:7]}.log"),
+                    caption=t.log_file,
+                    reply_parameters=self._reply(push),
+                )
+            )
+
+    # ---- CD ------------------------------------------------------------------------------------------------------
+
+    async def _start_cd(self, push: Push) -> None:
+        if self._deployer is None:  # the runner was switched off since the push
+            await self._set(push, cd=PipelineStatus.SKIPPED)
+            await self._refresh(push)
+            return
+        services = await self._not_superseded(push)
+        if not services:
+            await self._save_run(push, Stage.CD, RUNNER, PipelineStatus.SKIPPED, None, None)
+            await self._set(push, cd=PipelineStatus.SKIPPED)
+            await self._refresh(push)
+            return
+        await self._set(push, cd=PipelineStatus.RUNNING)
+        await self._refresh(push)
+        task = asyncio.create_task(self._deploy(push, services))
+        self._deploying[push.id] = task
+        task.add_done_callback(lambda _: self._deploying.pop(push.id, None))
+
+    async def _not_superseded(self, push: Push) -> list[str]:
+        """Services of this push that no newer passed push will deploy anyway (CI runs may finish out of order)."""
+        services = changed_services(push.paths)
+        async with self._sessions() as s:
+            newer = await s.scalars(select(Push.paths).where(Push.id > push.id, Push.ci_status.in_(PASSED)))
+            taken = {svc for paths in newer for svc in changed_services(paths)}
+        return [svc for svc in services if svc not in taken]
+
+    async def _deploy(self, push: Push, services: list[str]) -> None:
+        assert self._deployer is not None
+        outcomes: list[Outcome] = []
+
+        async def report(o: Outcome) -> None:
+            outcomes.append(o)
+            t = self._texts.pipeline
+            name = html.escape(o.service)
+            if o.result.ok and o.removed:
+                await self._notify(push, t.removed.format(service=name))
+            elif not o.result.ok:
+                text = t.deploy_failed.format(service=name, reason=html.escape(o.result.summary))
+                if o.result.tail:
+                    text += f"\n<pre>{html.escape(o.result.tail[-LOG_CHARS:])}</pre>"
+                await self._notify(push, text)
+
+        try:
+            await self._deployer.deploy(push.sha, services, report)
+        except Exception:
+            log.exception("deploy of push %s", push.sha[:7])
+        failed = [o.service for o in outcomes if not o.result.ok] + services[len(outcomes) :]
+        deployed = [o.service for o in outcomes if o.result.ok and not o.removed]
+        status = PipelineStatus.FAILED if failed else PipelineStatus.SUCCESS
+        await self._save_run(push, Stage.CD, RUNNER, status, ",".join(failed)[:100] or None, None)
+        await self._set(push, cd=status)
+        await self._refresh(push)
+        if deployed:
+            names = ", ".join(deployed)
+            await self._notify(push, self._texts.pipeline.notify_cd_ok.format(services=names, sha=push.sha[:7]))
+
+    # ---- state and messages --------------------------------------------------------------------------------------
+
+    async def _set(self, push: Push, ci: PipelineStatus | None = None, cd: PipelineStatus | None = None) -> None:
+        values = {}
+        if ci is not None:
+            values["ci_status"] = push.ci_status = ci
+        if cd is not None:
+            values["cd_status"] = push.cd_status = cd
+        async with self._sessions.begin() as s:
+            await s.execute(update(Push).where(Push.id == push.id).values(**values))
+
+    async def _save_run(
+        self, push: Push, stage: Stage, slug: str, status: PipelineStatus, failed: str | None, url: str | None
+    ) -> None:
+        async with self._sessions.begin() as s:
+            row = await s.scalar(select(PipelineRun).where(PipelineRun.push_id == push.id, PipelineRun.stage == stage))
+            if row is None:
+                row = PipelineRun(push_id=push.id, stage=stage)
+                s.add(row)
+            row.run_slug, row.status, row.failed_cube, row.url = slug, status, failed, url
+
+    async def _runs(self, push: Push) -> dict[Stage, PipelineRun]:
+        async with self._sessions() as s:
+            rows = await s.scalars(select(PipelineRun).where(PipelineRun.push_id == push.id))
+            return {r.stage: r for r in rows}
+
+    async def _refresh(self, push: Push) -> None:
+        """Re-render the push message from the DB."""
+        if push.message_id is None:
+            return
+        runs = await self._runs(push)
+        text = render(
+            self._texts,
+            push,
+            runs.get(Stage.CI),
+            runs.get(Stage.CD),
+            self._url,
+            self._timeout.total_seconds() / 60,
+        )
+        message_id = push.message_id
+        await self._send(lambda: self._bot.edit_message_text(text, chat_id=push.chat_id, message_id=message_id))
+
+    def _reply(self, push: Push) -> ReplyParameters | None:
+        if push.message_id is None:
+            return None
+        return ReplyParameters(message_id=push.message_id, allow_sending_without_reply=True)
+
+    async def _notify(self, push: Push, text: str) -> None:
+        await self._send(
+            lambda: self._bot.send_message(
+                push.chat_id, text, parse_mode=ParseMode.HTML, reply_parameters=self._reply(push)
+            )
+        )
+
+    @staticmethod
+    async def _send(call: Callable[[], Awaitable[object]]) -> None:
+        """The status in the DB is what matters: a message that didn't get through is logged, not retried."""
+        try:
+            await call()
+        except TelegramBadRequest as e:  # not modified · deleted by the user · too old to edit
+            log.info("pipeline message: %s", e)
+        except TelegramAPIError as e:  # blocked by the user · flood control
+            log.warning("pipeline message: %s", e)