From c443190bf493631c0a75a68b06b889f23e19f85d Mon Sep 17 00:00:00 2001 From: Lacky1234union Date: Sun, 18 Oct 2026 15:36:29 +0300 Subject: [PATCH] =?UTF-8?q?feat(runner):=20tgit-runner=20=E2=80=94=20?= =?UTF-8?q?=D1=81=D0=B5=D1=80=D0=B2=D0=B8=D1=81=D1=8B=20=D0=B2=20=D0=BF?= =?UTF-8?q?=D0=B5=D1=81=D0=BE=D1=87=D0=BD=D0=B8=D1=86=D0=B5=20=D0=BD=D0=B0?= =?UTF-8?q?=20=D1=85=D0=BE=D1=81=D1=82=D0=B5,=20=D0=B4=D0=B5=D0=BF=D0=BB?= =?UTF-8?q?=D0=BE=D0=B9=20=D0=BF=D0=BE=D1=81=D0=BB=D0=B5=20push=20(#13)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - infra/runner/tgit-runner: deploy/restart/remove/list по SSH forced command; аргументы не исполняются и не глобятся; архив — только файлы и папки, без «..», ≤ 20 МБ, есть main.py; health_url — только localhost - песочница: uid 10001, read-only, tmpfs noexec, cap-drop ALL, no-new-privileges, 128 МБ без swap, 0.25 CPU, 64 PID, --network none (сеть — только по egress-allow админа), лимит 10 сервисов, 2 образа - новая версия стартует рядом (-next) и заменяет старую, только став healthy; иначе удаляется, старая работает, exit 3 - бот: после push git archive изменённых services// → runner; автору — «🚀 работает» или «⛔ не развёрнут» с логом, команде — алерт ⛔; удалённая папка → remove; «♻️ Перезапустить» в «Статусе» для админа (S-5) - examples/escape_attempts: все попытки выбраться — BLOCKED на живом Docker - install.sh, authorized_keys.example; CD бота обновляет runner на хосте Co-Authored-By: Claude Opus 5.5 --- .env.example | 5 + README.md | 9 +- app/bot/routers/commit.py | 31 ++- app/bot/routers/status.py | 46 +++- app/config.py | 13 + app/main.py | 14 +- app/runner/__init__.py | 1 + app/runner/client.py | 85 +++++++ app/runner/deployer.py | 85 +++++++ app/texts.py | 7 + config/texts.yaml | 9 + docker-compose.yml | 3 + docs/ARCHITECTURE.md | 10 +- examples/README.md | 6 + examples/escape_attempts/main.py | 97 ++++++++ infra/README.md | 2 +- infra/deploy/remote.sh | 4 + infra/runner/README.md | 38 +++ infra/runner/authorized_keys.example | 6 + infra/runner/install.sh | 28 +++ infra/runner/tgit-runner | 184 ++++++++++++++ infra/uploads-template/.sourcecraft/ci.yaml | 5 +- tests/runner/__init__.py | 0 tests/runner/fake_docker.py | 83 +++++++ tests/runner/fake_ssh.py | 14 ++ tests/runner/test_deploy.py | 234 ++++++++++++++++++ tests/runner/test_tgit_runner.py | 250 ++++++++++++++++++++ 27 files changed, 1249 insertions(+), 20 deletions(-) create mode 100644 app/runner/__init__.py create mode 100644 app/runner/client.py create mode 100644 app/runner/deployer.py create mode 100644 examples/escape_attempts/main.py create mode 100644 infra/runner/README.md create mode 100644 infra/runner/authorized_keys.example create mode 100755 infra/runner/install.sh create mode 100755 infra/runner/tgit-runner create mode 100644 tests/runner/__init__.py create mode 100755 tests/runner/fake_docker.py create mode 100755 tests/runner/fake_ssh.py create mode 100644 tests/runner/test_deploy.py create mode 100644 tests/runner/test_tgit_runner.py diff --git a/.env.example b/.env.example index 7134415..0204d02 100644 --- a/.env.example +++ b/.env.example @@ -29,6 +29,11 @@ GIT_UPLOADS_URL=git@ssh.sourcecraft.dev:rogachevegorushka/tgit-uploads.git # DOCKER_URL= # DOCKER_LABEL=tgit.managed=true +# --- Runner: деплой сервисов из репозитория загрузок (#13, infra/runner/) --- +# RUNNER_HOST=host.docker.internal # пусто — бот сервисы не разворачивает +# RUNNER_USER=tgit-deploy +# RUNNER_SSH_KEY_B64= # (secret) base64 ключа, прибитого к tgit-runner в authorized_keys + # --- LLM (#17) --- LLM_PROVIDER=mock # mock | openai_compat; при mock агент логов выключен (#18) LLM_BASE_URL= diff --git a/README.md b/README.md index e0ccdc2..96b5ef4 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ | **📜 Логи** | хвост 50/200 строк или 5 мин/1 ч/24 ч; свой grep (`-i -v -E -c -C N`), выполняется в Python, не в shell; длинный ответ — файлом; журнал поисков | L-1…L-6 | | **🤖 Агент** | вопрос словами → фильтр → LLM-страж → LLM строит только `LogQuery` → валидация → grep; показывает построенную команду и строки; атака — отказ и 🛡 админам; 30 вопросов в сутки | L-10…L-17 | | **🚨 Алерты** | 💥 краш, 🧠 OOM, 🔁 цикл рестартов, 🩺 unhealthy, 🔥 перегрузка, 💾 мало места, 🔌 Docker недоступен, 🛡 атака; всем участникам, с хвостом лога и кнопками; повторы — счётчик, «✅ восстановлено» ответом; список за 7 дней | AL-1…AL-6 | +| **🚀 Деплой сервисов** | после push бот отправляет изменённые `services//` в `tgit-runner` на хосте: сборка образа, запуск в песочнице (uid 10001, read-only, без capabilities и сети, 128 МБ / 0.25 CPU / 64 процесса); старая версия заменяется только здоровой новой, иначе ⛔ с логом; «♻️ Перезапустить» в «Статусе» для админа | #13, S-5 | | **CI/CD бота** | Gitea Actions: ruff, mypy strict, миграции up/down/up на PostgreSQL, pytest с порогом покрытия 80 %; автодеплой `dev` на сервер с проверкой health | ADR-015 | Качество: 296 тестов, покрытие 92 %, `mypy --strict`, `ruff`. Telegram в тестах — фейковая сессия aiogram, @@ -148,6 +149,7 @@ make cov # pytest с покрытием (порог 80 %) | git (SSH) | чтение и коммиты в репозиторий загрузок | `git` подпроцессом со своим ключом, без shell | | Docker Engine API | контейнеры, stats, логи | только GET через `docker-socket-proxy` | | OpenAI-совместимый `/chat/completions` | страж и планировщик агента | httpx, `response_format: json_schema` | +| SSH → `tgit-runner` | деплой и перезапуск сервисов на хосте | `ssh` подпроцессом, ключ с forced command | | SourceCraft REST API | статус прогонов CI репозитория загрузок | httpx, клиент готов, в UI — #11 | ## 5. Ограничения и особенности @@ -176,8 +178,8 @@ make cov # pytest с покрытием (порог 80 %) | повтор алерта | в течение 10 мин — счётчик, а не новое сообщение | **Что пока не сделано** (открытые задачи): -- запуск сервисов из репозитория загрузок в песочнице (`--cap-drop ALL`, read-only FS, лимиты) и перезапуск из бота — - `tgit-runner`, #13. Сейчас сервисы поднимаются Compose, как в `examples/`; +- проверки CI репозитория загрузок (ruff, bandit, тесты) перед деплоем: сейчас бот разворачивает сервис сразу после + push, проверив только синтаксис; ожидание зелёного CI появится вместе с #11; - уведомления о прогонах CI/CD репозитория загрузок и алерты 📦/⛔ — #11, ждут возврата к SourceCraft; - «🔕 Тишина на час» для алертов (P2); - prod-окружение: нужен отдельный бот, деплой `main` пропускается, пока его нет. @@ -193,12 +195,13 @@ app/ logs/ grep-движок и журнал agent/ фильтр, страж, планировщик, схема LogQuery alerts/ детекторы, сервис алертов, рассылка, монитор + runner/ SSH-клиент tgit-runner и деплой после push llm/ провайдеры: OpenAI-совместимый, mock sourcecraft/ клиент API прогонов db/ модели, миграции Alembic, очистка config/ тексты интерфейса, пороги проверок и алертов, правила и промпты агента tests/ pytest; фейки Telegram, Docker и LLM examples/ демо-сервисы и сценарии -infra/ деплой на сервер, шаблон репозитория загрузок +infra/ деплой бота, tgit-runner (песочница сервисов), шаблон репозитория загрузок docs/ ТЗ, архитектура, ADR, процесс ``` diff --git a/app/bot/routers/commit.py b/app/bot/routers/commit.py index 3b720d0..8680983 100644 --- a/app/bot/routers/commit.py +++ b/app/bot/routers/commit.py @@ -4,6 +4,7 @@ import base64 import html import io import logging +from collections.abc import Awaitable, Callable from dataclasses import dataclass from enum import StrEnum from typing import Any @@ -33,6 +34,8 @@ from app.repo.validate import ( python_syntax, strip_fence, ) +from app.runner.deployer import Deployer +from app.runner.deployer import Outcome as DeployOutcome from app.texts import CommitTexts, Texts log = logging.getLogger(__name__) @@ -212,7 +215,12 @@ def to_dir_keyboard(texts: Texts) -> InlineKeyboardMarkup: def build_router( - texts: Texts, repo: UploadsRepo, drafts: DraftService, publisher: Publisher, commit_url_template: str = "" + texts: Texts, + repo: UploadsRepo, + drafts: DraftService, + publisher: Publisher, + commit_url_template: str = "", + deployer: Deployer | None = None, ) -> Router: router = Router(name="commit") t = texts.commit @@ -275,6 +283,21 @@ def build_router( ops = {ChangeOp.ADD: t.op_add, ChangeOp.MODIFY: t.op_modify, ChangeOp.DELETE: t.op_delete} return "\n".join(f"{ops[c.op]} /{c.path}" for c in draft.changes) + def deploy_report(msg: Message, sha: str) -> Callable[[DeployOutcome], Awaitable[None]]: + async def report(o: DeployOutcome) -> None: + name = html.escape(o.service) + if o.result.ok and o.removed: + text = t.removed.format(service=name) + elif o.result.ok: + text = t.deployed.format(service=name, sha=sha[:7]) + else: + text = t.deploy_failed.format(service=name, reason=html.escape(o.result.summary)) + if o.result.tail: + text += f"\n
{html.escape(o.result.tail[-3000:])}
" + await msg.answer(text, parse_mode=ParseMode.HTML) + + return report + async def do_push(msg: Message, state: FSMContext, member: User, *, overwrite: bool) -> None: draft = await drafts.get(member.tg_id) if draft is None or not draft.changes or not draft.message: @@ -291,6 +314,12 @@ def build_router( if commit_url_template: lines.append(commit_url_template.format(sha=result.sha)) lines.append(t.ci_pending) + if deployer is not None: + services = deployer.schedule( + result.sha, [c.path for c in draft.changes], deploy_report(msg, result.sha) + ) + if services: + lines.append(t.deploying.format(services=", ".join(services))) await msg.edit_text("\n".join(lines)) await publisher.record(member, result, draft.message, draft, msg.chat.id, msg.message_id) await drafts.clear(member.tg_id) diff --git a/app/bot/routers/status.py b/app/bot/routers/status.py index c18cf81..4af64ba 100644 --- a/app/bot/routers/status.py +++ b/app/bot/routers/status.py @@ -13,7 +13,8 @@ from aiogram.utils.keyboard import InlineKeyboardBuilder from app.bot.menu import Nav, NavAction from app.bot.routers.commit import human_size -from app.db.models import utcnow +from app.db.models import Role, User, utcnow +from app.runner.client import RunnerClient from app.runtime.checks import Checker, CheckName, ContainerReport from app.runtime.docker import Container, Details, DockerClient, DockerError, Stats from app.texts import StatusTexts, Texts @@ -27,6 +28,7 @@ class StatusAct(StrEnum): LIST = "l" CARD = "c" CHECK = "k" + RESTART = "r" class StatusCb(CallbackData, prefix="s"): @@ -75,7 +77,10 @@ def render_list(texts: Texts, containers: list[Container], now: datetime) -> tup return _clip("\n".join(lines)), kb.as_markup() -def render_card(texts: Texts, d: Details, stats: Stats | None, now: datetime) -> tuple[str, InlineKeyboardMarkup]: +def render_card( + texts: Texts, d: Details, stats: Stats | None, now: datetime, can_restart: bool = False +) -> tuple[str, InlineKeyboardMarkup]: + """can_restart: an admin looking at a service deployed by tgit-runner (S-5).""" t = texts.status running = d.state == "running" uptime = duration(t, (now - d.started_at).total_seconds()) if running and d.started_at else t.none @@ -105,11 +110,18 @@ def render_card(texts: Texts, d: Details, stats: Stats | None, now: datetime) -> kb = InlineKeyboardBuilder() kb.button(text=t.check_button, callback_data=StatusCb(a=StatusAct.CHECK, id=d.id[:12])) kb.button(text=t.refresh, callback_data=StatusCb(a=StatusAct.CARD, id=d.id[:12])) + if can_restart and runner_service(d): + kb.button(text=t.restart, callback_data=StatusCb(a=StatusAct.RESTART, id=d.id[:12])) kb.button(text=t.to_list, callback_data=StatusCb(a=StatusAct.LIST)) - kb.adjust(2, 1) + kb.adjust(2, 1, 1) return text, kb.as_markup() +def runner_service(d: Details) -> str | None: + """Only services tgit-runner deployed can be restarted through it — not the bot, its DB or the proxy.""" + return d.labels.get("tgit.service") or None + + def render_report( texts: Texts, reports: list[ContainerReport], now: datetime, back_id: str = "" ) -> tuple[str, InlineKeyboardMarkup]: @@ -141,7 +153,9 @@ def render_report( return _clip("\n".join(lines)), kb.as_markup() -def build_router(texts: Texts, docker: DockerClient | None, checker: Checker | None) -> Router: +def build_router( + texts: Texts, docker: DockerClient | None, checker: Checker | None, runner: RunnerClient | None = None +) -> Router: router = Router(name="status") t = texts.status @@ -150,7 +164,7 @@ def build_router(texts: Texts, docker: DockerClient | None, checker: Checker | N raise DockerError("docker is not configured (DOCKER_URL)") return render_list(texts, await docker.containers(), utcnow()) - async def card_screen(short_id: str) -> tuple[str, InlineKeyboardMarkup]: + async def card_screen(short_id: str, is_admin: bool = False) -> tuple[str, InlineKeyboardMarkup]: if docker is None: raise DockerError("docker is not configured (DOCKER_URL)") cid = await docker.resolve(short_id) @@ -161,7 +175,7 @@ def build_router(texts: Texts, docker: DockerClient | None, checker: Checker | N stats = await docker.stats(cid) except DockerError: log.warning("stats %s unavailable", short_id) - return render_card(texts, details, stats, utcnow()) + return render_card(texts, details, stats, utcnow(), can_restart=is_admin and runner is not None) @router.message(F.text == texts.menu.status) async def open_status(message: Message, state: FSMContext) -> None: @@ -175,7 +189,7 @@ def build_router(texts: Texts, docker: DockerClient | None, checker: Checker | N await message.answer(text, reply_markup=kb) @router.callback_query(StatusCb.filter()) - async def on_action(query: CallbackQuery, callback_data: StatusCb) -> None: + async def on_action(query: CallbackQuery, callback_data: StatusCb, member: User) -> None: if not isinstance(query.message, Message): await query.answer() return @@ -185,7 +199,23 @@ def build_router(texts: Texts, docker: DockerClient | None, checker: Checker | N case StatusAct.LIST: text, kb = await list_screen() case StatusAct.CARD: - text, kb = await card_screen(callback_data.id) + text, kb = await card_screen(callback_data.id, member.role is Role.ADMIN) + case StatusAct.RESTART: + if docker is None or runner is None or member.role is not Role.ADMIN: + await query.answer() + return + service = runner_service(await docker.inspect(await docker.resolve(callback_data.id))) + if service is None: + await query.answer() + return + result = await runner.restart(service) + if not result.ok: + await query.answer(t.restart_failed.format(reason=result.summary)[:200], show_alert=True) + return + await query.answer(t.restarted) + text, kb = await card_screen(callback_data.id, True) + await _edit(msg, text, kb) + return case StatusAct.CHECK: if docker is None or checker is None: raise DockerError("docker is not configured (DOCKER_URL)") diff --git a/app/config.py b/app/config.py index 4dd40ad..bfa4a8a 100644 --- a/app/config.py +++ b/app/config.py @@ -98,6 +98,18 @@ class DockerSettings(BaseSettings): max_log_lines: int = Field(default=50_000, gt=0) +class RunnerSettings(BaseSettings): + model_config = SettingsConfigDict(**_ENV, env_prefix="RUNNER_") + + # SSH to tgit-runner on the host (ADR-006); empty host — services are not deployed from the bot + host: str = "" + port: int = 22 + user: str = "tgit-deploy" + # The bot's runner key, base64; authorized_keys pins it to `command="tgit-runner"` (infra/runner/) + ssh_key_b64: SecretStr | None = None + timeout: float = Field(default=300.0, gt=0) + + class RuntimeSettings(BaseSettings): model_config = _ENV @@ -120,4 +132,5 @@ class Settings(BaseSettings): sourcecraft: SourceCraftSettings = Field(default_factory=SourceCraftSettings) git: GitSettings = Field(default_factory=GitSettings) docker: DockerSettings = Field(default_factory=DockerSettings) + runner: RunnerSettings = Field(default_factory=RunnerSettings) runtime: RuntimeSettings = Field(default_factory=RuntimeSettings) diff --git a/app/main.py b/app/main.py index fc87a3e..f9246c3 100644 --- a/app/main.py +++ b/app/main.py @@ -29,6 +29,8 @@ from app.logs_setup import setup_logging from app.repo.drafts import DraftService from app.repo.git import UploadsRepo from app.repo.publisher import Publisher +from app.runner.client import RunnerClient +from app.runner.deployer import Deployer from app.runtime.checks import Checker, load_checks from app.runtime.docker import DockerClient from app.runtime.snapshots import SnapshotStore, snapshot_loop @@ -47,6 +49,7 @@ def build_dispatcher( checker: Checker | None = None, agent: Agent | None = None, alerts: AlertService | None = None, + runner: RunnerClient | None = None, ) -> Dispatcher: dp = Dispatcher(storage=make_storage(sessions)) gate = AccessMiddleware(access, texts) @@ -54,10 +57,13 @@ def build_dispatcher( dp.message.outer_middleware(gate) dp.callback_query.outer_middleware(gate) dp.include_router(members.build_router(texts, access)) + deployer = Deployer(repo, runner, alerts) if runner else None dp.include_router( - commit.build_router(texts, repo, DraftService(sessions), Publisher(repo, sessions), commit_url_template) + commit.build_router( + texts, repo, DraftService(sessions), Publisher(repo, sessions), commit_url_template, deployer + ) ) - dp.include_router(status.build_router(texts, docker, checker)) + dp.include_router(status.build_router(texts, docker, checker, runner)) dp.include_router(logs.build_router(texts, docker, LogJournal(sessions), agent, alerts)) dp.include_router(alerts_router.build_router(texts, alerts, docker)) dp.include_router(common.build_router(texts, access)) @@ -99,6 +105,7 @@ async def run(settings: Settings) -> None: checker, agent, alerts, + RunnerClient(settings.runner) if settings.runner.host else None, ) rt = settings.runtime background = [ @@ -111,12 +118,13 @@ async def run(settings: Settings) -> None: asyncio.create_task(snapshot_loop(docker, store, settings.docker.snapshot_interval, monitor.on_round)) ) log.info( - "tgit %s starting, %d bootstrap admin(s), telegram %s, docker %s, agent %s", + "tgit %s starting, %d bootstrap admin(s), telegram %s, docker %s, agent %s, runner %s", __version__, len(settings.access.admin_ids), "via proxy" if proxy else "direct", "on" if docker else "off", settings.llm.provider if agent else "off", + settings.runner.host or "off", ) try: await dp.start_polling(bot, handle_signals=True) diff --git a/app/runner/__init__.py b/app/runner/__init__.py new file mode 100644 index 0000000..ea4090e --- /dev/null +++ b/app/runner/__init__.py @@ -0,0 +1 @@ +"""Deploying services of the uploads repo: the bot ships `git archive` to `tgit-runner` on the host (#13).""" diff --git a/app/runner/client.py b/app/runner/client.py new file mode 100644 index 0000000..7bee034 --- /dev/null +++ b/app/runner/client.py @@ -0,0 +1,85 @@ +"""SSH to `tgit-runner`: the key can run nothing else (forced command), so the «command» is just its arguments.""" + +import asyncio +import base64 +import logging +import os +import re +import tempfile +from dataclasses import dataclass +from pathlib import Path + +from app.config import RunnerSettings + +log = logging.getLogger(__name__) + +SERVICE = re.compile(r"^[a-z][a-z0-9-]{1,30}$") # the runner checks the same — refuse early, with a clear reason +SHA = re.compile(r"^[0-9a-f]{7,40}$") + + +@dataclass(frozen=True) +class RunnerResult: + ok: bool + code: int # 0 ok · 2 bad request · 3 not healthy · 4 limit · 5 build failed · 255 ssh · -1 timeout + summary: str # the runner's last line without OK/FAIL + output: str # everything, for logs and the user's log tail + + @property + def tail(self) -> str: + lines = self.output.strip().splitlines()[:-1] + return "\n".join(lines[-20:]) + + +class RunnerClient: + def __init__(self, settings: RunnerSettings) -> None: + self._s = settings + self._key = self._materialize_key() + + def _materialize_key(self) -> Path | None: + if self._s.ssh_key_b64 is None: + return None + key = base64.b64decode(self._s.ssh_key_b64.get_secret_value()) + fd, name = tempfile.mkstemp(prefix="tgit-runner-") + with os.fdopen(fd, "wb") as f: + f.write(key if key.endswith(b"\n") else key + b"\n") + return Path(name) + + def _ssh(self) -> list[str]: + cmd = ["ssh", "-F", "/dev/null", "-p", str(self._s.port), "-o", "BatchMode=yes", "-o", "ConnectTimeout=20"] + cmd += ["-o", "StrictHostKeyChecking=accept-new", "-o", f"UserKnownHostsFile={tempfile.gettempdir()}/tgit-kh"] + if self._key is not None: + cmd += ["-i", str(self._key), "-o", "IdentitiesOnly=yes"] + return [*cmd, f"{self._s.user}@{self._s.host}"] + + async def run(self, *args: str, stdin: bytes = b"") -> RunnerResult: + proc = await asyncio.create_subprocess_exec( + *self._ssh(), + *args, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + ) + try: + out, _ = await asyncio.wait_for(proc.communicate(stdin), timeout=self._s.timeout) + except TimeoutError: + proc.kill() + await proc.wait() + return RunnerResult(False, -1, "timeout", "") + text = out.decode(errors="replace") + last = text.strip().splitlines()[-1] if text.strip() else "" + code = proc.returncode if proc.returncode is not None else -1 + ok = code == 0 and last.startswith("OK") + summary = last.removeprefix("OK ").removeprefix("FAIL ") if last.startswith(("OK", "FAIL")) else last[:200] + if not ok: + log.warning("runner %s: exit %s: %s", args[0] if args else "", code, summary) + return RunnerResult(ok, code, summary, text) + + async def deploy(self, service: str, sha: str, archive: bytes) -> RunnerResult: + if not SERVICE.match(service) or not SHA.match(sha): + return RunnerResult(False, 2, "bad service name or sha", "") + return await self.run("deploy", service, sha, stdin=archive) + + async def restart(self, service: str) -> RunnerResult: + if not SERVICE.match(service): + return RunnerResult(False, 2, "bad service name", "") + return await self.run("restart", service) diff --git a/app/runner/deployer.py b/app/runner/deployer.py new file mode 100644 index 0000000..ef84c4e --- /dev/null +++ b/app/runner/deployer.py @@ -0,0 +1,85 @@ +"""After a push: the services it touched are (re)deployed through tgit-runner, the pusher hears how it went. + +A service = `services//`. Gone from the commit — removed on the host. A refused deploy keeps the old version +running (the runner switches only to a healthy one) and raises ⛔ for the whole team. +""" + +import asyncio +import logging +from collections.abc import Awaitable, Callable, Iterable +from dataclasses import dataclass + +from app.alerts.detectors import Condition +from app.alerts.service import AlertService +from app.db.models import AlertKind +from app.repo.git import GitError, UploadsRepo +from app.runner.client import SERVICE, RunnerClient, RunnerResult + +log = logging.getLogger(__name__) + + +def changed_services(paths: Iterable[str]) -> list[str]: + found = set() + for path in paths: + parts = path.split("/") + if len(parts) >= 3 and parts[0] == "services" and SERVICE.match(parts[1]): + found.add(parts[1]) + return sorted(found) + + +@dataclass(frozen=True) +class Outcome: + service: str + result: RunnerResult + removed: bool = False + + +Report = Callable[[Outcome], Awaitable[None]] + + +class Deployer: + def __init__(self, repo: UploadsRepo, runner: RunnerClient, alerts: AlertService | None = None) -> None: + self._repo = repo + self._runner = runner + self._alerts = alerts + self._lock = asyncio.Lock() # one deploy at a time: builds are heavy for the shared host + self._tasks: set[asyncio.Task[None]] = set() + + def schedule(self, sha: str, paths: Iterable[str], report: Report) -> list[str]: + """Start in the background; returns the services that will be deployed.""" + services = changed_services(paths) + if services: + task = asyncio.create_task(self.deploy(sha, services, report)) + self._tasks.add(task) + task.add_done_callback(self._tasks.discard) + return services + + async def deploy(self, sha: str, services: list[str], report: Report) -> None: + async with self._lock: + for service in services: + try: + outcome = await self._one(sha, service) + except Exception: # one broken service must not stop the others + log.exception("deploy %s @ %s", service, sha[:7]) + outcome = Outcome(service, RunnerResult(False, -1, "internal error", "")) + if not outcome.result.ok and self._alerts is not None: + await self._alerts.event( + Condition( + AlertKind.DEPLOY_REFUSED, + f"tgit-svc-{service}", + {"detail": outcome.result.summary, "sha": sha[:7], "log": outcome.result.tail}, + ) + ) + try: + await report(outcome) + except Exception: + log.exception("deploy report for %s", service) + + async def _one(self, sha: str, service: str) -> Outcome: + tree = f"{sha}:services/{service}" + try: + await self._repo.git("cat-file", "-e", tree) + except GitError: + return Outcome(service, await self._runner.run("remove", service), removed=True) + archive = await self._repo.git("archive", "--format=tar", tree) + return Outcome(service, await self._runner.deploy(service, sha, archive)) diff --git a/app/texts.py b/app/texts.py index a83b075..0dfc6cd 100644 --- a/app/texts.py +++ b/app/texts.py @@ -119,6 +119,10 @@ class CommitTexts(_Strict): push_empty: str push_limit: str push_failed: str + deploying: str + deployed: str + deploy_failed: str + removed: str class StatusTexts(_Strict): @@ -148,6 +152,9 @@ class StatusTexts(_Strict): hours: str minutes: str seconds: str + restart: str + restarted: str + restart_failed: str class LogsTexts(_Strict): diff --git a/config/texts.yaml b/config/texts.yaml index 3b21809..df2ebe0 100644 --- a/config/texts.yaml +++ b/config/texts.yaml @@ -129,6 +129,12 @@ commit: push_empty: "В main уже всё так же — коммитить нечего. Черновик очищен." push_limit: "Лимит — 20 пушей в час. Попробуйте позже." push_failed: "Не удалось запушить: репозиторий недоступен. Черновик сохранён — попробуйте позже." + deploying: "🚀 Разворачиваю: {services}…" + deployed: "🚀 {service} работает · {sha}" + deploy_failed: |- + ⛔ {service} не развёрнут: {reason} + Работает прежняя версия, если она была. + removed: "🗑 {service} удалён из репозитория — остановлен на хосте" status: title: "🩺 Статус · {count} конт. · {time} UTC" @@ -163,6 +169,9 @@ status: hours: "ч" minutes: "мин" seconds: "с" + restart: "♻️ Перезапустить" + restarted: "♻️ Перезапущен" + restart_failed: "Не удалось перезапустить: {reason}" logs: title: "📜 Логи · выберите контейнер" diff --git a/docker-compose.yml b/docker-compose.yml index 6cb565e..c9ba846 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,9 @@ services: DATABASE_URL: postgresql+asyncpg://tgit:${POSTGRES_PASSWORD:-tgit}@postgres:5432/tgit GIT_CLONE_DIR: /data/uploads DOCKER_URL: http://docker-proxy:2375 + # tgit-runner is reached over SSH on the host itself (RUNNER_HOST=host.docker.internal) + extra_hosts: + - host.docker.internal:host-gateway volumes: - uploads:/data depends_on: diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 64a9e92..889cb75 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -203,7 +203,9 @@ SQLAlchemy 2 (async) + Alembic. PostgreSQL в Compose; в тестах — SQLit `authorized_keys` пользователя `tgit-deploy`). Подкоманды: `deploy `, `restart `, `probe ` (HTTP-проба изнутри сети сервиса). Аргументы валидируются: имя `^[a-z][a-z0-9-]{1,30}$`, sha — hex. -`deploy`: забирает архив `services/` по sha из `tgit-uploads` → собирает образ `tgit-svc-:` из +`deploy`: получает архив `services/` в stdin — его делает бот (`git archive :services/`) +сразу после push, так что у хоста нет доступа к репозиторию → проверяет архив (только файлы и папки, без `..`, +≤ 20 МБ, есть `main.py`) → собирает образ `tgit-svc-:` из шаблона (`python:3.12-slim` + `requirements.txt`, `HEALTHCHECK` из `tgit.yaml` сервиса, если задан) → запускает: ``` @@ -216,7 +218,11 @@ docker run -d --name tgit-svc- --label tgit.managed=true --label tgit.s --restart on-failure:3 --log-opt max-size=5m --log-opt max-file=2 ``` -Если новый контейнер не стал healthy за 60 с — откат на предыдущий образ и ненулевой код выхода (CD падает → алерт ⛔). +Новая версия стартует рядом со старой как `tgit-svc--next` и заменяет её, только став healthy (с +`health_url` — по `HEALTHCHECK`, без него — 10 с без рестартов). Иначе новый контейнер удаляется, старый продолжает +работать, код выхода 3 → бот пишет автору push «⛔ не развёрнут» с хвостом лога и поднимает алерт ⛔ всей команде. +Сервис, папка которого исчезла из коммита, удаляется (`remove`). Перезапуск — `restart` из «Статуса» (админ). +Установка и обновление — [`infra/runner/README.md`](../infra/runner/README.md). ## 5. CI/CD diff --git a/examples/README.md b/examples/README.md index e569993..b04d0b3 100644 --- a/examples/README.md +++ b/examples/README.md @@ -15,6 +15,12 @@ После `make demo-down` открытые алерты закрываются сообщением «✅ Восстановлено». +## Песочница — `tgit-runner` + +[`escape_attempts/`](escape_attempts/) — сервис, который пробует выбраться: запись в код и `/etc`, сеть, DNS, +docker.sock, `setuid(0)`, raw-сокет, 200 процессов, запуск из `/tmp`. Положите его в `services/escape/` репозитория +загрузок — в «📜 Логи» каждая строка должна быть `BLOCKED`. + ## Коммит — «📝 Коммит» | Файл | Что показывает | diff --git a/examples/escape_attempts/main.py b/examples/escape_attempts/main.py new file mode 100644 index 0000000..dce0b85 --- /dev/null +++ b/examples/escape_attempts/main.py @@ -0,0 +1,97 @@ +"""Tries to break out of the tgit-runner sandbox and reports each attempt. Every line should say BLOCKED. + +Deploy it like any service (put it into services/escape/ of the uploads repo) and read its log in «📜 Логи». +Memory is not probed here: past 128 MB the kernel OOM-kills the process — the limit works, but the report would die. +""" + +import os +import socket +import subprocess +import time + + +def attempt(name: str, action) -> None: # type: ignore[no-untyped-def] + try: + detail = action() + except Exception as e: # each failure is the expected outcome + print(f"BLOCKED {name}: {type(e).__name__}: {e}", flush=True) + return + print(f"ALLOWED {name}: {detail}", flush=True) + + +def write_code() -> str: + with open("/app/pwned.txt", "w") as f: + f.write("x") + return "wrote into /app" + + +def write_root() -> str: + with open("/etc/pwned", "w") as f: + f.write("x") + return "wrote into /etc" + + +def internet() -> str: + socket.create_connection(("1.1.1.1", 443), timeout=3).close() + return "reached 1.1.1.1:443" + + +def dns() -> str: + return socket.gethostbyname("example.com") + + +def docker_socket() -> str: + s = socket.socket(socket.AF_UNIX) + s.connect("/var/run/docker.sock") + return "connected to docker.sock" + + +def become_root() -> str: + os.setuid(0) + return "setuid(0) worked" + + +def raw_socket() -> str: + socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_ICMP) + return "opened a raw socket" + + +def fork_bomb() -> str: + procs = [] + try: + for _ in range(200): + procs.append(subprocess.Popen(["sleep", "30"])) + finally: + started = len(procs) + for p in procs: + p.kill() + if started >= 200: + return "started 200 processes" + raise RuntimeError(f"stopped after {started} processes") + + +def exec_tmp() -> str: + path = "/tmp/run.sh" + with open(path, "w") as f: + f.write("#!/bin/sh\necho hi\n") + os.chmod(path, 0o755) + return subprocess.run([path], capture_output=True, check=True, text=True).stdout.strip() + + +if __name__ == "__main__": + print(f"uid={os.getuid()} gid={os.getgid()}", flush=True) + for name, fn in [ + ("write into the code dir", write_code), + ("write into /etc", write_root), + ("connect to the internet", internet), + ("resolve DNS", dns), + ("talk to docker.sock", docker_socket), + ("setuid(0)", become_root), + ("raw socket", raw_socket), + ("200 processes", fork_bomb), + ("execute from /tmp", exec_tmp), + ]: + attempt(name, fn) + print("done", flush=True) + while True: + time.sleep(3600) diff --git a/infra/README.md b/infra/README.md index c614ea4..ac7e83a 100644 --- a/infra/README.md +++ b/infra/README.md @@ -3,5 +3,5 @@ | Каталог | Что будет | Задача | |---|---|---| | [`deploy/`](deploy/) | `deploy.sh` + `remote.sh` — деплой бота на сервер (CD и руками), см. [`docs/dev/DEPLOY.md`](../docs/dev/DEPLOY.md) | #12 | -| `runner/` | `tgit-runner` — песочница и запуск сервисов на хосте | #13 | +| [`runner/`](runner/) | `tgit-runner` — песочница и запуск сервисов на хосте, установка | #13 | | [`uploads-template/`](uploads-template/) | шаблон [`tgit-uploads`](https://sourcecraft.dev/rogachevegorushka/tgit-uploads): CI/CD, служебные скрипты `.tgit/`, `ruff.toml`, демо-сервис `hello` — правки вносить здесь и переносить в репозиторий загрузок | #7 | diff --git a/infra/deploy/remote.sh b/infra/deploy/remote.sh index aa8627d..db34566 100755 --- a/infra/deploy/remote.sh +++ b/infra/deploy/remote.sh @@ -35,5 +35,9 @@ if [ "$status" != healthy ]; then exit 1 fi echo "$SHA" > ../REVISION +# Keep tgit-runner in step with the bot (installed once by infra/runner/install.sh; absent — nothing to update) +if [ -d /srv/tgit/runner ] && [ -w /srv/tgit/runner ]; then + install -m 755 infra/runner/tgit-runner /srv/tgit/runner/tgit-runner +fi docker image prune -f >/dev/null echo "bot healthy @ $SHA ($ENV)" diff --git a/infra/runner/README.md b/infra/runner/README.md new file mode 100644 index 0000000..fd154a0 --- /dev/null +++ b/infra/runner/README.md @@ -0,0 +1,38 @@ +# infra/runner — `tgit-runner` + +Запуск сервисов из репозитория загрузок (`services//`) на хосте в песочнице — ADR-005, ADR-006, +[ARCHITECTURE §4](../../docs/ARCHITECTURE.md). + +``` +бот: push ─► git archive :services/ ─► ssh tgit-deploy@host deploy (архив — в stdin) + │ forced command: только tgit-runner +host: tgit-runner ─► проверка аргументов и архива ─► docker build ─► docker run …-next (песочница) + ─► healthy за 60 с? ─► да: заменить старый контейнер · нет: удалить новый, старый работает, exit 3 +``` + +| Команда | Что делает | +|---|---| +| `deploy ` | собрать образ `tgit-svc-:` и запустить, старый заменяется только здоровым новым | +| `restart ` | перезапуск (кнопка «♻️ Перезапустить» в «Статусе», только админ) | +| `remove ` | сервис удалён из репозитория — остановить и удалить образы | +| `list` | контейнеры сервисов | + +**Песочница:** uid 10001, read-only FS, `/tmp` — tmpfs 16 МБ с `noexec`, `--cap-drop ALL`, `no-new-privileges`, +128 МБ без swap, 0.25 CPU, 64 процесса, `--network none` (интернет — только если `network: true` в `tgit.yaml` +**и** имя сервиса есть в `/srv/tgit/runner/egress-allow`), без Docker-сокета, `--restart on-failure:3`, ротация логов. +Не больше 10 сервисов на хост; хранятся два последних образа сервиса. + +**Проверено** [`examples/escape_attempts`](../../examples/escape_attempts/): запись в код и `/etc`, сеть, DNS, +docker.sock, `setuid(0)`, raw-сокет, 200 процессов, запуск из `/tmp` — всё `BLOCKED`. + +**Установка** (один раз, root на хосте): + +```bash +ssh-keygen -t ed25519 -N '' -C tgit-runner -f tgit-runner-key # ключ бота для раннера +sudo infra/runner/install.sh tgit-runner-key.pub tgit-deploy +# в .env бота на сервере: +# RUNNER_HOST=host.docker.internal RUNNER_SSH_KEY_B64=$(base64 -w0 tgit-runner-key) +``` + +Дальше скрипт обновляется сам: деплой бота (`infra/deploy/remote.sh`) копирует свежий `tgit-runner`, если раннер +установлен. Тесты — `tests/runner/` (фейковые `docker` и `ssh`, сам скрипт настоящий). diff --git a/infra/runner/authorized_keys.example b/infra/runner/authorized_keys.example new file mode 100644 index 0000000..7aa8032 --- /dev/null +++ b/infra/runner/authorized_keys.example @@ -0,0 +1,6 @@ +# ~tgit-deploy/.ssh/authorized_keys on the host. `restrict` = no-pty, no port/agent/X11 forwarding; +# command= makes the key run tgit-runner and nothing else, the requested command arrives as SSH_ORIGINAL_COMMAND. +command="/srv/tgit/runner/tgit-runner",restrict ssh-ed25519 AAAA…bot-runner-key… tgit-runner + +# The CD key of the bot itself stays unrestricted (it deploys the bot; Gitea secret DEPLOY_SSH_KEY): +ssh-ed25519 AAAA…cd-key… tgit-cd@gitea diff --git a/infra/runner/install.sh b/infra/runner/install.sh new file mode 100755 index 0000000..35848e2 --- /dev/null +++ b/infra/runner/install.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# One-time setup of tgit-runner on the host (as root). Safe to re-run. +# sudo infra/runner/install.sh /path/to/tgit-runner-key.pub [deploy-user] +# The key is the bot's runner key (RUNNER_SSH_KEY_B64 holds its private half); it is pinned to the runner only. +set -euo pipefail +PUB=${1:?usage: install.sh [user]} +USER_NAME=${2:-tgit-deploy} +DIR=/srv/tgit/runner +HERE=$(cd "$(dirname "$0")" && pwd) + +id "$USER_NAME" >/dev/null +install -d -o "$USER_NAME" -g "$USER_NAME" -m 700 "$DIR" +install -o "$USER_NAME" -g "$USER_NAME" -m 755 "$HERE/tgit-runner" "$DIR/tgit-runner" +touch "$DIR/egress-allow" && chown "$USER_NAME:" "$DIR/egress-allow" && chmod 600 "$DIR/egress-allow" + +HOME_DIR=$(getent passwd "$USER_NAME" | cut -d: -f6) +install -d -o "$USER_NAME" -g "$USER_NAME" -m 700 "$HOME_DIR/.ssh" +AK=$HOME_DIR/.ssh/authorized_keys +touch "$AK" && chown "$USER_NAME:" "$AK" && chmod 600 "$AK" +KEY=$(awk '{print $1" "$2}' "$PUB") +LINE="command=\"$DIR/tgit-runner\",restrict $KEY tgit-runner" +if grep -qF "$KEY" "$AK"; then + echo "key already in $AK — leaving it" +else + echo "$LINE" >>"$AK" + echo "added the runner key to $AK (forced command)" +fi +echo "runner installed in $DIR; allow internet for a service: echo >> $DIR/egress-allow" diff --git a/infra/runner/tgit-runner b/infra/runner/tgit-runner new file mode 100755 index 0000000..c73be45 --- /dev/null +++ b/infra/runner/tgit-runner @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +# tgit-runner — the only command the bot's SSH key may run on the host (ADR-006, ARCHITECTURE §4). +# authorized_keys: command="/srv/tgit/runner/tgit-runner",restrict ssh-ed25519 AAAA… tgit-runner +# +# deploy stdin: tar of services// (files at the archive root) +# restart +# remove +# list +# +# Exit codes: 0 ok · 2 bad request · 3 not healthy (old container kept) · 4 limit · 5 build failed. +# The last line of output is machine-readable: «OK …» or «FAIL ». +set -euo pipefail +umask 077 +export LC_ALL=C + +STATE=${TGIT_RUNNER_STATE:-/srv/tgit/runner} +MAX_SERVICES=${TGIT_MAX_SERVICES:-10} +HEALTH_TIMEOUT=${TGIT_HEALTH_TIMEOUT:-60} +SETTLE=${TGIT_SETTLE:-10} # no healthcheck: must stay up this long without restarts +MAX_TAR=${TGIT_MAX_TAR:-$((20 * 1024 * 1024))} +BASE_IMAGE=${TGIT_BASE_IMAGE:-python:3.12-slim} +EGRESS_NET=tgit-svc-egress +NAME_RE='^[a-z][a-z0-9-]{1,30}$' +SHA_RE='^[0-9a-f]{7,40}$' +HEALTH_RE='^http://(localhost|127\.0\.0\.1):[0-9]{2,5}(/[A-Za-z0-9._/-]*)?$' + +fail() { echo "FAIL $2"; exit "$1"; } + +# Called through SSH the arguments come in SSH_ORIGINAL_COMMAND: split on spaces, never evaluated, never globbed +if [ -n "${SSH_ORIGINAL_COMMAND+x}" ]; then + set -f + read -r -a ARGS <<<"$SSH_ORIGINAL_COMMAND" + set +f +else + ARGS=("$@") +fi +[ "${#ARGS[@]}" -ge 1 ] || fail 2 "usage: deploy|restart|remove|list" +CMD=${ARGS[0]} + +need_service() { + [ "${#ARGS[@]}" -eq "$1" ] || fail 2 "wrong number of arguments for $CMD" + SVC=${ARGS[1]} + [[ $SVC =~ $NAME_RE ]] || fail 2 "bad service name" + NAME=tgit-svc-$SVC +} + +exists() { docker container inspect "$1" >/dev/null 2>&1; } + +# tgit.yaml is read with grep, not executed or parsed by anything clever +yaml_value() { grep -E "^$1:" "$2" 2>/dev/null | head -1 | sed -E "s/^$1:[[:space:]]*//; s/[[:space:]]+#.*$//; s/^[\"']//; s/[\"']$//" || true; } + +wait_ready() { # $1 container, $2 has healthcheck (yes/no) + local deadline=$((SECONDS + HEALTH_TIMEOUT)) status restarts + if [ "$2" = yes ]; then + while [ $SECONDS -lt $deadline ]; do + status=$(docker inspect -f '{{.State.Health.Status}}' "$1" 2>/dev/null || echo gone) + case $status in healthy) return 0 ;; unhealthy | gone) return 1 ;; esac + sleep 2 + done + return 1 + fi + sleep "$SETTLE" + status=$(docker inspect -f '{{.State.Status}}' "$1" 2>/dev/null || echo gone) + restarts=$(docker inspect -f '{{.RestartCount}}' "$1" 2>/dev/null || echo 1) + [ "$status" = running ] && [ "$restarts" = 0 ] +} + +deploy() { + need_service 3 + SHA=${ARGS[2]} + [[ $SHA =~ $SHA_RE ]] || fail 2 "bad sha" + + if ! exists "$NAME"; then + local count + count=$(docker ps -a --filter label=tgit.service --format '{{.Names}}' | grep -cv -- '-next$' || true) + [ "$count" -lt "$MAX_SERVICES" ] || fail 4 "service limit $MAX_SERVICES reached" + fi + + mkdir -p "$STATE" + WORK=$(mktemp -d "$STATE/build.XXXXXX") + trap 'rm -rf "$WORK"' EXIT + head -c $((MAX_TAR + 1)) >"$WORK/src.tar" + [ "$(stat -c %s "$WORK/src.tar")" -le "$MAX_TAR" ] || fail 2 "archive larger than $MAX_TAR bytes" + # Only plain files and directories, relative paths without "..": no symlinks, devices or hard links + tar -tvf "$WORK/src.tar" >"$WORK/list" 2>/dev/null || fail 2 "not a tar archive" + if grep -qvE '^[-d]' "$WORK/list"; then fail 2 "archive may contain only files and directories"; fi + if tar -tf "$WORK/src.tar" | grep -qE '(^/|(^|/)\.\.(/|$))'; then fail 2 "unsafe path in archive"; fi + mkdir -p "$WORK/ctx/app" + tar -xf "$WORK/src.tar" -C "$WORK/ctx/app" --no-same-owner --no-same-permissions + [ -f "$WORK/ctx/app/main.py" ] || fail 2 "services/$SVC/main.py is missing" + # umask 077 above protects the build dir; inside the image the code must be readable (not writable) by uid 10001 + chmod -R u=rwX,go=rX "$WORK/ctx" + + local health network healthcheck=no net=none + health=$(yaml_value health_url "$WORK/ctx/app/tgit.yaml") + network=$(yaml_value network "$WORK/ctx/app/tgit.yaml") + if [ -n "$health" ]; then + [[ $health =~ $HEALTH_RE ]] || fail 2 "health_url must be http://localhost:/" + healthcheck=yes + fi + # Internet only if tgit.yaml asks AND the admin allowed this service on the host + if [ "$network" = true ] && grep -qxF "$SVC" "$STATE/egress-allow" 2>/dev/null; then + docker network inspect "$EGRESS_NET" >/dev/null 2>&1 || docker network create "$EGRESS_NET" >/dev/null + net=$EGRESS_NET + fi + + { + echo "FROM $BASE_IMAGE" + echo "RUN useradd --uid 10001 --no-create-home --home-dir /tmp svc" + echo "WORKDIR /app" + echo "COPY app/ /app/" + echo "RUN if [ -f requirements.txt ]; then pip install --no-cache-dir --disable-pip-version-check -r requirements.txt; fi" + echo "ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 HOME=/tmp" + if [ "$healthcheck" = yes ]; then + # exec form, the URL is an argument — nothing here goes through a shell + printf 'HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 CMD ["python", "-c", "import sys, urllib.request; urllib.request.urlopen(sys.argv[1], timeout=2)", "%s"]\n' "$health" + fi + echo "USER 10001:10001" + echo 'CMD ["python", "main.py"]' + } >"$WORK/ctx/Dockerfile" + + IMAGE=$NAME:$SHA + echo "building $IMAGE" + if ! docker build -q -t "$IMAGE" --label tgit.service="$SVC" "$WORK/ctx" >"$WORK/build.log" 2>&1; then + tail -20 "$WORK/build.log" + fail 5 "build failed" + fi + + NEXT=$NAME-next + docker rm -f "$NEXT" >/dev/null 2>&1 || true + docker run -d --name "$NEXT" \ + --label tgit.managed=true --label tgit.service="$SVC" \ + --label tgit.sha="$SHA" --label tgit.deployed_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --user 10001:10001 --read-only --tmpfs /tmp:rw,noexec,nosuid,size=16m \ + --cap-drop ALL --security-opt no-new-privileges \ + --memory 128m --memory-swap 128m --cpus 0.25 --pids-limit 64 \ + --network "$net" --restart on-failure:3 \ + --log-opt max-size=5m --log-opt max-file=2 \ + "$IMAGE" >/dev/null + + if ! wait_ready "$NEXT" "$healthcheck"; then + echo "--- last log lines of the new version ---" + docker logs --tail 20 "$NEXT" 2>&1 || true + docker rm -f "$NEXT" >/dev/null 2>&1 || true + if exists "$NAME"; then fail 3 "not healthy within ${HEALTH_TIMEOUT}s, previous version kept"; fi + fail 3 "not healthy within ${HEALTH_TIMEOUT}s" + fi + docker rm -f "$NAME" >/dev/null 2>&1 || true + docker rename "$NEXT" "$NAME" + + # Keep the current image and one previous; older ones go + { docker image ls "$NAME" --format '{{.Tag}}' | grep -vxF "$SHA" || true; } | tail -n +2 | while read -r old; do + docker image rm "$NAME:$old" >/dev/null 2>&1 || true + done + echo "OK deployed $SVC $SHA network=$net healthcheck=$healthcheck" +} + +restart() { + need_service 2 + exists "$NAME" || fail 2 "no such service" + docker restart -t 10 "$NAME" >/dev/null + echo "OK restarted $SVC" +} + +remove() { + need_service 2 + docker rm -f "$NAME" "$NAME-next" >/dev/null 2>&1 || true + docker image ls "$NAME" --format '{{.Repository}}:{{.Tag}}' | while read -r img; do + docker image rm "$img" >/dev/null 2>&1 || true + done + echo "OK removed $SVC" +} + +case $CMD in + deploy) deploy ;; + restart) restart ;; + remove) remove ;; + list) + [ "${#ARGS[@]}" -eq 1 ] || fail 2 "list takes no arguments" + docker ps -a --filter label=tgit.service --format '{{.Names}} {{.Status}} {{.Label "tgit.sha"}}' + echo "OK list" + ;; + *) fail 2 "unknown command" ;; +esac diff --git a/infra/uploads-template/.sourcecraft/ci.yaml b/infra/uploads-template/.sourcecraft/ci.yaml index 6456ab1..1fd4d5c 100644 --- a/infra/uploads-template/.sourcecraft/ci.yaml +++ b/infra/uploads-template/.sourcecraft/ci.yaml @@ -47,5 +47,6 @@ workflows: script: - python .tgit/changed_services.py > changed.txt - cat changed.txt - # Запуск через tgit-runner по SSH — #13. До этого — только список сервисов к развёртыванию. - - 'echo "deploy stub: $(wc -l < changed.txt) service(s) to deploy"' + # Разворачивает бот сразу после push (tgit-runner, #13); здесь — только список для лога. + # Когда CI переедет сюда целиком (#11), бот будет ждать зелёного check перед деплоем. + - 'echo "$(wc -l < changed.txt) service(s) changed — deployed by the bot via tgit-runner"' diff --git a/tests/runner/__init__.py b/tests/runner/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/runner/fake_docker.py b/tests/runner/fake_docker.py new file mode 100755 index 0000000..b7db9c3 --- /dev/null +++ b/tests/runner/fake_docker.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""A fake `docker` CLI for tgit-runner tests: records every call, keeps containers in a JSON file. + +Behaviour knobs (env): FAKE_HEALTH (healthy|unhealthy|none), FAKE_STATUS (running|exited), FAKE_RESTARTS, +FAKE_BUILD_FAIL=1. State: $FAKE_DOCKER_STATE (json), calls: $FAKE_DOCKER_LOG (one json array per line). +""" + +import json +import os +import sys + +state_path = os.environ["FAKE_DOCKER_STATE"] +args = sys.argv[1:] +with open(os.environ["FAKE_DOCKER_LOG"], "a") as log: + log.write(json.dumps(args) + "\n") +try: + with open(state_path) as f: + state = json.load(f) +except FileNotFoundError: + state = {"containers": {}, "images": []} + + +def save() -> None: + with open(state_path, "w") as f: + json.dump(state, f) + + +def done(code: int = 0, out: str = "") -> None: + if out: + print(out) + save() + sys.exit(code) + + +cmd = args[0] if args else "" +if cmd == "container" and args[1] == "inspect": + done(0 if args[2] in state["containers"] else 1) +if cmd == "inspect": + fmt, name = args[2], args[3] + if name not in state["containers"]: + done(1) + if "Health" in fmt: + done(0, os.environ.get("FAKE_HEALTH", "healthy")) + if "RestartCount" in fmt: + done(0, os.environ.get("FAKE_RESTARTS", "0")) + done(0, os.environ.get("FAKE_STATUS", "running")) +if cmd == "ps": + done(0, "\n".join(n for n in state["containers"])) +if cmd == "build": + if os.environ.get("FAKE_BUILD_FAIL"): + done(1, "ERROR: could not install requirements") + tag = args[args.index("-t") + 1] + ctx = args[-1] + with open(os.path.join(ctx, "Dockerfile")) as f: + state.setdefault("dockerfiles", {})[tag] = f.read() + state["images"].append(tag) + done(0, "sha256:feed") +if cmd == "run": + name = args[args.index("--name") + 1] + state["containers"][name] = {"image": args[-1], "args": args} + done(0, "cid") +if cmd == "rm": + for name in args[2:]: + state["containers"].pop(name, None) + done(0) +if cmd == "rename": + state["containers"][args[2]] = state["containers"].pop(args[1]) + done(0) +if cmd == "restart": + done(0) +if cmd == "logs": + done(0, "Traceback: boom") +if cmd == "image" and args[1] == "ls": + repo = args[2] + tags = [t.split(":", 1)[1] for t in reversed(state["images"]) if t.split(":", 1)[0] == repo] + full = "{{.Repository}}" in args[-1] + done(0, "\n".join(f"{repo}:{t}" if full else t for t in tags)) +if cmd == "image" and args[1] == "rm": + state["images"] = [t for t in state["images"] if t != args[2]] + done(0) +if cmd == "network": + done(0) +done(2, f"fake docker: unsupported {args}") diff --git a/tests/runner/fake_ssh.py b/tests/runner/fake_ssh.py new file mode 100755 index 0000000..f819696 --- /dev/null +++ b/tests/runner/fake_ssh.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +"""A fake `ssh`: like sshd with a forced command — the remote command becomes SSH_ORIGINAL_COMMAND of tgit-runner.""" + +import os +import sys + +args = sys.argv[1:] +with_value = {"-F", "-p", "-o", "-i"} +i = 0 +while i < len(args) and args[i].startswith("-"): + i += 2 if args[i] in with_value else 1 +remote = " ".join(args[i + 1 :]) +os.environ["SSH_ORIGINAL_COMMAND"] = remote +os.execvp("bash", ["bash", os.environ["FAKE_RUNNER"]]) diff --git a/tests/runner/test_deploy.py b/tests/runner/test_deploy.py new file mode 100644 index 0000000..0c8f38f --- /dev/null +++ b/tests/runner/test_deploy.py @@ -0,0 +1,234 @@ +"""Bot side of #13: the SSH client, the deployer and the push → deploy flow, down to the real tgit-runner script.""" + +import asyncio +import base64 +import json +import subprocess +from collections.abc import Callable +from dataclasses import replace +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from aiogram import Bot, Dispatcher +from aiogram.methods import AnswerCallbackQuery, EditMessageText, SendMessage +from aiogram.types import InlineKeyboardMarkup + +from app.access.service import AccessService +from app.alerts.broadcast import TelegramNotifier +from app.alerts.detectors import load_alerts_config +from app.alerts.service import AlertService +from app.bot.routers.commit import Act, CommitCb +from app.bot.routers.status import StatusAct, StatusCb, render_card, runner_service +from app.config import RunnerSettings +from app.db.models import Alert, AlertKind +from app.db.session import SessionMaker +from app.main import build_dispatcher +from app.repo.git import UploadsRepo +from app.runner.client import RunnerClient, RunnerResult +from app.runner.deployer import Deployer, Outcome, changed_services +from app.texts import Texts +from tests.alerts.test_detectors import RUNNING +from tests.conftest import ADMIN, ROOT +from tests.fakes import TgClient +from tests.runner.test_tgit_runner import FAKE, RUNNER + +HERE = Path(__file__).parent + + +@pytest.fixture +def fake_host(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """ssh → tgit-runner → fake docker, all on PATH.""" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + (bin_dir / "ssh").symlink_to(HERE / "fake_ssh.py") + (bin_dir / "docker").symlink_to(FAKE) + monkeypatch.setenv("PATH", f"{bin_dir}:/usr/bin:/bin") + monkeypatch.setenv("FAKE_RUNNER", str(RUNNER)) + monkeypatch.setenv("FAKE_DOCKER_STATE", str(tmp_path / "state.json")) + monkeypatch.setenv("FAKE_DOCKER_LOG", str(tmp_path / "calls.log")) + monkeypatch.setenv("TGIT_RUNNER_STATE", str(tmp_path / "runner")) + monkeypatch.setenv("TGIT_HEALTH_TIMEOUT", "2") + monkeypatch.setenv("TGIT_SETTLE", "0") + return tmp_path + + +def docker_state(host: Path) -> dict[str, object]: + return json.loads((host / "state.json").read_text()) + + +def client(key: bytes | None = None) -> RunnerClient: + b64 = base64.b64encode(key).decode() if key else None + return RunnerClient(RunnerSettings(host="host.example", ssh_key_b64=b64, timeout=30)) # type: ignore[arg-type] + + +def test_changed_services() -> None: + paths = ["services/hello/main.py", "services/hello/x/y.py", "services/Bad/main.py", "README.md", "services/a.py"] + assert changed_services(paths) == ["hello"] + + +async def test_client_validates_before_ssh(fake_host: Path) -> None: + result = await client().deploy("../etc", "abc1234", b"") + + assert (result.ok, result.code) == (False, 2) + assert not (fake_host / "calls.log").exists() + + +def archive_of(remote: Path) -> bytes: + return subprocess.run( + ["git", "archive", "--format=tar", "HEAD:services/hello"], cwd=remote, capture_output=True, check=True + ).stdout + + +async def until(condition: Callable[[], bool]) -> None: + """The deploy runs in a background task: wait for its visible effect.""" + async with asyncio.timeout(15): + while not condition(): # noqa: ASYNC110 - polling a fake Telegram session, no event to wait on + await asyncio.sleep(0.05) + + +async def test_client_through_forced_command(fake_host: Path, remote: Path) -> None: + c = client(b"-----BEGIN KEY-----") + archive = archive_of(remote) + + ok = await c.deploy("hello", "abc1234", archive) + restart = await c.restart("hello") + bad = await c.run("deploy", "hello;", "rm") + + assert ok.ok and ok.summary.startswith("deployed hello abc1234") + assert restart.ok + assert (bad.ok, bad.code) == (False, 2) + assert "tgit-svc-hello" in docker_state(fake_host)["containers"] # type: ignore[operator] + + +class Recorder: + def __init__(self) -> None: + self.outcomes: list[Outcome] = [] + + async def __call__(self, o: Outcome) -> None: + self.outcomes.append(o) + + +async def test_deployer_deploys_and_removes(fake_host: Path, repo: UploadsRepo, remote: Path, tmp_path: Path) -> None: + await repo.ensure() + sha = await repo.head() + rec = Recorder() + deployer = Deployer(repo, client()) + + await deployer.deploy(sha, ["hello", "gone"], rec) + + hello, gone = rec.outcomes + assert hello.result.ok and not hello.removed + assert gone.removed and gone.result.ok + calls = [json.loads(line) for line in (fake_host / "calls.log").read_text().splitlines()] + assert any(c[0] == "run" for c in calls) + + +async def test_refused_deploy_raises_alert( + fake_host: Path, repo: UploadsRepo, sessions: SessionMaker, monkeypatch: pytest.MonkeyPatch +) -> None: + await repo.ensure() + monkeypatch.setenv("FAKE_BUILD_FAIL", "1") + events: list[Alert] = [] + + class Notes: + async def opened(self, alert: Alert, admins_only: bool) -> None: + events.append(alert) + + async def repeated(self, alert: Alert) -> None: ... + + async def resolved(self, alert: Alert) -> None: ... + + alerts = AlertService(sessions, Notes(), load_alerts_config(ROOT / "config")) + rec = Recorder() + await Deployer(repo, client(), alerts).deploy(await repo.head(), ["hello"], rec) + + [o] = rec.outcomes + assert (o.result.ok, o.result.code, o.result.summary) == (False, 5, "build failed") + [alert] = events + assert alert.kind is AlertKind.DEPLOY_REFUSED and alert.service == "tgit-svc-hello" + assert "could not install" in alert.payload["log"] + + +async def test_one_broken_service_does_not_stop_others(repo: UploadsRepo) -> None: + class Flaky(RunnerClient): + async def run(self, *args: str, stdin: bytes = b"") -> RunnerResult: + if args[1] == "aaa": + raise RuntimeError("ssh exploded") + return RunnerResult(True, 0, f"deployed {args[1]}", "OK") + + await repo.ensure() + rec = Recorder() + await Deployer(repo, Flaky(RunnerSettings(host="h"))).deploy(await repo.head(), ["aaa", "hello"], rec) + + assert [(o.service, o.result.ok) for o in rec.outcomes] == [("aaa", False), ("hello", True)] + + +# ---- in the bot ------------------------------------------------------------------------------------------------- + + +@pytest.fixture +def tg(texts: Texts, sessions: SessionMaker, access: AccessService, repo: UploadsRepo, fake_host: Path) -> TgClient: + def dispatcher(bot: Bot) -> Dispatcher: + alerts = AlertService( + sessions, TelegramNotifier(bot, access, sessions, texts), load_alerts_config(ROOT / "config") + ) + return build_dispatcher(texts, sessions, access, repo, alerts=alerts, runner=client()) + + return TgClient(dispatcher) + + +async def test_push_deploys_touched_service(tg: TgClient, texts: Texts, fake_host: Path, remote: Path) -> None: + await tg.send(ADMIN, texts.menu.commit) + await tg.press(ADMIN, CommitCb(a=Act.OPEN, i=2).pack()) # services + await tg.press(ADMIN, CommitCb(a=Act.OPEN, i=0).pack()) # services/hello + await tg.press(ADMIN, CommitCb(a=Act.NEW_FILE).pack()) + await tg.send(ADMIN, "util.py") + await tg.send(ADMIN, "X = 1\n") + await tg.press(ADMIN, CommitCb(a=Act.MESSAGE).pack()) + await tg.send(ADMIN, "add util") + await tg.press(ADMIN, CommitCb(a=Act.PUSH).pack()) + await tg.press(ADMIN, CommitCb(a=Act.PUSH_GO).pack()) + await until(lambda: any("работает" in (m.text or "") for m in tg.session.sent(SendMessage))) + + pushed = [e for e in tg.session.sent(EditMessageText) if "Запушено" in (e.text or "")] + assert pushed and "🚀 Разворачиваю: hello…" in pushed[-1].text + [done] = [m for m in tg.session.sent(SendMessage) if "работает" in (m.text or "")] + assert done.text.startswith("🚀 hello работает · ") + assert "tgit-svc-hello" in docker_state(fake_host)["containers"] # type: ignore[operator] + + +def buttons(markup: InlineKeyboardMarkup) -> list[str]: + return [b.text for row in markup.inline_keyboard for b in row] + + +def test_restart_button_only_for_runner_services(texts: Texts) -> None: + service = replace(RUNNING, labels={"tgit.service": "hello"}) + now = datetime.now(UTC) + + assert texts.status.restart in buttons(render_card(texts, service, None, now, can_restart=True)[1]) + assert texts.status.restart not in buttons(render_card(texts, service, None, now)[1]) # not an admin + assert texts.status.restart not in buttons(render_card(texts, RUNNING, None, now, can_restart=True)[1]) # the bot + assert runner_service(service) == "hello" and runner_service(RUNNING) is None + + +async def test_restart_through_status( + texts: Texts, sessions: SessionMaker, access: AccessService, repo: UploadsRepo, monkeypatch: pytest.MonkeyPatch +) -> None: + from tests.runtime.fake_docker import FakeDocker + + fake = FakeDocker() + svc = fake.add("tgit-svc-hello", labels={"tgit.service": "hello"}) + calls: list[tuple[str, ...]] = [] + + async def run(self: RunnerClient, *args: str, stdin: bytes = b"") -> RunnerResult: + calls.append(args) + return RunnerResult(True, 0, "restarted hello", "OK restarted hello") + + monkeypatch.setattr(RunnerClient, "run", run) + async with fake.client() as docker: + tg = TgClient(build_dispatcher(texts, sessions, access, repo, docker=docker, runner=client())) + await tg.press(ADMIN, StatusCb(a=StatusAct.RESTART, id=svc.id[:12]).pack()) + + assert calls == [("restart", "hello")] + assert tg.session.sent(AnswerCallbackQuery)[0].text == texts.status.restarted diff --git a/tests/runner/test_tgit_runner.py b/tests/runner/test_tgit_runner.py new file mode 100644 index 0000000..f830c07 --- /dev/null +++ b/tests/runner/test_tgit_runner.py @@ -0,0 +1,250 @@ +"""tgit-runner against a fake docker: argument validation, archive checks, sandbox flags, rollback.""" + +import io +import json +import shutil +import subprocess +import tarfile +from collections.abc import Callable +from pathlib import Path + +import pytest + +from tests.conftest import ROOT + +RUNNER = ROOT / "infra" / "runner" / "tgit-runner" +FAKE = Path(__file__).with_name("fake_docker.py") + +pytestmark = pytest.mark.skipif(shutil.which("bash") is None, reason="needs bash") + + +def tar_of(files: dict[str, str | bytes], extra: Callable[[tarfile.TarFile], None] | None = None) -> bytes: + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w") as tar: + for name, content in files.items(): + data = content.encode() if isinstance(content, str) else content + info = tarfile.TarInfo(name) + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + if extra: + extra(tar) + return buf.getvalue() + + +HELLO = tar_of({"main.py": "print('hi')\n"}) + + +class Runner: + def __init__(self, tmp: Path) -> None: + self.tmp = tmp + bin_dir = tmp / "bin" + bin_dir.mkdir() + (bin_dir / "docker").symlink_to(FAKE) + self.env = { + "PATH": f"{bin_dir}:/usr/bin:/bin", + "FAKE_DOCKER_STATE": str(tmp / "state.json"), + "FAKE_DOCKER_LOG": str(tmp / "calls.log"), + "TGIT_RUNNER_STATE": str(tmp / "runner"), + "TGIT_HEALTH_TIMEOUT": "4", + "TGIT_SETTLE": "0", + } + + def __call__( + self, *args: str, stdin: bytes = b"", ssh: str | None = None, **env: str + ) -> subprocess.CompletedProcess[str]: + e = {**self.env, **env} + if ssh is not None: + e["SSH_ORIGINAL_COMMAND"] = ssh + return subprocess.run( + ["bash", str(RUNNER), *args], + input=stdin.decode("latin-1"), + env=e, + capture_output=True, + text=True, + timeout=30, + encoding="latin-1", + ) + + def calls(self) -> list[list[str]]: + path = self.tmp / "calls.log" + return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else [] + + def state(self) -> dict[str, object]: + return json.loads((self.tmp / "state.json").read_text()) + + +@pytest.fixture +def runner(tmp_path: Path) -> Runner: + return Runner(tmp_path) + + +def last(out: subprocess.CompletedProcess[str]) -> str: + return out.stdout.strip().splitlines()[-1] + + +@pytest.mark.parametrize( + ("ssh", "reason"), + [ + ("", "usage"), + ("rm -rf /", "unknown command"), + ("deploy hello; rm -rf /", "wrong number of arguments"), + ("deploy Hello abc1234", "bad service name"), + ("deploy ../etc abc1234", "bad service name"), + ("deploy hello $(id)", "bad sha"), + ("deploy hello abc1234 extra", "wrong number of arguments"), + ("restart *", "bad service name"), + ("list now", "list takes no arguments"), + ], +) +def test_bad_requests_rejected_before_docker(runner: Runner, ssh: str, reason: str) -> None: + out = runner(ssh=ssh) + + assert out.returncode == 2 + assert reason in last(out) + assert runner.calls() == [] + + +def test_deploy_runs_in_sandbox(runner: Runner) -> None: + out = runner(ssh="deploy hello abc1234", stdin=tar_of({"main.py": "x", "tgit.yaml": "network: true\n"})) + + assert out.returncode == 0, out.stdout + out.stderr + assert last(out) == "OK deployed hello abc1234 network=none healthcheck=no" # network not allowed by admin + [run] = [c for c in runner.calls() if c[0] == "run"] + flags = " ".join(run) + for flag in [ + "--user 10001:10001", + "--read-only", + "--cap-drop ALL", + "--security-opt no-new-privileges", + "--memory 128m --memory-swap 128m", + "--cpus 0.25", + "--pids-limit 64", + "--network none", + "--label tgit.managed=true", + "--label tgit.sha=abc1234", + ]: + assert flag in flags + assert "/var/run/docker.sock" not in flags + assert list(runner.state()["containers"]) == ["tgit-svc-hello"] # -next renamed into place + + +def test_egress_only_when_admin_allows(runner: Runner) -> None: + (runner.tmp / "runner").mkdir() + (runner.tmp / "runner" / "egress-allow").write_text("weather\n") + + out = runner(ssh="deploy weather abc1234", stdin=tar_of({"main.py": "x", "tgit.yaml": "network: true\n"})) + + assert "network=tgit-svc-egress" in last(out) + + +def test_healthcheck_from_tgit_yaml_without_a_shell(runner: Runner) -> None: + yaml = "health_url: http://localhost:8080/health # probe\n" + out = runner(ssh="deploy web abc1234", stdin=tar_of({"main.py": "x", "tgit.yaml": yaml})) + + assert out.returncode == 0 + dockerfile = runner.state()["dockerfiles"]["tgit-svc-web:abc1234"] # type: ignore[index] + assert 'CMD ["python", "-c", "import sys, urllib.request' in dockerfile + assert '"http://localhost:8080/health"]' in dockerfile + + +@pytest.mark.parametrize( + "url", ["http://evil.com/x", "http://localhost:80/$(id)", 'http://localhost:80/"]; RUN id #', "file:///etc/passwd"] +) +def test_health_url_must_be_local(runner: Runner, url: str) -> None: + out = runner(ssh="deploy web abc1234", stdin=tar_of({"main.py": "x", "tgit.yaml": f"health_url: {url}\n"})) + + assert out.returncode == 2 and "health_url" in last(out) + assert not [c for c in runner.calls() if c[0] == "build"] + + +def symlink_to_etc(tar: tarfile.TarFile) -> None: + info = tarfile.TarInfo("link") + info.type = tarfile.SYMTYPE + info.linkname = "/etc/passwd" + tar.addfile(info) + + +@pytest.mark.parametrize( + ("archive", "reason"), + [ + (b"not a tar", "not a tar archive"), + (tar_of({"main.py": "x"}, symlink_to_etc), "only files and directories"), + (tar_of({"main.py": "x", "../escape.py": "x"}), "unsafe path"), + (tar_of({"lib.py": "x"}), "main.py is missing"), + ], +) +def test_bad_archives(runner: Runner, archive: bytes, reason: str) -> None: + out = runner(ssh="deploy hello abc1234", stdin=archive) + + assert out.returncode == 2 and reason in last(out) + + +def test_archive_size_limit(runner: Runner) -> None: + out = runner(ssh="deploy hello abc1234", stdin=HELLO, TGIT_MAX_TAR="100") + + assert out.returncode == 2 and "larger than" in last(out) + + +def test_unhealthy_new_version_keeps_the_old_one(runner: Runner) -> None: + assert runner(ssh="deploy hello abc1234", stdin=HELLO).returncode == 0 + + out = runner(ssh="deploy hello def5678", stdin=HELLO, FAKE_HEALTH="unhealthy", FAKE_STATUS="exited") + + assert out.returncode == 3 + assert last(out) == "FAIL not healthy within 4s, previous version kept" + assert "Traceback: boom" in out.stdout # the new version's log tail for the user + assert list(runner.state()["containers"]) == ["tgit-svc-hello"] + assert runner.state()["containers"]["tgit-svc-hello"]["image"] == "tgit-svc-hello:abc1234" # type: ignore[index] + + +def test_restarting_without_healthcheck_is_not_ready(runner: Runner) -> None: + out = runner(ssh="deploy hello abc1234", stdin=HELLO, FAKE_RESTARTS="2") + + assert out.returncode == 3 and last(out) == "FAIL not healthy within 4s" + + +def test_build_failure(runner: Runner) -> None: + out = runner(ssh="deploy hello abc1234", stdin=HELLO, FAKE_BUILD_FAIL="1") + + assert out.returncode == 5 and "could not install requirements" in out.stdout + + +def test_service_limit(runner: Runner) -> None: + for i in range(3): + assert runner(ssh=f"deploy s{i}x abc1234", stdin=HELLO, TGIT_MAX_SERVICES="3").returncode == 0 + + out = runner(ssh="deploy extra abc1234", stdin=HELLO, TGIT_MAX_SERVICES="3") + redeploy = runner(ssh="deploy s0x def5678", stdin=HELLO, TGIT_MAX_SERVICES="3") + + assert out.returncode == 4 and "limit 3" in last(out) + assert redeploy.returncode == 0 # an existing service may always be updated + + +def test_old_images_pruned(runner: Runner) -> None: + for sha in ("aaa1111", "bbb2222", "ccc3333"): + assert runner(ssh=f"deploy hello {sha}", stdin=HELLO).returncode == 0 + + assert sorted(runner.state()["images"]) == ["tgit-svc-hello:bbb2222", "tgit-svc-hello:ccc3333"] # type: ignore[type-var] + + +def test_restart_remove_list(runner: Runner) -> None: + assert last(runner(ssh="restart hello")) == "FAIL no such service" + runner(ssh="deploy hello abc1234", stdin=HELLO) + + assert last(runner(ssh="restart hello")) == "OK restarted hello" + assert last(runner(ssh="list")) == "OK list" + assert last(runner(ssh="remove hello")) == "OK removed hello" + assert runner.state()["containers"] == {} + assert runner.state()["images"] == [] + + +def test_direct_invocation_without_ssh(runner: Runner) -> None: + out = runner("list") + assert out.returncode == 0 + + +def test_glob_in_ssh_command_is_not_expanded(runner: Runner, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.chdir(tmp_path) + (tmp_path / "abc1234").write_text("") + out = runner(ssh="deploy hello abc12*") + assert out.returncode == 2 and "bad sha" in last(out)