feat(runner): tgit-runner — сервисы в песочнице на хосте, деплой после push (#13)
CI / ruff · mypy · pytest (pull_request) Successful in 4m3s
CI / ruff · mypy · pytest (pull_request) Successful in 4m3s
- infra/runner/tgit-runner: deploy/restart/remove/list по SSH forced command; аргументы не исполняются и не глобятся; архив — только файлы и папки, без «..», ≤ 20 МБ, есть main.py; health_url — только localhost - песочница: uid 10001, read-only, tmpfs noexec, cap-drop ALL, no-new-privileges, 128 МБ без swap, 0.25 CPU, 64 PID, --network none (сеть — только по egress-allow админа), лимит 10 сервисов, 2 образа - новая версия стартует рядом (-next) и заменяет старую, только став healthy; иначе удаляется, старая работает, exit 3 - бот: после push git archive изменённых services/<name>/ → runner; автору — «🚀 работает» или «⛔ не развёрнут» с логом, команде — алерт ⛔; удалённая папка → remove; «♻️ Перезапустить» в «Статусе» для админа (S-5) - examples/escape_attempts: все попытки выбраться — BLOCKED на живом Docker - install.sh, authorized_keys.example; CD бота обновляет runner на хосте Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""Deploying services of the uploads repo: the bot ships `git archive` to `tgit-runner` on the host (#13)."""
|
||||
@@ -0,0 +1,85 @@
|
||||
"""SSH to `tgit-runner`: the key can run nothing else (forced command), so the «command» is just its arguments."""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from app.config import RunnerSettings
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SERVICE = re.compile(r"^[a-z][a-z0-9-]{1,30}$") # the runner checks the same — refuse early, with a clear reason
|
||||
SHA = re.compile(r"^[0-9a-f]{7,40}$")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RunnerResult:
|
||||
ok: bool
|
||||
code: int # 0 ok · 2 bad request · 3 not healthy · 4 limit · 5 build failed · 255 ssh · -1 timeout
|
||||
summary: str # the runner's last line without OK/FAIL
|
||||
output: str # everything, for logs and the user's log tail
|
||||
|
||||
@property
|
||||
def tail(self) -> str:
|
||||
lines = self.output.strip().splitlines()[:-1]
|
||||
return "\n".join(lines[-20:])
|
||||
|
||||
|
||||
class RunnerClient:
|
||||
def __init__(self, settings: RunnerSettings) -> None:
|
||||
self._s = settings
|
||||
self._key = self._materialize_key()
|
||||
|
||||
def _materialize_key(self) -> Path | None:
|
||||
if self._s.ssh_key_b64 is None:
|
||||
return None
|
||||
key = base64.b64decode(self._s.ssh_key_b64.get_secret_value())
|
||||
fd, name = tempfile.mkstemp(prefix="tgit-runner-")
|
||||
with os.fdopen(fd, "wb") as f:
|
||||
f.write(key if key.endswith(b"\n") else key + b"\n")
|
||||
return Path(name)
|
||||
|
||||
def _ssh(self) -> list[str]:
|
||||
cmd = ["ssh", "-F", "/dev/null", "-p", str(self._s.port), "-o", "BatchMode=yes", "-o", "ConnectTimeout=20"]
|
||||
cmd += ["-o", "StrictHostKeyChecking=accept-new", "-o", f"UserKnownHostsFile={tempfile.gettempdir()}/tgit-kh"]
|
||||
if self._key is not None:
|
||||
cmd += ["-i", str(self._key), "-o", "IdentitiesOnly=yes"]
|
||||
return [*cmd, f"{self._s.user}@{self._s.host}"]
|
||||
|
||||
async def run(self, *args: str, stdin: bytes = b"") -> RunnerResult:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*self._ssh(),
|
||||
*args,
|
||||
stdin=asyncio.subprocess.PIPE,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.STDOUT,
|
||||
)
|
||||
try:
|
||||
out, _ = await asyncio.wait_for(proc.communicate(stdin), timeout=self._s.timeout)
|
||||
except TimeoutError:
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
return RunnerResult(False, -1, "timeout", "")
|
||||
text = out.decode(errors="replace")
|
||||
last = text.strip().splitlines()[-1] if text.strip() else ""
|
||||
code = proc.returncode if proc.returncode is not None else -1
|
||||
ok = code == 0 and last.startswith("OK")
|
||||
summary = last.removeprefix("OK ").removeprefix("FAIL ") if last.startswith(("OK", "FAIL")) else last[:200]
|
||||
if not ok:
|
||||
log.warning("runner %s: exit %s: %s", args[0] if args else "", code, summary)
|
||||
return RunnerResult(ok, code, summary, text)
|
||||
|
||||
async def deploy(self, service: str, sha: str, archive: bytes) -> RunnerResult:
|
||||
if not SERVICE.match(service) or not SHA.match(sha):
|
||||
return RunnerResult(False, 2, "bad service name or sha", "")
|
||||
return await self.run("deploy", service, sha, stdin=archive)
|
||||
|
||||
async def restart(self, service: str) -> RunnerResult:
|
||||
if not SERVICE.match(service):
|
||||
return RunnerResult(False, 2, "bad service name", "")
|
||||
return await self.run("restart", service)
|
||||
@@ -0,0 +1,85 @@
|
||||
"""After a push: the services it touched are (re)deployed through tgit-runner, the pusher hears how it went.
|
||||
|
||||
A service = `services/<name>/`. Gone from the commit — removed on the host. A refused deploy keeps the old version
|
||||
running (the runner switches only to a healthy one) and raises ⛔ for the whole team.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable, Iterable
|
||||
from dataclasses import dataclass
|
||||
|
||||
from app.alerts.detectors import Condition
|
||||
from app.alerts.service import AlertService
|
||||
from app.db.models import AlertKind
|
||||
from app.repo.git import GitError, UploadsRepo
|
||||
from app.runner.client import SERVICE, RunnerClient, RunnerResult
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def changed_services(paths: Iterable[str]) -> list[str]:
|
||||
found = set()
|
||||
for path in paths:
|
||||
parts = path.split("/")
|
||||
if len(parts) >= 3 and parts[0] == "services" and SERVICE.match(parts[1]):
|
||||
found.add(parts[1])
|
||||
return sorted(found)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Outcome:
|
||||
service: str
|
||||
result: RunnerResult
|
||||
removed: bool = False
|
||||
|
||||
|
||||
Report = Callable[[Outcome], Awaitable[None]]
|
||||
|
||||
|
||||
class Deployer:
|
||||
def __init__(self, repo: UploadsRepo, runner: RunnerClient, alerts: AlertService | None = None) -> None:
|
||||
self._repo = repo
|
||||
self._runner = runner
|
||||
self._alerts = alerts
|
||||
self._lock = asyncio.Lock() # one deploy at a time: builds are heavy for the shared host
|
||||
self._tasks: set[asyncio.Task[None]] = set()
|
||||
|
||||
def schedule(self, sha: str, paths: Iterable[str], report: Report) -> list[str]:
|
||||
"""Start in the background; returns the services that will be deployed."""
|
||||
services = changed_services(paths)
|
||||
if services:
|
||||
task = asyncio.create_task(self.deploy(sha, services, report))
|
||||
self._tasks.add(task)
|
||||
task.add_done_callback(self._tasks.discard)
|
||||
return services
|
||||
|
||||
async def deploy(self, sha: str, services: list[str], report: Report) -> None:
|
||||
async with self._lock:
|
||||
for service in services:
|
||||
try:
|
||||
outcome = await self._one(sha, service)
|
||||
except Exception: # one broken service must not stop the others
|
||||
log.exception("deploy %s @ %s", service, sha[:7])
|
||||
outcome = Outcome(service, RunnerResult(False, -1, "internal error", ""))
|
||||
if not outcome.result.ok and self._alerts is not None:
|
||||
await self._alerts.event(
|
||||
Condition(
|
||||
AlertKind.DEPLOY_REFUSED,
|
||||
f"tgit-svc-{service}",
|
||||
{"detail": outcome.result.summary, "sha": sha[:7], "log": outcome.result.tail},
|
||||
)
|
||||
)
|
||||
try:
|
||||
await report(outcome)
|
||||
except Exception:
|
||||
log.exception("deploy report for %s", service)
|
||||
|
||||
async def _one(self, sha: str, service: str) -> Outcome:
|
||||
tree = f"{sha}:services/{service}"
|
||||
try:
|
||||
await self._repo.git("cat-file", "-e", tree)
|
||||
except GitError:
|
||||
return Outcome(service, await self._runner.run("remove", service), removed=True)
|
||||
archive = await self._repo.git("archive", "--format=tar", tree)
|
||||
return Outcome(service, await self._runner.deploy(service, sha, archive))
|
||||
Reference in New Issue
Block a user