release: dev → main #54
@@ -29,6 +29,11 @@ GIT_UPLOADS_URL=git@ssh.sourcecraft.dev:rogachevegorushka/tgit-uploads.git
|
||||
# DOCKER_URL=
|
||||
# DOCKER_LABEL=tgit.managed=true
|
||||
|
||||
# --- Runner: деплой сервисов из репозитория загрузок (#13, infra/runner/) ---
|
||||
# RUNNER_HOST=host.docker.internal # пусто — бот сервисы не разворачивает
|
||||
# RUNNER_USER=tgit-deploy
|
||||
# RUNNER_SSH_KEY_B64= # (secret) base64 ключа, прибитого к tgit-runner в authorized_keys
|
||||
|
||||
# --- LLM (#17) ---
|
||||
LLM_PROVIDER=mock # mock | openai_compat; при mock агент логов выключен (#18)
|
||||
LLM_BASE_URL=
|
||||
|
||||
@@ -41,6 +41,7 @@
|
||||
| **📜 Логи** | хвост 50/200 строк или 5 мин/1 ч/24 ч; свой grep (`-i -v -E -c -C N`), выполняется в Python, не в shell; длинный ответ — файлом; журнал поисков | L-1…L-6 |
|
||||
| **🤖 Агент** | вопрос словами → фильтр → LLM-страж → LLM строит только `LogQuery` → валидация → grep; показывает построенную команду и строки; атака — отказ и 🛡 админам; 30 вопросов в сутки | L-10…L-17 |
|
||||
| **🚨 Алерты** | 💥 краш, 🧠 OOM, 🔁 цикл рестартов, 🩺 unhealthy, 🔥 перегрузка, 💾 мало места, 🔌 Docker недоступен, 🛡 атака; всем участникам, с хвостом лога и кнопками; повторы — счётчик, «✅ восстановлено» ответом; список за 7 дней | AL-1…AL-6 |
|
||||
| **🚀 Деплой сервисов** | после push бот отправляет изменённые `services/<name>/` в `tgit-runner` на хосте: сборка образа, запуск в песочнице (uid 10001, read-only, без capabilities и сети, 128 МБ / 0.25 CPU / 64 процесса); старая версия заменяется только здоровой новой, иначе ⛔ с логом; «♻️ Перезапустить» в «Статусе» для админа | #13, S-5 |
|
||||
| **CI/CD бота** | Gitea Actions: ruff, mypy strict, миграции up/down/up на PostgreSQL, pytest с порогом покрытия 80 %; автодеплой `dev` на сервер с проверкой health | ADR-015 |
|
||||
|
||||
Качество: 296 тестов, покрытие 92 %, `mypy --strict`, `ruff`. Telegram в тестах — фейковая сессия aiogram,
|
||||
@@ -148,6 +149,7 @@ make cov # pytest с покрытием (порог 80 %)
|
||||
| git (SSH) | чтение и коммиты в репозиторий загрузок | `git` подпроцессом со своим ключом, без shell |
|
||||
| Docker Engine API | контейнеры, stats, логи | только GET через `docker-socket-proxy` |
|
||||
| OpenAI-совместимый `/chat/completions` | страж и планировщик агента | httpx, `response_format: json_schema` |
|
||||
| SSH → `tgit-runner` | деплой и перезапуск сервисов на хосте | `ssh` подпроцессом, ключ с forced command |
|
||||
| SourceCraft REST API | статус прогонов CI репозитория загрузок | httpx, клиент готов, в UI — #11 |
|
||||
|
||||
## 5. Ограничения и особенности
|
||||
@@ -176,8 +178,8 @@ make cov # pytest с покрытием (порог 80 %)
|
||||
| повтор алерта | в течение 10 мин — счётчик, а не новое сообщение |
|
||||
|
||||
**Что пока не сделано** (открытые задачи):
|
||||
- запуск сервисов из репозитория загрузок в песочнице (`--cap-drop ALL`, read-only FS, лимиты) и перезапуск из бота —
|
||||
`tgit-runner`, #13. Сейчас сервисы поднимаются Compose, как в `examples/`;
|
||||
- проверки CI репозитория загрузок (ruff, bandit, тесты) перед деплоем: сейчас бот разворачивает сервис сразу после
|
||||
push, проверив только синтаксис; ожидание зелёного CI появится вместе с #11;
|
||||
- уведомления о прогонах CI/CD репозитория загрузок и алерты 📦/⛔ — #11, ждут возврата к SourceCraft;
|
||||
- «🔕 Тишина на час» для алертов (P2);
|
||||
- prod-окружение: нужен отдельный бот, деплой `main` пропускается, пока его нет.
|
||||
@@ -193,12 +195,13 @@ app/
|
||||
logs/ grep-движок и журнал
|
||||
agent/ фильтр, страж, планировщик, схема LogQuery
|
||||
alerts/ детекторы, сервис алертов, рассылка, монитор
|
||||
runner/ SSH-клиент tgit-runner и деплой после push
|
||||
llm/ провайдеры: OpenAI-совместимый, mock
|
||||
sourcecraft/ клиент API прогонов
|
||||
db/ модели, миграции Alembic, очистка
|
||||
config/ тексты интерфейса, пороги проверок и алертов, правила и промпты агента
|
||||
tests/ pytest; фейки Telegram, Docker и LLM
|
||||
examples/ демо-сервисы и сценарии
|
||||
infra/ деплой на сервер, шаблон репозитория загрузок
|
||||
infra/ деплой бота, tgit-runner (песочница сервисов), шаблон репозитория загрузок
|
||||
docs/ ТЗ, архитектура, ADR, процесс
|
||||
```
|
||||
|
||||
@@ -4,6 +4,7 @@ import base64
|
||||
import html
|
||||
import io
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass
|
||||
from enum import StrEnum
|
||||
from typing import Any
|
||||
@@ -33,6 +34,8 @@ from app.repo.validate import (
|
||||
python_syntax,
|
||||
strip_fence,
|
||||
)
|
||||
from app.runner.deployer import Deployer
|
||||
from app.runner.deployer import Outcome as DeployOutcome
|
||||
from app.texts import CommitTexts, Texts
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -212,7 +215,12 @@ def to_dir_keyboard(texts: Texts) -> InlineKeyboardMarkup:
|
||||
|
||||
|
||||
def build_router(
|
||||
texts: Texts, repo: UploadsRepo, drafts: DraftService, publisher: Publisher, commit_url_template: str = ""
|
||||
texts: Texts,
|
||||
repo: UploadsRepo,
|
||||
drafts: DraftService,
|
||||
publisher: Publisher,
|
||||
commit_url_template: str = "",
|
||||
deployer: Deployer | None = None,
|
||||
) -> Router:
|
||||
router = Router(name="commit")
|
||||
t = texts.commit
|
||||
@@ -275,6 +283,21 @@ def build_router(
|
||||
ops = {ChangeOp.ADD: t.op_add, ChangeOp.MODIFY: t.op_modify, ChangeOp.DELETE: t.op_delete}
|
||||
return "\n".join(f"{ops[c.op]} /{c.path}" for c in draft.changes)
|
||||
|
||||
def deploy_report(msg: Message, sha: str) -> Callable[[DeployOutcome], Awaitable[None]]:
|
||||
async def report(o: DeployOutcome) -> None:
|
||||
name = html.escape(o.service)
|
||||
if o.result.ok and o.removed:
|
||||
text = t.removed.format(service=name)
|
||||
elif o.result.ok:
|
||||
text = t.deployed.format(service=name, sha=sha[:7])
|
||||
else:
|
||||
text = t.deploy_failed.format(service=name, reason=html.escape(o.result.summary))
|
||||
if o.result.tail:
|
||||
text += f"\n<pre>{html.escape(o.result.tail[-3000:])}</pre>"
|
||||
await msg.answer(text, parse_mode=ParseMode.HTML)
|
||||
|
||||
return report
|
||||
|
||||
async def do_push(msg: Message, state: FSMContext, member: User, *, overwrite: bool) -> None:
|
||||
draft = await drafts.get(member.tg_id)
|
||||
if draft is None or not draft.changes or not draft.message:
|
||||
@@ -291,6 +314,12 @@ def build_router(
|
||||
if commit_url_template:
|
||||
lines.append(commit_url_template.format(sha=result.sha))
|
||||
lines.append(t.ci_pending)
|
||||
if deployer is not None:
|
||||
services = deployer.schedule(
|
||||
result.sha, [c.path for c in draft.changes], deploy_report(msg, result.sha)
|
||||
)
|
||||
if services:
|
||||
lines.append(t.deploying.format(services=", ".join(services)))
|
||||
await msg.edit_text("\n".join(lines))
|
||||
await publisher.record(member, result, draft.message, draft, msg.chat.id, msg.message_id)
|
||||
await drafts.clear(member.tg_id)
|
||||
|
||||
@@ -13,7 +13,8 @@ from aiogram.utils.keyboard import InlineKeyboardBuilder
|
||||
|
||||
from app.bot.menu import Nav, NavAction
|
||||
from app.bot.routers.commit import human_size
|
||||
from app.db.models import utcnow
|
||||
from app.db.models import Role, User, utcnow
|
||||
from app.runner.client import RunnerClient
|
||||
from app.runtime.checks import Checker, CheckName, ContainerReport
|
||||
from app.runtime.docker import Container, Details, DockerClient, DockerError, Stats
|
||||
from app.texts import StatusTexts, Texts
|
||||
@@ -27,6 +28,7 @@ class StatusAct(StrEnum):
|
||||
LIST = "l"
|
||||
CARD = "c"
|
||||
CHECK = "k"
|
||||
RESTART = "r"
|
||||
|
||||
|
||||
class StatusCb(CallbackData, prefix="s"):
|
||||
@@ -75,7 +77,10 @@ def render_list(texts: Texts, containers: list[Container], now: datetime) -> tup
|
||||
return _clip("\n".join(lines)), kb.as_markup()
|
||||
|
||||
|
||||
def render_card(texts: Texts, d: Details, stats: Stats | None, now: datetime) -> tuple[str, InlineKeyboardMarkup]:
|
||||
def render_card(
|
||||
texts: Texts, d: Details, stats: Stats | None, now: datetime, can_restart: bool = False
|
||||
) -> tuple[str, InlineKeyboardMarkup]:
|
||||
"""can_restart: an admin looking at a service deployed by tgit-runner (S-5)."""
|
||||
t = texts.status
|
||||
running = d.state == "running"
|
||||
uptime = duration(t, (now - d.started_at).total_seconds()) if running and d.started_at else t.none
|
||||
@@ -105,11 +110,18 @@ def render_card(texts: Texts, d: Details, stats: Stats | None, now: datetime) ->
|
||||
kb = InlineKeyboardBuilder()
|
||||
kb.button(text=t.check_button, callback_data=StatusCb(a=StatusAct.CHECK, id=d.id[:12]))
|
||||
kb.button(text=t.refresh, callback_data=StatusCb(a=StatusAct.CARD, id=d.id[:12]))
|
||||
if can_restart and runner_service(d):
|
||||
kb.button(text=t.restart, callback_data=StatusCb(a=StatusAct.RESTART, id=d.id[:12]))
|
||||
kb.button(text=t.to_list, callback_data=StatusCb(a=StatusAct.LIST))
|
||||
kb.adjust(2, 1)
|
||||
kb.adjust(2, 1, 1)
|
||||
return text, kb.as_markup()
|
||||
|
||||
|
||||
def runner_service(d: Details) -> str | None:
|
||||
"""Only services tgit-runner deployed can be restarted through it — not the bot, its DB or the proxy."""
|
||||
return d.labels.get("tgit.service") or None
|
||||
|
||||
|
||||
def render_report(
|
||||
texts: Texts, reports: list[ContainerReport], now: datetime, back_id: str = ""
|
||||
) -> tuple[str, InlineKeyboardMarkup]:
|
||||
@@ -141,7 +153,9 @@ def render_report(
|
||||
return _clip("\n".join(lines)), kb.as_markup()
|
||||
|
||||
|
||||
def build_router(texts: Texts, docker: DockerClient | None, checker: Checker | None) -> Router:
|
||||
def build_router(
|
||||
texts: Texts, docker: DockerClient | None, checker: Checker | None, runner: RunnerClient | None = None
|
||||
) -> Router:
|
||||
router = Router(name="status")
|
||||
t = texts.status
|
||||
|
||||
@@ -150,7 +164,7 @@ def build_router(texts: Texts, docker: DockerClient | None, checker: Checker | N
|
||||
raise DockerError("docker is not configured (DOCKER_URL)")
|
||||
return render_list(texts, await docker.containers(), utcnow())
|
||||
|
||||
async def card_screen(short_id: str) -> tuple[str, InlineKeyboardMarkup]:
|
||||
async def card_screen(short_id: str, is_admin: bool = False) -> tuple[str, InlineKeyboardMarkup]:
|
||||
if docker is None:
|
||||
raise DockerError("docker is not configured (DOCKER_URL)")
|
||||
cid = await docker.resolve(short_id)
|
||||
@@ -161,7 +175,7 @@ def build_router(texts: Texts, docker: DockerClient | None, checker: Checker | N
|
||||
stats = await docker.stats(cid)
|
||||
except DockerError:
|
||||
log.warning("stats %s unavailable", short_id)
|
||||
return render_card(texts, details, stats, utcnow())
|
||||
return render_card(texts, details, stats, utcnow(), can_restart=is_admin and runner is not None)
|
||||
|
||||
@router.message(F.text == texts.menu.status)
|
||||
async def open_status(message: Message, state: FSMContext) -> None:
|
||||
@@ -175,7 +189,7 @@ def build_router(texts: Texts, docker: DockerClient | None, checker: Checker | N
|
||||
await message.answer(text, reply_markup=kb)
|
||||
|
||||
@router.callback_query(StatusCb.filter())
|
||||
async def on_action(query: CallbackQuery, callback_data: StatusCb) -> None:
|
||||
async def on_action(query: CallbackQuery, callback_data: StatusCb, member: User) -> None:
|
||||
if not isinstance(query.message, Message):
|
||||
await query.answer()
|
||||
return
|
||||
@@ -185,7 +199,23 @@ def build_router(texts: Texts, docker: DockerClient | None, checker: Checker | N
|
||||
case StatusAct.LIST:
|
||||
text, kb = await list_screen()
|
||||
case StatusAct.CARD:
|
||||
text, kb = await card_screen(callback_data.id)
|
||||
text, kb = await card_screen(callback_data.id, member.role is Role.ADMIN)
|
||||
case StatusAct.RESTART:
|
||||
if docker is None or runner is None or member.role is not Role.ADMIN:
|
||||
await query.answer()
|
||||
return
|
||||
service = runner_service(await docker.inspect(await docker.resolve(callback_data.id)))
|
||||
if service is None:
|
||||
await query.answer()
|
||||
return
|
||||
result = await runner.restart(service)
|
||||
if not result.ok:
|
||||
await query.answer(t.restart_failed.format(reason=result.summary)[:200], show_alert=True)
|
||||
return
|
||||
await query.answer(t.restarted)
|
||||
text, kb = await card_screen(callback_data.id, True)
|
||||
await _edit(msg, text, kb)
|
||||
return
|
||||
case StatusAct.CHECK:
|
||||
if docker is None or checker is None:
|
||||
raise DockerError("docker is not configured (DOCKER_URL)")
|
||||
|
||||
@@ -98,6 +98,18 @@ class DockerSettings(BaseSettings):
|
||||
max_log_lines: int = Field(default=50_000, gt=0)
|
||||
|
||||
|
||||
class RunnerSettings(BaseSettings):
|
||||
model_config = SettingsConfigDict(**_ENV, env_prefix="RUNNER_")
|
||||
|
||||
# SSH to tgit-runner on the host (ADR-006); empty host — services are not deployed from the bot
|
||||
host: str = ""
|
||||
port: int = 22
|
||||
user: str = "tgit-deploy"
|
||||
# The bot's runner key, base64; authorized_keys pins it to `command="tgit-runner"` (infra/runner/)
|
||||
ssh_key_b64: SecretStr | None = None
|
||||
timeout: float = Field(default=300.0, gt=0)
|
||||
|
||||
|
||||
class RuntimeSettings(BaseSettings):
|
||||
model_config = _ENV
|
||||
|
||||
@@ -120,4 +132,5 @@ class Settings(BaseSettings):
|
||||
sourcecraft: SourceCraftSettings = Field(default_factory=SourceCraftSettings)
|
||||
git: GitSettings = Field(default_factory=GitSettings)
|
||||
docker: DockerSettings = Field(default_factory=DockerSettings)
|
||||
runner: RunnerSettings = Field(default_factory=RunnerSettings)
|
||||
runtime: RuntimeSettings = Field(default_factory=RuntimeSettings)
|
||||
|
||||
+11
-3
@@ -29,6 +29,8 @@ from app.logs_setup import setup_logging
|
||||
from app.repo.drafts import DraftService
|
||||
from app.repo.git import UploadsRepo
|
||||
from app.repo.publisher import Publisher
|
||||
from app.runner.client import RunnerClient
|
||||
from app.runner.deployer import Deployer
|
||||
from app.runtime.checks import Checker, load_checks
|
||||
from app.runtime.docker import DockerClient
|
||||
from app.runtime.snapshots import SnapshotStore, snapshot_loop
|
||||
@@ -47,6 +49,7 @@ def build_dispatcher(
|
||||
checker: Checker | None = None,
|
||||
agent: Agent | None = None,
|
||||
alerts: AlertService | None = None,
|
||||
runner: RunnerClient | None = None,
|
||||
) -> Dispatcher:
|
||||
dp = Dispatcher(storage=make_storage(sessions))
|
||||
gate = AccessMiddleware(access, texts)
|
||||
@@ -54,10 +57,13 @@ def build_dispatcher(
|
||||
dp.message.outer_middleware(gate)
|
||||
dp.callback_query.outer_middleware(gate)
|
||||
dp.include_router(members.build_router(texts, access))
|
||||
deployer = Deployer(repo, runner, alerts) if runner else None
|
||||
dp.include_router(
|
||||
commit.build_router(texts, repo, DraftService(sessions), Publisher(repo, sessions), commit_url_template)
|
||||
commit.build_router(
|
||||
texts, repo, DraftService(sessions), Publisher(repo, sessions), commit_url_template, deployer
|
||||
)
|
||||
dp.include_router(status.build_router(texts, docker, checker))
|
||||
)
|
||||
dp.include_router(status.build_router(texts, docker, checker, runner))
|
||||
dp.include_router(logs.build_router(texts, docker, LogJournal(sessions), agent, alerts))
|
||||
dp.include_router(alerts_router.build_router(texts, alerts, docker))
|
||||
dp.include_router(common.build_router(texts, access))
|
||||
@@ -99,6 +105,7 @@ async def run(settings: Settings) -> None:
|
||||
checker,
|
||||
agent,
|
||||
alerts,
|
||||
RunnerClient(settings.runner) if settings.runner.host else None,
|
||||
)
|
||||
rt = settings.runtime
|
||||
background = [
|
||||
@@ -111,12 +118,13 @@ async def run(settings: Settings) -> None:
|
||||
asyncio.create_task(snapshot_loop(docker, store, settings.docker.snapshot_interval, monitor.on_round))
|
||||
)
|
||||
log.info(
|
||||
"tgit %s starting, %d bootstrap admin(s), telegram %s, docker %s, agent %s",
|
||||
"tgit %s starting, %d bootstrap admin(s), telegram %s, docker %s, agent %s, runner %s",
|
||||
__version__,
|
||||
len(settings.access.admin_ids),
|
||||
"via proxy" if proxy else "direct",
|
||||
"on" if docker else "off",
|
||||
settings.llm.provider if agent else "off",
|
||||
settings.runner.host or "off",
|
||||
)
|
||||
try:
|
||||
await dp.start_polling(bot, handle_signals=True)
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""Deploying services of the uploads repo: the bot ships `git archive` to `tgit-runner` on the host (#13)."""
|
||||
@@ -0,0 +1,85 @@
|
||||
"""SSH to `tgit-runner`: the key can run nothing else (forced command), so the «command» is just its arguments."""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from app.config import RunnerSettings
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SERVICE = re.compile(r"^[a-z][a-z0-9-]{1,30}$") # the runner checks the same — refuse early, with a clear reason
|
||||
SHA = re.compile(r"^[0-9a-f]{7,40}$")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RunnerResult:
|
||||
ok: bool
|
||||
code: int # 0 ok · 2 bad request · 3 not healthy · 4 limit · 5 build failed · 255 ssh · -1 timeout
|
||||
summary: str # the runner's last line without OK/FAIL
|
||||
output: str # everything, for logs and the user's log tail
|
||||
|
||||
@property
|
||||
def tail(self) -> str:
|
||||
lines = self.output.strip().splitlines()[:-1]
|
||||
return "\n".join(lines[-20:])
|
||||
|
||||
|
||||
class RunnerClient:
|
||||
def __init__(self, settings: RunnerSettings) -> None:
|
||||
self._s = settings
|
||||
self._key = self._materialize_key()
|
||||
|
||||
def _materialize_key(self) -> Path | None:
|
||||
if self._s.ssh_key_b64 is None:
|
||||
return None
|
||||
key = base64.b64decode(self._s.ssh_key_b64.get_secret_value())
|
||||
fd, name = tempfile.mkstemp(prefix="tgit-runner-")
|
||||
with os.fdopen(fd, "wb") as f:
|
||||
f.write(key if key.endswith(b"\n") else key + b"\n")
|
||||
return Path(name)
|
||||
|
||||
def _ssh(self) -> list[str]:
|
||||
cmd = ["ssh", "-F", "/dev/null", "-p", str(self._s.port), "-o", "BatchMode=yes", "-o", "ConnectTimeout=20"]
|
||||
cmd += ["-o", "StrictHostKeyChecking=accept-new", "-o", f"UserKnownHostsFile={tempfile.gettempdir()}/tgit-kh"]
|
||||
if self._key is not None:
|
||||
cmd += ["-i", str(self._key), "-o", "IdentitiesOnly=yes"]
|
||||
return [*cmd, f"{self._s.user}@{self._s.host}"]
|
||||
|
||||
async def run(self, *args: str, stdin: bytes = b"") -> RunnerResult:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*self._ssh(),
|
||||
*args,
|
||||
stdin=asyncio.subprocess.PIPE,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.STDOUT,
|
||||
)
|
||||
try:
|
||||
out, _ = await asyncio.wait_for(proc.communicate(stdin), timeout=self._s.timeout)
|
||||
except TimeoutError:
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
return RunnerResult(False, -1, "timeout", "")
|
||||
text = out.decode(errors="replace")
|
||||
last = text.strip().splitlines()[-1] if text.strip() else ""
|
||||
code = proc.returncode if proc.returncode is not None else -1
|
||||
ok = code == 0 and last.startswith("OK")
|
||||
summary = last.removeprefix("OK ").removeprefix("FAIL ") if last.startswith(("OK", "FAIL")) else last[:200]
|
||||
if not ok:
|
||||
log.warning("runner %s: exit %s: %s", args[0] if args else "", code, summary)
|
||||
return RunnerResult(ok, code, summary, text)
|
||||
|
||||
async def deploy(self, service: str, sha: str, archive: bytes) -> RunnerResult:
|
||||
if not SERVICE.match(service) or not SHA.match(sha):
|
||||
return RunnerResult(False, 2, "bad service name or sha", "")
|
||||
return await self.run("deploy", service, sha, stdin=archive)
|
||||
|
||||
async def restart(self, service: str) -> RunnerResult:
|
||||
if not SERVICE.match(service):
|
||||
return RunnerResult(False, 2, "bad service name", "")
|
||||
return await self.run("restart", service)
|
||||
@@ -0,0 +1,85 @@
|
||||
"""After a push: the services it touched are (re)deployed through tgit-runner, the pusher hears how it went.
|
||||
|
||||
A service = `services/<name>/`. Gone from the commit — removed on the host. A refused deploy keeps the old version
|
||||
running (the runner switches only to a healthy one) and raises ⛔ for the whole team.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable, Iterable
|
||||
from dataclasses import dataclass
|
||||
|
||||
from app.alerts.detectors import Condition
|
||||
from app.alerts.service import AlertService
|
||||
from app.db.models import AlertKind
|
||||
from app.repo.git import GitError, UploadsRepo
|
||||
from app.runner.client import SERVICE, RunnerClient, RunnerResult
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def changed_services(paths: Iterable[str]) -> list[str]:
|
||||
found = set()
|
||||
for path in paths:
|
||||
parts = path.split("/")
|
||||
if len(parts) >= 3 and parts[0] == "services" and SERVICE.match(parts[1]):
|
||||
found.add(parts[1])
|
||||
return sorted(found)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Outcome:
|
||||
service: str
|
||||
result: RunnerResult
|
||||
removed: bool = False
|
||||
|
||||
|
||||
Report = Callable[[Outcome], Awaitable[None]]
|
||||
|
||||
|
||||
class Deployer:
|
||||
def __init__(self, repo: UploadsRepo, runner: RunnerClient, alerts: AlertService | None = None) -> None:
|
||||
self._repo = repo
|
||||
self._runner = runner
|
||||
self._alerts = alerts
|
||||
self._lock = asyncio.Lock() # one deploy at a time: builds are heavy for the shared host
|
||||
self._tasks: set[asyncio.Task[None]] = set()
|
||||
|
||||
def schedule(self, sha: str, paths: Iterable[str], report: Report) -> list[str]:
|
||||
"""Start in the background; returns the services that will be deployed."""
|
||||
services = changed_services(paths)
|
||||
if services:
|
||||
task = asyncio.create_task(self.deploy(sha, services, report))
|
||||
self._tasks.add(task)
|
||||
task.add_done_callback(self._tasks.discard)
|
||||
return services
|
||||
|
||||
async def deploy(self, sha: str, services: list[str], report: Report) -> None:
|
||||
async with self._lock:
|
||||
for service in services:
|
||||
try:
|
||||
outcome = await self._one(sha, service)
|
||||
except Exception: # one broken service must not stop the others
|
||||
log.exception("deploy %s @ %s", service, sha[:7])
|
||||
outcome = Outcome(service, RunnerResult(False, -1, "internal error", ""))
|
||||
if not outcome.result.ok and self._alerts is not None:
|
||||
await self._alerts.event(
|
||||
Condition(
|
||||
AlertKind.DEPLOY_REFUSED,
|
||||
f"tgit-svc-{service}",
|
||||
{"detail": outcome.result.summary, "sha": sha[:7], "log": outcome.result.tail},
|
||||
)
|
||||
)
|
||||
try:
|
||||
await report(outcome)
|
||||
except Exception:
|
||||
log.exception("deploy report for %s", service)
|
||||
|
||||
async def _one(self, sha: str, service: str) -> Outcome:
|
||||
tree = f"{sha}:services/{service}"
|
||||
try:
|
||||
await self._repo.git("cat-file", "-e", tree)
|
||||
except GitError:
|
||||
return Outcome(service, await self._runner.run("remove", service), removed=True)
|
||||
archive = await self._repo.git("archive", "--format=tar", tree)
|
||||
return Outcome(service, await self._runner.deploy(service, sha, archive))
|
||||
@@ -119,6 +119,10 @@ class CommitTexts(_Strict):
|
||||
push_empty: str
|
||||
push_limit: str
|
||||
push_failed: str
|
||||
deploying: str
|
||||
deployed: str
|
||||
deploy_failed: str
|
||||
removed: str
|
||||
|
||||
|
||||
class StatusTexts(_Strict):
|
||||
@@ -148,6 +152,9 @@ class StatusTexts(_Strict):
|
||||
hours: str
|
||||
minutes: str
|
||||
seconds: str
|
||||
restart: str
|
||||
restarted: str
|
||||
restart_failed: str
|
||||
|
||||
|
||||
class LogsTexts(_Strict):
|
||||
|
||||
@@ -129,6 +129,12 @@ commit:
|
||||
push_empty: "В main уже всё так же — коммитить нечего. Черновик очищен."
|
||||
push_limit: "Лимит — 20 пушей в час. Попробуйте позже."
|
||||
push_failed: "Не удалось запушить: репозиторий недоступен. Черновик сохранён — попробуйте позже."
|
||||
deploying: "🚀 Разворачиваю: {services}…"
|
||||
deployed: "🚀 <b>{service}</b> работает · {sha}"
|
||||
deploy_failed: |-
|
||||
⛔ <b>{service}</b> не развёрнут: {reason}
|
||||
Работает прежняя версия, если она была.
|
||||
removed: "🗑 <b>{service}</b> удалён из репозитория — остановлен на хосте"
|
||||
|
||||
status:
|
||||
title: "🩺 Статус · {count} конт. · {time} UTC"
|
||||
@@ -163,6 +169,9 @@ status:
|
||||
hours: "ч"
|
||||
minutes: "мин"
|
||||
seconds: "с"
|
||||
restart: "♻️ Перезапустить"
|
||||
restarted: "♻️ Перезапущен"
|
||||
restart_failed: "Не удалось перезапустить: {reason}"
|
||||
|
||||
logs:
|
||||
title: "📜 Логи · выберите контейнер"
|
||||
|
||||
@@ -7,6 +7,9 @@ services:
|
||||
DATABASE_URL: postgresql+asyncpg://tgit:${POSTGRES_PASSWORD:-tgit}@postgres:5432/tgit
|
||||
GIT_CLONE_DIR: /data/uploads
|
||||
DOCKER_URL: http://docker-proxy:2375
|
||||
# tgit-runner is reached over SSH on the host itself (RUNNER_HOST=host.docker.internal)
|
||||
extra_hosts:
|
||||
- host.docker.internal:host-gateway
|
||||
volumes:
|
||||
- uploads:/data
|
||||
depends_on:
|
||||
|
||||
@@ -203,7 +203,9 @@ SQLAlchemy 2 (async) + Alembic. PostgreSQL в Compose; в тестах — SQLit
|
||||
`authorized_keys` пользователя `tgit-deploy`). Подкоманды: `deploy <service> <sha>`, `restart <service>`,
|
||||
`probe <service> <path>` (HTTP-проба изнутри сети сервиса). Аргументы валидируются: имя `^[a-z][a-z0-9-]{1,30}$`, sha — hex.
|
||||
|
||||
`deploy`: забирает архив `services/<service>` по sha из `tgit-uploads` → собирает образ `tgit-svc-<service>:<sha>` из
|
||||
`deploy`: получает архив `services/<service>` в stdin — его делает бот (`git archive <sha>:services/<service>`)
|
||||
сразу после push, так что у хоста нет доступа к репозиторию → проверяет архив (только файлы и папки, без `..`,
|
||||
≤ 20 МБ, есть `main.py`) → собирает образ `tgit-svc-<service>:<sha>` из
|
||||
шаблона (`python:3.12-slim` + `requirements.txt`, `HEALTHCHECK` из `tgit.yaml` сервиса, если задан) → запускает:
|
||||
|
||||
```
|
||||
@@ -216,7 +218,11 @@ docker run -d --name tgit-svc-<service> --label tgit.managed=true --label tgit.s
|
||||
--restart on-failure:3
|
||||
--log-opt max-size=5m --log-opt max-file=2
|
||||
```
|
||||
Если новый контейнер не стал healthy за 60 с — откат на предыдущий образ и ненулевой код выхода (CD падает → алерт ⛔).
|
||||
Новая версия стартует рядом со старой как `tgit-svc-<service>-next` и заменяет её, только став healthy (с
|
||||
`health_url` — по `HEALTHCHECK`, без него — 10 с без рестартов). Иначе новый контейнер удаляется, старый продолжает
|
||||
работать, код выхода 3 → бот пишет автору push «⛔ не развёрнут» с хвостом лога и поднимает алерт ⛔ всей команде.
|
||||
Сервис, папка которого исчезла из коммита, удаляется (`remove`). Перезапуск — `restart` из «Статуса» (админ).
|
||||
Установка и обновление — [`infra/runner/README.md`](../infra/runner/README.md).
|
||||
|
||||
## 5. CI/CD
|
||||
|
||||
|
||||
@@ -15,6 +15,12 @@
|
||||
|
||||
После `make demo-down` открытые алерты закрываются сообщением «✅ Восстановлено».
|
||||
|
||||
## Песочница — `tgit-runner`
|
||||
|
||||
[`escape_attempts/`](escape_attempts/) — сервис, который пробует выбраться: запись в код и `/etc`, сеть, DNS,
|
||||
docker.sock, `setuid(0)`, raw-сокет, 200 процессов, запуск из `/tmp`. Положите его в `services/escape/` репозитория
|
||||
загрузок — в «📜 Логи» каждая строка должна быть `BLOCKED`.
|
||||
|
||||
## Коммит — «📝 Коммит»
|
||||
|
||||
| Файл | Что показывает |
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Tries to break out of the tgit-runner sandbox and reports each attempt. Every line should say BLOCKED.
|
||||
|
||||
Deploy it like any service (put it into services/escape/ of the uploads repo) and read its log in «📜 Логи».
|
||||
Memory is not probed here: past 128 MB the kernel OOM-kills the process — the limit works, but the report would die.
|
||||
"""
|
||||
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
|
||||
def attempt(name: str, action) -> None: # type: ignore[no-untyped-def]
|
||||
try:
|
||||
detail = action()
|
||||
except Exception as e: # each failure is the expected outcome
|
||||
print(f"BLOCKED {name}: {type(e).__name__}: {e}", flush=True)
|
||||
return
|
||||
print(f"ALLOWED {name}: {detail}", flush=True)
|
||||
|
||||
|
||||
def write_code() -> str:
|
||||
with open("/app/pwned.txt", "w") as f:
|
||||
f.write("x")
|
||||
return "wrote into /app"
|
||||
|
||||
|
||||
def write_root() -> str:
|
||||
with open("/etc/pwned", "w") as f:
|
||||
f.write("x")
|
||||
return "wrote into /etc"
|
||||
|
||||
|
||||
def internet() -> str:
|
||||
socket.create_connection(("1.1.1.1", 443), timeout=3).close()
|
||||
return "reached 1.1.1.1:443"
|
||||
|
||||
|
||||
def dns() -> str:
|
||||
return socket.gethostbyname("example.com")
|
||||
|
||||
|
||||
def docker_socket() -> str:
|
||||
s = socket.socket(socket.AF_UNIX)
|
||||
s.connect("/var/run/docker.sock")
|
||||
return "connected to docker.sock"
|
||||
|
||||
|
||||
def become_root() -> str:
|
||||
os.setuid(0)
|
||||
return "setuid(0) worked"
|
||||
|
||||
|
||||
def raw_socket() -> str:
|
||||
socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_ICMP)
|
||||
return "opened a raw socket"
|
||||
|
||||
|
||||
def fork_bomb() -> str:
|
||||
procs = []
|
||||
try:
|
||||
for _ in range(200):
|
||||
procs.append(subprocess.Popen(["sleep", "30"]))
|
||||
finally:
|
||||
started = len(procs)
|
||||
for p in procs:
|
||||
p.kill()
|
||||
if started >= 200:
|
||||
return "started 200 processes"
|
||||
raise RuntimeError(f"stopped after {started} processes")
|
||||
|
||||
|
||||
def exec_tmp() -> str:
|
||||
path = "/tmp/run.sh"
|
||||
with open(path, "w") as f:
|
||||
f.write("#!/bin/sh\necho hi\n")
|
||||
os.chmod(path, 0o755)
|
||||
return subprocess.run([path], capture_output=True, check=True, text=True).stdout.strip()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(f"uid={os.getuid()} gid={os.getgid()}", flush=True)
|
||||
for name, fn in [
|
||||
("write into the code dir", write_code),
|
||||
("write into /etc", write_root),
|
||||
("connect to the internet", internet),
|
||||
("resolve DNS", dns),
|
||||
("talk to docker.sock", docker_socket),
|
||||
("setuid(0)", become_root),
|
||||
("raw socket", raw_socket),
|
||||
("200 processes", fork_bomb),
|
||||
("execute from /tmp", exec_tmp),
|
||||
]:
|
||||
attempt(name, fn)
|
||||
print("done", flush=True)
|
||||
while True:
|
||||
time.sleep(3600)
|
||||
+1
-1
@@ -3,5 +3,5 @@
|
||||
| Каталог | Что будет | Задача |
|
||||
|---|---|---|
|
||||
| [`deploy/`](deploy/) | `deploy.sh` + `remote.sh` — деплой бота на сервер (CD и руками), см. [`docs/dev/DEPLOY.md`](../docs/dev/DEPLOY.md) | #12 |
|
||||
| `runner/` | `tgit-runner` — песочница и запуск сервисов на хосте | #13 |
|
||||
| [`runner/`](runner/) | `tgit-runner` — песочница и запуск сервисов на хосте, установка | #13 |
|
||||
| [`uploads-template/`](uploads-template/) | шаблон [`tgit-uploads`](https://sourcecraft.dev/rogachevegorushka/tgit-uploads): CI/CD, служебные скрипты `.tgit/`, `ruff.toml`, демо-сервис `hello` — правки вносить здесь и переносить в репозиторий загрузок | #7 |
|
||||
|
||||
@@ -35,5 +35,9 @@ if [ "$status" != healthy ]; then
|
||||
exit 1
|
||||
fi
|
||||
echo "$SHA" > ../REVISION
|
||||
# Keep tgit-runner in step with the bot (installed once by infra/runner/install.sh; absent — nothing to update)
|
||||
if [ -d /srv/tgit/runner ] && [ -w /srv/tgit/runner ]; then
|
||||
install -m 755 infra/runner/tgit-runner /srv/tgit/runner/tgit-runner
|
||||
fi
|
||||
docker image prune -f >/dev/null
|
||||
echo "bot healthy @ $SHA ($ENV)"
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
# infra/runner — `tgit-runner`
|
||||
|
||||
Запуск сервисов из репозитория загрузок (`services/<name>/`) на хосте в песочнице — ADR-005, ADR-006,
|
||||
[ARCHITECTURE §4](../../docs/ARCHITECTURE.md).
|
||||
|
||||
```
|
||||
бот: push ─► git archive <sha>:services/<name> ─► ssh tgit-deploy@host deploy <name> <sha> (архив — в stdin)
|
||||
│ forced command: только tgit-runner
|
||||
host: tgit-runner ─► проверка аргументов и архива ─► docker build ─► docker run …-next (песочница)
|
||||
─► healthy за 60 с? ─► да: заменить старый контейнер · нет: удалить новый, старый работает, exit 3
|
||||
```
|
||||
|
||||
| Команда | Что делает |
|
||||
|---|---|
|
||||
| `deploy <service> <sha>` | собрать образ `tgit-svc-<service>:<sha>` и запустить, старый заменяется только здоровым новым |
|
||||
| `restart <service>` | перезапуск (кнопка «♻️ Перезапустить» в «Статусе», только админ) |
|
||||
| `remove <service>` | сервис удалён из репозитория — остановить и удалить образы |
|
||||
| `list` | контейнеры сервисов |
|
||||
|
||||
**Песочница:** uid 10001, read-only FS, `/tmp` — tmpfs 16 МБ с `noexec`, `--cap-drop ALL`, `no-new-privileges`,
|
||||
128 МБ без swap, 0.25 CPU, 64 процесса, `--network none` (интернет — только если `network: true` в `tgit.yaml`
|
||||
**и** имя сервиса есть в `/srv/tgit/runner/egress-allow`), без Docker-сокета, `--restart on-failure:3`, ротация логов.
|
||||
Не больше 10 сервисов на хост; хранятся два последних образа сервиса.
|
||||
|
||||
**Проверено** [`examples/escape_attempts`](../../examples/escape_attempts/): запись в код и `/etc`, сеть, DNS,
|
||||
docker.sock, `setuid(0)`, raw-сокет, 200 процессов, запуск из `/tmp` — всё `BLOCKED`.
|
||||
|
||||
**Установка** (один раз, root на хосте):
|
||||
|
||||
```bash
|
||||
ssh-keygen -t ed25519 -N '' -C tgit-runner -f tgit-runner-key # ключ бота для раннера
|
||||
sudo infra/runner/install.sh tgit-runner-key.pub tgit-deploy
|
||||
# в .env бота на сервере:
|
||||
# RUNNER_HOST=host.docker.internal RUNNER_SSH_KEY_B64=$(base64 -w0 tgit-runner-key)
|
||||
```
|
||||
|
||||
Дальше скрипт обновляется сам: деплой бота (`infra/deploy/remote.sh`) копирует свежий `tgit-runner`, если раннер
|
||||
установлен. Тесты — `tests/runner/` (фейковые `docker` и `ssh`, сам скрипт настоящий).
|
||||
@@ -0,0 +1,6 @@
|
||||
# ~tgit-deploy/.ssh/authorized_keys on the host. `restrict` = no-pty, no port/agent/X11 forwarding;
|
||||
# command= makes the key run tgit-runner and nothing else, the requested command arrives as SSH_ORIGINAL_COMMAND.
|
||||
command="/srv/tgit/runner/tgit-runner",restrict ssh-ed25519 AAAA…bot-runner-key… tgit-runner
|
||||
|
||||
# The CD key of the bot itself stays unrestricted (it deploys the bot; Gitea secret DEPLOY_SSH_KEY):
|
||||
ssh-ed25519 AAAA…cd-key… tgit-cd@gitea
|
||||
Executable
+28
@@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env bash
|
||||
# One-time setup of tgit-runner on the host (as root). Safe to re-run.
|
||||
# sudo infra/runner/install.sh /path/to/tgit-runner-key.pub [deploy-user]
|
||||
# The key is the bot's runner key (RUNNER_SSH_KEY_B64 holds its private half); it is pinned to the runner only.
|
||||
set -euo pipefail
|
||||
PUB=${1:?usage: install.sh <runner-key.pub> [user]}
|
||||
USER_NAME=${2:-tgit-deploy}
|
||||
DIR=/srv/tgit/runner
|
||||
HERE=$(cd "$(dirname "$0")" && pwd)
|
||||
|
||||
id "$USER_NAME" >/dev/null
|
||||
install -d -o "$USER_NAME" -g "$USER_NAME" -m 700 "$DIR"
|
||||
install -o "$USER_NAME" -g "$USER_NAME" -m 755 "$HERE/tgit-runner" "$DIR/tgit-runner"
|
||||
touch "$DIR/egress-allow" && chown "$USER_NAME:" "$DIR/egress-allow" && chmod 600 "$DIR/egress-allow"
|
||||
|
||||
HOME_DIR=$(getent passwd "$USER_NAME" | cut -d: -f6)
|
||||
install -d -o "$USER_NAME" -g "$USER_NAME" -m 700 "$HOME_DIR/.ssh"
|
||||
AK=$HOME_DIR/.ssh/authorized_keys
|
||||
touch "$AK" && chown "$USER_NAME:" "$AK" && chmod 600 "$AK"
|
||||
KEY=$(awk '{print $1" "$2}' "$PUB")
|
||||
LINE="command=\"$DIR/tgit-runner\",restrict $KEY tgit-runner"
|
||||
if grep -qF "$KEY" "$AK"; then
|
||||
echo "key already in $AK — leaving it"
|
||||
else
|
||||
echo "$LINE" >>"$AK"
|
||||
echo "added the runner key to $AK (forced command)"
|
||||
fi
|
||||
echo "runner installed in $DIR; allow internet for a service: echo <service> >> $DIR/egress-allow"
|
||||
Executable
+184
@@ -0,0 +1,184 @@
|
||||
#!/usr/bin/env bash
|
||||
# tgit-runner — the only command the bot's SSH key may run on the host (ADR-006, ARCHITECTURE §4).
|
||||
# authorized_keys: command="/srv/tgit/runner/tgit-runner",restrict ssh-ed25519 AAAA… tgit-runner
|
||||
#
|
||||
# deploy <service> <sha> stdin: tar of services/<service>/ (files at the archive root)
|
||||
# restart <service>
|
||||
# remove <service>
|
||||
# list
|
||||
#
|
||||
# Exit codes: 0 ok · 2 bad request · 3 not healthy (old container kept) · 4 limit · 5 build failed.
|
||||
# The last line of output is machine-readable: «OK …» or «FAIL <reason>».
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
export LC_ALL=C
|
||||
|
||||
STATE=${TGIT_RUNNER_STATE:-/srv/tgit/runner}
|
||||
MAX_SERVICES=${TGIT_MAX_SERVICES:-10}
|
||||
HEALTH_TIMEOUT=${TGIT_HEALTH_TIMEOUT:-60}
|
||||
SETTLE=${TGIT_SETTLE:-10} # no healthcheck: must stay up this long without restarts
|
||||
MAX_TAR=${TGIT_MAX_TAR:-$((20 * 1024 * 1024))}
|
||||
BASE_IMAGE=${TGIT_BASE_IMAGE:-python:3.12-slim}
|
||||
EGRESS_NET=tgit-svc-egress
|
||||
NAME_RE='^[a-z][a-z0-9-]{1,30}$'
|
||||
SHA_RE='^[0-9a-f]{7,40}$'
|
||||
HEALTH_RE='^http://(localhost|127\.0\.0\.1):[0-9]{2,5}(/[A-Za-z0-9._/-]*)?$'
|
||||
|
||||
fail() { echo "FAIL $2"; exit "$1"; }
|
||||
|
||||
# Called through SSH the arguments come in SSH_ORIGINAL_COMMAND: split on spaces, never evaluated, never globbed
|
||||
if [ -n "${SSH_ORIGINAL_COMMAND+x}" ]; then
|
||||
set -f
|
||||
read -r -a ARGS <<<"$SSH_ORIGINAL_COMMAND"
|
||||
set +f
|
||||
else
|
||||
ARGS=("$@")
|
||||
fi
|
||||
[ "${#ARGS[@]}" -ge 1 ] || fail 2 "usage: deploy|restart|remove|list"
|
||||
CMD=${ARGS[0]}
|
||||
|
||||
need_service() {
|
||||
[ "${#ARGS[@]}" -eq "$1" ] || fail 2 "wrong number of arguments for $CMD"
|
||||
SVC=${ARGS[1]}
|
||||
[[ $SVC =~ $NAME_RE ]] || fail 2 "bad service name"
|
||||
NAME=tgit-svc-$SVC
|
||||
}
|
||||
|
||||
exists() { docker container inspect "$1" >/dev/null 2>&1; }
|
||||
|
||||
# tgit.yaml is read with grep, not executed or parsed by anything clever
|
||||
yaml_value() { grep -E "^$1:" "$2" 2>/dev/null | head -1 | sed -E "s/^$1:[[:space:]]*//; s/[[:space:]]+#.*$//; s/^[\"']//; s/[\"']$//" || true; }
|
||||
|
||||
wait_ready() { # $1 container, $2 has healthcheck (yes/no)
|
||||
local deadline=$((SECONDS + HEALTH_TIMEOUT)) status restarts
|
||||
if [ "$2" = yes ]; then
|
||||
while [ $SECONDS -lt $deadline ]; do
|
||||
status=$(docker inspect -f '{{.State.Health.Status}}' "$1" 2>/dev/null || echo gone)
|
||||
case $status in healthy) return 0 ;; unhealthy | gone) return 1 ;; esac
|
||||
sleep 2
|
||||
done
|
||||
return 1
|
||||
fi
|
||||
sleep "$SETTLE"
|
||||
status=$(docker inspect -f '{{.State.Status}}' "$1" 2>/dev/null || echo gone)
|
||||
restarts=$(docker inspect -f '{{.RestartCount}}' "$1" 2>/dev/null || echo 1)
|
||||
[ "$status" = running ] && [ "$restarts" = 0 ]
|
||||
}
|
||||
|
||||
deploy() {
|
||||
need_service 3
|
||||
SHA=${ARGS[2]}
|
||||
[[ $SHA =~ $SHA_RE ]] || fail 2 "bad sha"
|
||||
|
||||
if ! exists "$NAME"; then
|
||||
local count
|
||||
count=$(docker ps -a --filter label=tgit.service --format '{{.Names}}' | grep -cv -- '-next$' || true)
|
||||
[ "$count" -lt "$MAX_SERVICES" ] || fail 4 "service limit $MAX_SERVICES reached"
|
||||
fi
|
||||
|
||||
mkdir -p "$STATE"
|
||||
WORK=$(mktemp -d "$STATE/build.XXXXXX")
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
head -c $((MAX_TAR + 1)) >"$WORK/src.tar"
|
||||
[ "$(stat -c %s "$WORK/src.tar")" -le "$MAX_TAR" ] || fail 2 "archive larger than $MAX_TAR bytes"
|
||||
# Only plain files and directories, relative paths without "..": no symlinks, devices or hard links
|
||||
tar -tvf "$WORK/src.tar" >"$WORK/list" 2>/dev/null || fail 2 "not a tar archive"
|
||||
if grep -qvE '^[-d]' "$WORK/list"; then fail 2 "archive may contain only files and directories"; fi
|
||||
if tar -tf "$WORK/src.tar" | grep -qE '(^/|(^|/)\.\.(/|$))'; then fail 2 "unsafe path in archive"; fi
|
||||
mkdir -p "$WORK/ctx/app"
|
||||
tar -xf "$WORK/src.tar" -C "$WORK/ctx/app" --no-same-owner --no-same-permissions
|
||||
[ -f "$WORK/ctx/app/main.py" ] || fail 2 "services/$SVC/main.py is missing"
|
||||
# umask 077 above protects the build dir; inside the image the code must be readable (not writable) by uid 10001
|
||||
chmod -R u=rwX,go=rX "$WORK/ctx"
|
||||
|
||||
local health network healthcheck=no net=none
|
||||
health=$(yaml_value health_url "$WORK/ctx/app/tgit.yaml")
|
||||
network=$(yaml_value network "$WORK/ctx/app/tgit.yaml")
|
||||
if [ -n "$health" ]; then
|
||||
[[ $health =~ $HEALTH_RE ]] || fail 2 "health_url must be http://localhost:<port>/<path>"
|
||||
healthcheck=yes
|
||||
fi
|
||||
# Internet only if tgit.yaml asks AND the admin allowed this service on the host
|
||||
if [ "$network" = true ] && grep -qxF "$SVC" "$STATE/egress-allow" 2>/dev/null; then
|
||||
docker network inspect "$EGRESS_NET" >/dev/null 2>&1 || docker network create "$EGRESS_NET" >/dev/null
|
||||
net=$EGRESS_NET
|
||||
fi
|
||||
|
||||
{
|
||||
echo "FROM $BASE_IMAGE"
|
||||
echo "RUN useradd --uid 10001 --no-create-home --home-dir /tmp svc"
|
||||
echo "WORKDIR /app"
|
||||
echo "COPY app/ /app/"
|
||||
echo "RUN if [ -f requirements.txt ]; then pip install --no-cache-dir --disable-pip-version-check -r requirements.txt; fi"
|
||||
echo "ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 HOME=/tmp"
|
||||
if [ "$healthcheck" = yes ]; then
|
||||
# exec form, the URL is an argument — nothing here goes through a shell
|
||||
printf 'HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 CMD ["python", "-c", "import sys, urllib.request; urllib.request.urlopen(sys.argv[1], timeout=2)", "%s"]\n' "$health"
|
||||
fi
|
||||
echo "USER 10001:10001"
|
||||
echo 'CMD ["python", "main.py"]'
|
||||
} >"$WORK/ctx/Dockerfile"
|
||||
|
||||
IMAGE=$NAME:$SHA
|
||||
echo "building $IMAGE"
|
||||
if ! docker build -q -t "$IMAGE" --label tgit.service="$SVC" "$WORK/ctx" >"$WORK/build.log" 2>&1; then
|
||||
tail -20 "$WORK/build.log"
|
||||
fail 5 "build failed"
|
||||
fi
|
||||
|
||||
NEXT=$NAME-next
|
||||
docker rm -f "$NEXT" >/dev/null 2>&1 || true
|
||||
docker run -d --name "$NEXT" \
|
||||
--label tgit.managed=true --label tgit.service="$SVC" \
|
||||
--label tgit.sha="$SHA" --label tgit.deployed_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
|
||||
--user 10001:10001 --read-only --tmpfs /tmp:rw,noexec,nosuid,size=16m \
|
||||
--cap-drop ALL --security-opt no-new-privileges \
|
||||
--memory 128m --memory-swap 128m --cpus 0.25 --pids-limit 64 \
|
||||
--network "$net" --restart on-failure:3 \
|
||||
--log-opt max-size=5m --log-opt max-file=2 \
|
||||
"$IMAGE" >/dev/null
|
||||
|
||||
if ! wait_ready "$NEXT" "$healthcheck"; then
|
||||
echo "--- last log lines of the new version ---"
|
||||
docker logs --tail 20 "$NEXT" 2>&1 || true
|
||||
docker rm -f "$NEXT" >/dev/null 2>&1 || true
|
||||
if exists "$NAME"; then fail 3 "not healthy within ${HEALTH_TIMEOUT}s, previous version kept"; fi
|
||||
fail 3 "not healthy within ${HEALTH_TIMEOUT}s"
|
||||
fi
|
||||
docker rm -f "$NAME" >/dev/null 2>&1 || true
|
||||
docker rename "$NEXT" "$NAME"
|
||||
|
||||
# Keep the current image and one previous; older ones go
|
||||
{ docker image ls "$NAME" --format '{{.Tag}}' | grep -vxF "$SHA" || true; } | tail -n +2 | while read -r old; do
|
||||
docker image rm "$NAME:$old" >/dev/null 2>&1 || true
|
||||
done
|
||||
echo "OK deployed $SVC $SHA network=$net healthcheck=$healthcheck"
|
||||
}
|
||||
|
||||
restart() {
|
||||
need_service 2
|
||||
exists "$NAME" || fail 2 "no such service"
|
||||
docker restart -t 10 "$NAME" >/dev/null
|
||||
echo "OK restarted $SVC"
|
||||
}
|
||||
|
||||
remove() {
|
||||
need_service 2
|
||||
docker rm -f "$NAME" "$NAME-next" >/dev/null 2>&1 || true
|
||||
docker image ls "$NAME" --format '{{.Repository}}:{{.Tag}}' | while read -r img; do
|
||||
docker image rm "$img" >/dev/null 2>&1 || true
|
||||
done
|
||||
echo "OK removed $SVC"
|
||||
}
|
||||
|
||||
case $CMD in
|
||||
deploy) deploy ;;
|
||||
restart) restart ;;
|
||||
remove) remove ;;
|
||||
list)
|
||||
[ "${#ARGS[@]}" -eq 1 ] || fail 2 "list takes no arguments"
|
||||
docker ps -a --filter label=tgit.service --format '{{.Names}} {{.Status}} {{.Label "tgit.sha"}}'
|
||||
echo "OK list"
|
||||
;;
|
||||
*) fail 2 "unknown command" ;;
|
||||
esac
|
||||
@@ -47,5 +47,6 @@ workflows:
|
||||
script:
|
||||
- python .tgit/changed_services.py > changed.txt
|
||||
- cat changed.txt
|
||||
# Запуск через tgit-runner по SSH — #13. До этого — только список сервисов к развёртыванию.
|
||||
- 'echo "deploy stub: $(wc -l < changed.txt) service(s) to deploy"'
|
||||
# Разворачивает бот сразу после push (tgit-runner, #13); здесь — только список для лога.
|
||||
# Когда CI переедет сюда целиком (#11), бот будет ждать зелёного check перед деплоем.
|
||||
- 'echo "$(wc -l < changed.txt) service(s) changed — deployed by the bot via tgit-runner"'
|
||||
|
||||
Executable
+83
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env python3
|
||||
"""A fake `docker` CLI for tgit-runner tests: records every call, keeps containers in a JSON file.
|
||||
|
||||
Behaviour knobs (env): FAKE_HEALTH (healthy|unhealthy|none), FAKE_STATUS (running|exited), FAKE_RESTARTS,
|
||||
FAKE_BUILD_FAIL=1. State: $FAKE_DOCKER_STATE (json), calls: $FAKE_DOCKER_LOG (one json array per line).
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
state_path = os.environ["FAKE_DOCKER_STATE"]
|
||||
args = sys.argv[1:]
|
||||
with open(os.environ["FAKE_DOCKER_LOG"], "a") as log:
|
||||
log.write(json.dumps(args) + "\n")
|
||||
try:
|
||||
with open(state_path) as f:
|
||||
state = json.load(f)
|
||||
except FileNotFoundError:
|
||||
state = {"containers": {}, "images": []}
|
||||
|
||||
|
||||
def save() -> None:
|
||||
with open(state_path, "w") as f:
|
||||
json.dump(state, f)
|
||||
|
||||
|
||||
def done(code: int = 0, out: str = "") -> None:
|
||||
if out:
|
||||
print(out)
|
||||
save()
|
||||
sys.exit(code)
|
||||
|
||||
|
||||
cmd = args[0] if args else ""
|
||||
if cmd == "container" and args[1] == "inspect":
|
||||
done(0 if args[2] in state["containers"] else 1)
|
||||
if cmd == "inspect":
|
||||
fmt, name = args[2], args[3]
|
||||
if name not in state["containers"]:
|
||||
done(1)
|
||||
if "Health" in fmt:
|
||||
done(0, os.environ.get("FAKE_HEALTH", "healthy"))
|
||||
if "RestartCount" in fmt:
|
||||
done(0, os.environ.get("FAKE_RESTARTS", "0"))
|
||||
done(0, os.environ.get("FAKE_STATUS", "running"))
|
||||
if cmd == "ps":
|
||||
done(0, "\n".join(n for n in state["containers"]))
|
||||
if cmd == "build":
|
||||
if os.environ.get("FAKE_BUILD_FAIL"):
|
||||
done(1, "ERROR: could not install requirements")
|
||||
tag = args[args.index("-t") + 1]
|
||||
ctx = args[-1]
|
||||
with open(os.path.join(ctx, "Dockerfile")) as f:
|
||||
state.setdefault("dockerfiles", {})[tag] = f.read()
|
||||
state["images"].append(tag)
|
||||
done(0, "sha256:feed")
|
||||
if cmd == "run":
|
||||
name = args[args.index("--name") + 1]
|
||||
state["containers"][name] = {"image": args[-1], "args": args}
|
||||
done(0, "cid")
|
||||
if cmd == "rm":
|
||||
for name in args[2:]:
|
||||
state["containers"].pop(name, None)
|
||||
done(0)
|
||||
if cmd == "rename":
|
||||
state["containers"][args[2]] = state["containers"].pop(args[1])
|
||||
done(0)
|
||||
if cmd == "restart":
|
||||
done(0)
|
||||
if cmd == "logs":
|
||||
done(0, "Traceback: boom")
|
||||
if cmd == "image" and args[1] == "ls":
|
||||
repo = args[2]
|
||||
tags = [t.split(":", 1)[1] for t in reversed(state["images"]) if t.split(":", 1)[0] == repo]
|
||||
full = "{{.Repository}}" in args[-1]
|
||||
done(0, "\n".join(f"{repo}:{t}" if full else t for t in tags))
|
||||
if cmd == "image" and args[1] == "rm":
|
||||
state["images"] = [t for t in state["images"] if t != args[2]]
|
||||
done(0)
|
||||
if cmd == "network":
|
||||
done(0)
|
||||
done(2, f"fake docker: unsupported {args}")
|
||||
Executable
+14
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env python3
|
||||
"""A fake `ssh`: like sshd with a forced command — the remote command becomes SSH_ORIGINAL_COMMAND of tgit-runner."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
args = sys.argv[1:]
|
||||
with_value = {"-F", "-p", "-o", "-i"}
|
||||
i = 0
|
||||
while i < len(args) and args[i].startswith("-"):
|
||||
i += 2 if args[i] in with_value else 1
|
||||
remote = " ".join(args[i + 1 :])
|
||||
os.environ["SSH_ORIGINAL_COMMAND"] = remote
|
||||
os.execvp("bash", ["bash", os.environ["FAKE_RUNNER"]])
|
||||
@@ -0,0 +1,234 @@
|
||||
"""Bot side of #13: the SSH client, the deployer and the push → deploy flow, down to the real tgit-runner script."""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import subprocess
|
||||
from collections.abc import Callable
|
||||
from dataclasses import replace
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from aiogram import Bot, Dispatcher
|
||||
from aiogram.methods import AnswerCallbackQuery, EditMessageText, SendMessage
|
||||
from aiogram.types import InlineKeyboardMarkup
|
||||
|
||||
from app.access.service import AccessService
|
||||
from app.alerts.broadcast import TelegramNotifier
|
||||
from app.alerts.detectors import load_alerts_config
|
||||
from app.alerts.service import AlertService
|
||||
from app.bot.routers.commit import Act, CommitCb
|
||||
from app.bot.routers.status import StatusAct, StatusCb, render_card, runner_service
|
||||
from app.config import RunnerSettings
|
||||
from app.db.models import Alert, AlertKind
|
||||
from app.db.session import SessionMaker
|
||||
from app.main import build_dispatcher
|
||||
from app.repo.git import UploadsRepo
|
||||
from app.runner.client import RunnerClient, RunnerResult
|
||||
from app.runner.deployer import Deployer, Outcome, changed_services
|
||||
from app.texts import Texts
|
||||
from tests.alerts.test_detectors import RUNNING
|
||||
from tests.conftest import ADMIN, ROOT
|
||||
from tests.fakes import TgClient
|
||||
from tests.runner.test_tgit_runner import FAKE, RUNNER
|
||||
|
||||
HERE = Path(__file__).parent
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_host(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
|
||||
"""ssh → tgit-runner → fake docker, all on PATH."""
|
||||
bin_dir = tmp_path / "bin"
|
||||
bin_dir.mkdir()
|
||||
(bin_dir / "ssh").symlink_to(HERE / "fake_ssh.py")
|
||||
(bin_dir / "docker").symlink_to(FAKE)
|
||||
monkeypatch.setenv("PATH", f"{bin_dir}:/usr/bin:/bin")
|
||||
monkeypatch.setenv("FAKE_RUNNER", str(RUNNER))
|
||||
monkeypatch.setenv("FAKE_DOCKER_STATE", str(tmp_path / "state.json"))
|
||||
monkeypatch.setenv("FAKE_DOCKER_LOG", str(tmp_path / "calls.log"))
|
||||
monkeypatch.setenv("TGIT_RUNNER_STATE", str(tmp_path / "runner"))
|
||||
monkeypatch.setenv("TGIT_HEALTH_TIMEOUT", "2")
|
||||
monkeypatch.setenv("TGIT_SETTLE", "0")
|
||||
return tmp_path
|
||||
|
||||
|
||||
def docker_state(host: Path) -> dict[str, object]:
|
||||
return json.loads((host / "state.json").read_text())
|
||||
|
||||
|
||||
def client(key: bytes | None = None) -> RunnerClient:
|
||||
b64 = base64.b64encode(key).decode() if key else None
|
||||
return RunnerClient(RunnerSettings(host="host.example", ssh_key_b64=b64, timeout=30)) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_changed_services() -> None:
|
||||
paths = ["services/hello/main.py", "services/hello/x/y.py", "services/Bad/main.py", "README.md", "services/a.py"]
|
||||
assert changed_services(paths) == ["hello"]
|
||||
|
||||
|
||||
async def test_client_validates_before_ssh(fake_host: Path) -> None:
|
||||
result = await client().deploy("../etc", "abc1234", b"")
|
||||
|
||||
assert (result.ok, result.code) == (False, 2)
|
||||
assert not (fake_host / "calls.log").exists()
|
||||
|
||||
|
||||
def archive_of(remote: Path) -> bytes:
|
||||
return subprocess.run(
|
||||
["git", "archive", "--format=tar", "HEAD:services/hello"], cwd=remote, capture_output=True, check=True
|
||||
).stdout
|
||||
|
||||
|
||||
async def until(condition: Callable[[], bool]) -> None:
|
||||
"""The deploy runs in a background task: wait for its visible effect."""
|
||||
async with asyncio.timeout(15):
|
||||
while not condition(): # noqa: ASYNC110 - polling a fake Telegram session, no event to wait on
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
|
||||
async def test_client_through_forced_command(fake_host: Path, remote: Path) -> None:
|
||||
c = client(b"-----BEGIN KEY-----")
|
||||
archive = archive_of(remote)
|
||||
|
||||
ok = await c.deploy("hello", "abc1234", archive)
|
||||
restart = await c.restart("hello")
|
||||
bad = await c.run("deploy", "hello;", "rm")
|
||||
|
||||
assert ok.ok and ok.summary.startswith("deployed hello abc1234")
|
||||
assert restart.ok
|
||||
assert (bad.ok, bad.code) == (False, 2)
|
||||
assert "tgit-svc-hello" in docker_state(fake_host)["containers"] # type: ignore[operator]
|
||||
|
||||
|
||||
class Recorder:
|
||||
def __init__(self) -> None:
|
||||
self.outcomes: list[Outcome] = []
|
||||
|
||||
async def __call__(self, o: Outcome) -> None:
|
||||
self.outcomes.append(o)
|
||||
|
||||
|
||||
async def test_deployer_deploys_and_removes(fake_host: Path, repo: UploadsRepo, remote: Path, tmp_path: Path) -> None:
|
||||
await repo.ensure()
|
||||
sha = await repo.head()
|
||||
rec = Recorder()
|
||||
deployer = Deployer(repo, client())
|
||||
|
||||
await deployer.deploy(sha, ["hello", "gone"], rec)
|
||||
|
||||
hello, gone = rec.outcomes
|
||||
assert hello.result.ok and not hello.removed
|
||||
assert gone.removed and gone.result.ok
|
||||
calls = [json.loads(line) for line in (fake_host / "calls.log").read_text().splitlines()]
|
||||
assert any(c[0] == "run" for c in calls)
|
||||
|
||||
|
||||
async def test_refused_deploy_raises_alert(
|
||||
fake_host: Path, repo: UploadsRepo, sessions: SessionMaker, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
await repo.ensure()
|
||||
monkeypatch.setenv("FAKE_BUILD_FAIL", "1")
|
||||
events: list[Alert] = []
|
||||
|
||||
class Notes:
|
||||
async def opened(self, alert: Alert, admins_only: bool) -> None:
|
||||
events.append(alert)
|
||||
|
||||
async def repeated(self, alert: Alert) -> None: ...
|
||||
|
||||
async def resolved(self, alert: Alert) -> None: ...
|
||||
|
||||
alerts = AlertService(sessions, Notes(), load_alerts_config(ROOT / "config"))
|
||||
rec = Recorder()
|
||||
await Deployer(repo, client(), alerts).deploy(await repo.head(), ["hello"], rec)
|
||||
|
||||
[o] = rec.outcomes
|
||||
assert (o.result.ok, o.result.code, o.result.summary) == (False, 5, "build failed")
|
||||
[alert] = events
|
||||
assert alert.kind is AlertKind.DEPLOY_REFUSED and alert.service == "tgit-svc-hello"
|
||||
assert "could not install" in alert.payload["log"]
|
||||
|
||||
|
||||
async def test_one_broken_service_does_not_stop_others(repo: UploadsRepo) -> None:
|
||||
class Flaky(RunnerClient):
|
||||
async def run(self, *args: str, stdin: bytes = b"") -> RunnerResult:
|
||||
if args[1] == "aaa":
|
||||
raise RuntimeError("ssh exploded")
|
||||
return RunnerResult(True, 0, f"deployed {args[1]}", "OK")
|
||||
|
||||
await repo.ensure()
|
||||
rec = Recorder()
|
||||
await Deployer(repo, Flaky(RunnerSettings(host="h"))).deploy(await repo.head(), ["aaa", "hello"], rec)
|
||||
|
||||
assert [(o.service, o.result.ok) for o in rec.outcomes] == [("aaa", False), ("hello", True)]
|
||||
|
||||
|
||||
# ---- in the bot -------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tg(texts: Texts, sessions: SessionMaker, access: AccessService, repo: UploadsRepo, fake_host: Path) -> TgClient:
|
||||
def dispatcher(bot: Bot) -> Dispatcher:
|
||||
alerts = AlertService(
|
||||
sessions, TelegramNotifier(bot, access, sessions, texts), load_alerts_config(ROOT / "config")
|
||||
)
|
||||
return build_dispatcher(texts, sessions, access, repo, alerts=alerts, runner=client())
|
||||
|
||||
return TgClient(dispatcher)
|
||||
|
||||
|
||||
async def test_push_deploys_touched_service(tg: TgClient, texts: Texts, fake_host: Path, remote: Path) -> None:
|
||||
await tg.send(ADMIN, texts.menu.commit)
|
||||
await tg.press(ADMIN, CommitCb(a=Act.OPEN, i=2).pack()) # services
|
||||
await tg.press(ADMIN, CommitCb(a=Act.OPEN, i=0).pack()) # services/hello
|
||||
await tg.press(ADMIN, CommitCb(a=Act.NEW_FILE).pack())
|
||||
await tg.send(ADMIN, "util.py")
|
||||
await tg.send(ADMIN, "X = 1\n")
|
||||
await tg.press(ADMIN, CommitCb(a=Act.MESSAGE).pack())
|
||||
await tg.send(ADMIN, "add util")
|
||||
await tg.press(ADMIN, CommitCb(a=Act.PUSH).pack())
|
||||
await tg.press(ADMIN, CommitCb(a=Act.PUSH_GO).pack())
|
||||
await until(lambda: any("работает" in (m.text or "") for m in tg.session.sent(SendMessage)))
|
||||
|
||||
pushed = [e for e in tg.session.sent(EditMessageText) if "Запушено" in (e.text or "")]
|
||||
assert pushed and "🚀 Разворачиваю: hello…" in pushed[-1].text
|
||||
[done] = [m for m in tg.session.sent(SendMessage) if "работает" in (m.text or "")]
|
||||
assert done.text.startswith("🚀 <b>hello</b> работает · ")
|
||||
assert "tgit-svc-hello" in docker_state(fake_host)["containers"] # type: ignore[operator]
|
||||
|
||||
|
||||
def buttons(markup: InlineKeyboardMarkup) -> list[str]:
|
||||
return [b.text for row in markup.inline_keyboard for b in row]
|
||||
|
||||
|
||||
def test_restart_button_only_for_runner_services(texts: Texts) -> None:
|
||||
service = replace(RUNNING, labels={"tgit.service": "hello"})
|
||||
now = datetime.now(UTC)
|
||||
|
||||
assert texts.status.restart in buttons(render_card(texts, service, None, now, can_restart=True)[1])
|
||||
assert texts.status.restart not in buttons(render_card(texts, service, None, now)[1]) # not an admin
|
||||
assert texts.status.restart not in buttons(render_card(texts, RUNNING, None, now, can_restart=True)[1]) # the bot
|
||||
assert runner_service(service) == "hello" and runner_service(RUNNING) is None
|
||||
|
||||
|
||||
async def test_restart_through_status(
|
||||
texts: Texts, sessions: SessionMaker, access: AccessService, repo: UploadsRepo, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from tests.runtime.fake_docker import FakeDocker
|
||||
|
||||
fake = FakeDocker()
|
||||
svc = fake.add("tgit-svc-hello", labels={"tgit.service": "hello"})
|
||||
calls: list[tuple[str, ...]] = []
|
||||
|
||||
async def run(self: RunnerClient, *args: str, stdin: bytes = b"") -> RunnerResult:
|
||||
calls.append(args)
|
||||
return RunnerResult(True, 0, "restarted hello", "OK restarted hello")
|
||||
|
||||
monkeypatch.setattr(RunnerClient, "run", run)
|
||||
async with fake.client() as docker:
|
||||
tg = TgClient(build_dispatcher(texts, sessions, access, repo, docker=docker, runner=client()))
|
||||
await tg.press(ADMIN, StatusCb(a=StatusAct.RESTART, id=svc.id[:12]).pack())
|
||||
|
||||
assert calls == [("restart", "hello")]
|
||||
assert tg.session.sent(AnswerCallbackQuery)[0].text == texts.status.restarted
|
||||
@@ -0,0 +1,250 @@
|
||||
"""tgit-runner against a fake docker: argument validation, archive checks, sandbox flags, rollback."""
|
||||
|
||||
import io
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import tarfile
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import ROOT
|
||||
|
||||
RUNNER = ROOT / "infra" / "runner" / "tgit-runner"
|
||||
FAKE = Path(__file__).with_name("fake_docker.py")
|
||||
|
||||
pytestmark = pytest.mark.skipif(shutil.which("bash") is None, reason="needs bash")
|
||||
|
||||
|
||||
def tar_of(files: dict[str, str | bytes], extra: Callable[[tarfile.TarFile], None] | None = None) -> bytes:
|
||||
buf = io.BytesIO()
|
||||
with tarfile.open(fileobj=buf, mode="w") as tar:
|
||||
for name, content in files.items():
|
||||
data = content.encode() if isinstance(content, str) else content
|
||||
info = tarfile.TarInfo(name)
|
||||
info.size = len(data)
|
||||
tar.addfile(info, io.BytesIO(data))
|
||||
if extra:
|
||||
extra(tar)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
HELLO = tar_of({"main.py": "print('hi')\n"})
|
||||
|
||||
|
||||
class Runner:
|
||||
def __init__(self, tmp: Path) -> None:
|
||||
self.tmp = tmp
|
||||
bin_dir = tmp / "bin"
|
||||
bin_dir.mkdir()
|
||||
(bin_dir / "docker").symlink_to(FAKE)
|
||||
self.env = {
|
||||
"PATH": f"{bin_dir}:/usr/bin:/bin",
|
||||
"FAKE_DOCKER_STATE": str(tmp / "state.json"),
|
||||
"FAKE_DOCKER_LOG": str(tmp / "calls.log"),
|
||||
"TGIT_RUNNER_STATE": str(tmp / "runner"),
|
||||
"TGIT_HEALTH_TIMEOUT": "4",
|
||||
"TGIT_SETTLE": "0",
|
||||
}
|
||||
|
||||
def __call__(
|
||||
self, *args: str, stdin: bytes = b"", ssh: str | None = None, **env: str
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
e = {**self.env, **env}
|
||||
if ssh is not None:
|
||||
e["SSH_ORIGINAL_COMMAND"] = ssh
|
||||
return subprocess.run(
|
||||
["bash", str(RUNNER), *args],
|
||||
input=stdin.decode("latin-1"),
|
||||
env=e,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
encoding="latin-1",
|
||||
)
|
||||
|
||||
def calls(self) -> list[list[str]]:
|
||||
path = self.tmp / "calls.log"
|
||||
return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else []
|
||||
|
||||
def state(self) -> dict[str, object]:
|
||||
return json.loads((self.tmp / "state.json").read_text())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def runner(tmp_path: Path) -> Runner:
|
||||
return Runner(tmp_path)
|
||||
|
||||
|
||||
def last(out: subprocess.CompletedProcess[str]) -> str:
|
||||
return out.stdout.strip().splitlines()[-1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("ssh", "reason"),
|
||||
[
|
||||
("", "usage"),
|
||||
("rm -rf /", "unknown command"),
|
||||
("deploy hello; rm -rf /", "wrong number of arguments"),
|
||||
("deploy Hello abc1234", "bad service name"),
|
||||
("deploy ../etc abc1234", "bad service name"),
|
||||
("deploy hello $(id)", "bad sha"),
|
||||
("deploy hello abc1234 extra", "wrong number of arguments"),
|
||||
("restart *", "bad service name"),
|
||||
("list now", "list takes no arguments"),
|
||||
],
|
||||
)
|
||||
def test_bad_requests_rejected_before_docker(runner: Runner, ssh: str, reason: str) -> None:
|
||||
out = runner(ssh=ssh)
|
||||
|
||||
assert out.returncode == 2
|
||||
assert reason in last(out)
|
||||
assert runner.calls() == []
|
||||
|
||||
|
||||
def test_deploy_runs_in_sandbox(runner: Runner) -> None:
|
||||
out = runner(ssh="deploy hello abc1234", stdin=tar_of({"main.py": "x", "tgit.yaml": "network: true\n"}))
|
||||
|
||||
assert out.returncode == 0, out.stdout + out.stderr
|
||||
assert last(out) == "OK deployed hello abc1234 network=none healthcheck=no" # network not allowed by admin
|
||||
[run] = [c for c in runner.calls() if c[0] == "run"]
|
||||
flags = " ".join(run)
|
||||
for flag in [
|
||||
"--user 10001:10001",
|
||||
"--read-only",
|
||||
"--cap-drop ALL",
|
||||
"--security-opt no-new-privileges",
|
||||
"--memory 128m --memory-swap 128m",
|
||||
"--cpus 0.25",
|
||||
"--pids-limit 64",
|
||||
"--network none",
|
||||
"--label tgit.managed=true",
|
||||
"--label tgit.sha=abc1234",
|
||||
]:
|
||||
assert flag in flags
|
||||
assert "/var/run/docker.sock" not in flags
|
||||
assert list(runner.state()["containers"]) == ["tgit-svc-hello"] # -next renamed into place
|
||||
|
||||
|
||||
def test_egress_only_when_admin_allows(runner: Runner) -> None:
|
||||
(runner.tmp / "runner").mkdir()
|
||||
(runner.tmp / "runner" / "egress-allow").write_text("weather\n")
|
||||
|
||||
out = runner(ssh="deploy weather abc1234", stdin=tar_of({"main.py": "x", "tgit.yaml": "network: true\n"}))
|
||||
|
||||
assert "network=tgit-svc-egress" in last(out)
|
||||
|
||||
|
||||
def test_healthcheck_from_tgit_yaml_without_a_shell(runner: Runner) -> None:
|
||||
yaml = "health_url: http://localhost:8080/health # probe\n"
|
||||
out = runner(ssh="deploy web abc1234", stdin=tar_of({"main.py": "x", "tgit.yaml": yaml}))
|
||||
|
||||
assert out.returncode == 0
|
||||
dockerfile = runner.state()["dockerfiles"]["tgit-svc-web:abc1234"] # type: ignore[index]
|
||||
assert 'CMD ["python", "-c", "import sys, urllib.request' in dockerfile
|
||||
assert '"http://localhost:8080/health"]' in dockerfile
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"url", ["http://evil.com/x", "http://localhost:80/$(id)", 'http://localhost:80/"]; RUN id #', "file:///etc/passwd"]
|
||||
)
|
||||
def test_health_url_must_be_local(runner: Runner, url: str) -> None:
|
||||
out = runner(ssh="deploy web abc1234", stdin=tar_of({"main.py": "x", "tgit.yaml": f"health_url: {url}\n"}))
|
||||
|
||||
assert out.returncode == 2 and "health_url" in last(out)
|
||||
assert not [c for c in runner.calls() if c[0] == "build"]
|
||||
|
||||
|
||||
def symlink_to_etc(tar: tarfile.TarFile) -> None:
|
||||
info = tarfile.TarInfo("link")
|
||||
info.type = tarfile.SYMTYPE
|
||||
info.linkname = "/etc/passwd"
|
||||
tar.addfile(info)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("archive", "reason"),
|
||||
[
|
||||
(b"not a tar", "not a tar archive"),
|
||||
(tar_of({"main.py": "x"}, symlink_to_etc), "only files and directories"),
|
||||
(tar_of({"main.py": "x", "../escape.py": "x"}), "unsafe path"),
|
||||
(tar_of({"lib.py": "x"}), "main.py is missing"),
|
||||
],
|
||||
)
|
||||
def test_bad_archives(runner: Runner, archive: bytes, reason: str) -> None:
|
||||
out = runner(ssh="deploy hello abc1234", stdin=archive)
|
||||
|
||||
assert out.returncode == 2 and reason in last(out)
|
||||
|
||||
|
||||
def test_archive_size_limit(runner: Runner) -> None:
|
||||
out = runner(ssh="deploy hello abc1234", stdin=HELLO, TGIT_MAX_TAR="100")
|
||||
|
||||
assert out.returncode == 2 and "larger than" in last(out)
|
||||
|
||||
|
||||
def test_unhealthy_new_version_keeps_the_old_one(runner: Runner) -> None:
|
||||
assert runner(ssh="deploy hello abc1234", stdin=HELLO).returncode == 0
|
||||
|
||||
out = runner(ssh="deploy hello def5678", stdin=HELLO, FAKE_HEALTH="unhealthy", FAKE_STATUS="exited")
|
||||
|
||||
assert out.returncode == 3
|
||||
assert last(out) == "FAIL not healthy within 4s, previous version kept"
|
||||
assert "Traceback: boom" in out.stdout # the new version's log tail for the user
|
||||
assert list(runner.state()["containers"]) == ["tgit-svc-hello"]
|
||||
assert runner.state()["containers"]["tgit-svc-hello"]["image"] == "tgit-svc-hello:abc1234" # type: ignore[index]
|
||||
|
||||
|
||||
def test_restarting_without_healthcheck_is_not_ready(runner: Runner) -> None:
|
||||
out = runner(ssh="deploy hello abc1234", stdin=HELLO, FAKE_RESTARTS="2")
|
||||
|
||||
assert out.returncode == 3 and last(out) == "FAIL not healthy within 4s"
|
||||
|
||||
|
||||
def test_build_failure(runner: Runner) -> None:
|
||||
out = runner(ssh="deploy hello abc1234", stdin=HELLO, FAKE_BUILD_FAIL="1")
|
||||
|
||||
assert out.returncode == 5 and "could not install requirements" in out.stdout
|
||||
|
||||
|
||||
def test_service_limit(runner: Runner) -> None:
|
||||
for i in range(3):
|
||||
assert runner(ssh=f"deploy s{i}x abc1234", stdin=HELLO, TGIT_MAX_SERVICES="3").returncode == 0
|
||||
|
||||
out = runner(ssh="deploy extra abc1234", stdin=HELLO, TGIT_MAX_SERVICES="3")
|
||||
redeploy = runner(ssh="deploy s0x def5678", stdin=HELLO, TGIT_MAX_SERVICES="3")
|
||||
|
||||
assert out.returncode == 4 and "limit 3" in last(out)
|
||||
assert redeploy.returncode == 0 # an existing service may always be updated
|
||||
|
||||
|
||||
def test_old_images_pruned(runner: Runner) -> None:
|
||||
for sha in ("aaa1111", "bbb2222", "ccc3333"):
|
||||
assert runner(ssh=f"deploy hello {sha}", stdin=HELLO).returncode == 0
|
||||
|
||||
assert sorted(runner.state()["images"]) == ["tgit-svc-hello:bbb2222", "tgit-svc-hello:ccc3333"] # type: ignore[type-var]
|
||||
|
||||
|
||||
def test_restart_remove_list(runner: Runner) -> None:
|
||||
assert last(runner(ssh="restart hello")) == "FAIL no such service"
|
||||
runner(ssh="deploy hello abc1234", stdin=HELLO)
|
||||
|
||||
assert last(runner(ssh="restart hello")) == "OK restarted hello"
|
||||
assert last(runner(ssh="list")) == "OK list"
|
||||
assert last(runner(ssh="remove hello")) == "OK removed hello"
|
||||
assert runner.state()["containers"] == {}
|
||||
assert runner.state()["images"] == []
|
||||
|
||||
|
||||
def test_direct_invocation_without_ssh(runner: Runner) -> None:
|
||||
out = runner("list")
|
||||
assert out.returncode == 0
|
||||
|
||||
|
||||
def test_glob_in_ssh_command_is_not_expanded(runner: Runner, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.chdir(tmp_path)
|
||||
(tmp_path / "abc1234").write_text("")
|
||||
out = runner(ssh="deploy hello abc12*")
|
||||
assert out.returncode == 2 and "bad sha" in last(out)
|
||||
Reference in New Issue
Block a user