feat(runner): tgit-runner — сервисы в песочнице на хосте, деплой после push (#13)

- infra/runner/tgit-runner: deploy/restart/remove/list по SSH forced command;
  аргументы не исполняются и не глобятся; архив — только файлы и папки,
  без «..», ≤ 20 МБ, есть main.py; health_url — только localhost
- песочница: uid 10001, read-only, tmpfs noexec, cap-drop ALL,
  no-new-privileges, 128 МБ без swap, 0.25 CPU, 64 PID, --network none
  (сеть — только по egress-allow админа), лимит 10 сервисов, 2 образа
- новая версия стартует рядом (-next) и заменяет старую, только став
  healthy; иначе удаляется, старая работает, exit 3
- бот: после push git archive изменённых services/<name>/ → runner;
  автору — «🚀 работает» или «⛔ не развёрнут» с логом, команде — алерт ⛔;
  удалённая папка → remove; «♻️ Перезапустить» в «Статусе» для админа (S-5)
- examples/escape_attempts: все попытки выбраться — BLOCKED на живом Docker
- install.sh, authorized_keys.example; CD бота обновляет runner на хосте

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner/fake_docker.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+"""A fake `docker` CLI for tgit-runner tests: records every call, keeps containers in a JSON file.
+
+Behaviour knobs (env): FAKE_HEALTH (healthy|unhealthy|none), FAKE_STATUS (running|exited), FAKE_RESTARTS,
+FAKE_BUILD_FAIL=1. State: $FAKE_DOCKER_STATE (json), calls: $FAKE_DOCKER_LOG (one json array per line).
+"""
+
+import json
+import os
+import sys
+
+state_path = os.environ["FAKE_DOCKER_STATE"]
+args = sys.argv[1:]
+with open(os.environ["FAKE_DOCKER_LOG"], "a") as log:
+    log.write(json.dumps(args) + "\n")
+try:
+    with open(state_path) as f:
+        state = json.load(f)
+except FileNotFoundError:
+    state = {"containers": {}, "images": []}
+
+
+def save() -> None:
+    with open(state_path, "w") as f:
+        json.dump(state, f)
+
+
+def done(code: int = 0, out: str = "") -> None:
+    if out:
+        print(out)
+    save()
+    sys.exit(code)
+
+
+cmd = args[0] if args else ""
+if cmd == "container" and args[1] == "inspect":
+    done(0 if args[2] in state["containers"] else 1)
+if cmd == "inspect":
+    fmt, name = args[2], args[3]
+    if name not in state["containers"]:
+        done(1)
+    if "Health" in fmt:
+        done(0, os.environ.get("FAKE_HEALTH", "healthy"))
+    if "RestartCount" in fmt:
+        done(0, os.environ.get("FAKE_RESTARTS", "0"))
+    done(0, os.environ.get("FAKE_STATUS", "running"))
+if cmd == "ps":
+    done(0, "\n".join(n for n in state["containers"]))
+if cmd == "build":
+    if os.environ.get("FAKE_BUILD_FAIL"):
+        done(1, "ERROR: could not install requirements")
+    tag = args[args.index("-t") + 1]
+    ctx = args[-1]
+    with open(os.path.join(ctx, "Dockerfile")) as f:
+        state.setdefault("dockerfiles", {})[tag] = f.read()
+    state["images"].append(tag)
+    done(0, "sha256:feed")
+if cmd == "run":
+    name = args[args.index("--name") + 1]
+    state["containers"][name] = {"image": args[-1], "args": args}
+    done(0, "cid")
+if cmd == "rm":
+    for name in args[2:]:
+        state["containers"].pop(name, None)
+    done(0)
+if cmd == "rename":
+    state["containers"][args[2]] = state["containers"].pop(args[1])
+    done(0)
+if cmd == "restart":
+    done(0)
+if cmd == "logs":
+    done(0, "Traceback: boom")
+if cmd == "image" and args[1] == "ls":
+    repo = args[2]
+    tags = [t.split(":", 1)[1] for t in reversed(state["images"]) if t.split(":", 1)[0] == repo]
+    full = "{{.Repository}}" in args[-1]
+    done(0, "\n".join(f"{repo}:{t}" if full else t for t in tags))
+if cmd == "image" and args[1] == "rm":
+    state["images"] = [t for t in state["images"] if t != args[2]]
+    done(0)
+if cmd == "network":
+    done(0)
+done(2, f"fake docker: unsupported {args}")

tests/runner/fake_ssh.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+"""A fake `ssh`: like sshd with a forced command — the remote command becomes SSH_ORIGINAL_COMMAND of tgit-runner."""
+
+import os
+import sys
+
+args = sys.argv[1:]
+with_value = {"-F", "-p", "-o", "-i"}
+i = 0
+while i < len(args) and args[i].startswith("-"):
+    i += 2 if args[i] in with_value else 1
+remote = " ".join(args[i + 1 :])
+os.environ["SSH_ORIGINAL_COMMAND"] = remote
+os.execvp("bash", ["bash", os.environ["FAKE_RUNNER"]])

tests/runner/test_deploy.py

@@ -0,0 +1,234 @@
+"""Bot side of #13: the SSH client, the deployer and the push → deploy flow, down to the real tgit-runner script."""
+
+import asyncio
+import base64
+import json
+import subprocess
+from collections.abc import Callable
+from dataclasses import replace
+from datetime import UTC, datetime
+from pathlib import Path
+
+import pytest
+from aiogram import Bot, Dispatcher
+from aiogram.methods import AnswerCallbackQuery, EditMessageText, SendMessage
+from aiogram.types import InlineKeyboardMarkup
+
+from app.access.service import AccessService
+from app.alerts.broadcast import TelegramNotifier
+from app.alerts.detectors import load_alerts_config
+from app.alerts.service import AlertService
+from app.bot.routers.commit import Act, CommitCb
+from app.bot.routers.status import StatusAct, StatusCb, render_card, runner_service
+from app.config import RunnerSettings
+from app.db.models import Alert, AlertKind
+from app.db.session import SessionMaker
+from app.main import build_dispatcher
+from app.repo.git import UploadsRepo
+from app.runner.client import RunnerClient, RunnerResult
+from app.runner.deployer import Deployer, Outcome, changed_services
+from app.texts import Texts
+from tests.alerts.test_detectors import RUNNING
+from tests.conftest import ADMIN, ROOT
+from tests.fakes import TgClient
+from tests.runner.test_tgit_runner import FAKE, RUNNER
+
+HERE = Path(__file__).parent
+
+
+@pytest.fixture
+def fake_host(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
+    """ssh → tgit-runner → fake docker, all on PATH."""
+    bin_dir = tmp_path / "bin"
+    bin_dir.mkdir()
+    (bin_dir / "ssh").symlink_to(HERE / "fake_ssh.py")
+    (bin_dir / "docker").symlink_to(FAKE)
+    monkeypatch.setenv("PATH", f"{bin_dir}:/usr/bin:/bin")
+    monkeypatch.setenv("FAKE_RUNNER", str(RUNNER))
+    monkeypatch.setenv("FAKE_DOCKER_STATE", str(tmp_path / "state.json"))
+    monkeypatch.setenv("FAKE_DOCKER_LOG", str(tmp_path / "calls.log"))
+    monkeypatch.setenv("TGIT_RUNNER_STATE", str(tmp_path / "runner"))
+    monkeypatch.setenv("TGIT_HEALTH_TIMEOUT", "2")
+    monkeypatch.setenv("TGIT_SETTLE", "0")
+    return tmp_path
+
+
+def docker_state(host: Path) -> dict[str, object]:
+    return json.loads((host / "state.json").read_text())
+
+
+def client(key: bytes | None = None) -> RunnerClient:
+    b64 = base64.b64encode(key).decode() if key else None
+    return RunnerClient(RunnerSettings(host="host.example", ssh_key_b64=b64, timeout=30))  # type: ignore[arg-type]
+
+
+def test_changed_services() -> None:
+    paths = ["services/hello/main.py", "services/hello/x/y.py", "services/Bad/main.py", "README.md", "services/a.py"]
+    assert changed_services(paths) == ["hello"]
+
+
+async def test_client_validates_before_ssh(fake_host: Path) -> None:
+    result = await client().deploy("../etc", "abc1234", b"")
+
+    assert (result.ok, result.code) == (False, 2)
+    assert not (fake_host / "calls.log").exists()
+
+
+def archive_of(remote: Path) -> bytes:
+    return subprocess.run(
+        ["git", "archive", "--format=tar", "HEAD:services/hello"], cwd=remote, capture_output=True, check=True
+    ).stdout
+
+
+async def until(condition: Callable[[], bool]) -> None:
+    """The deploy runs in a background task: wait for its visible effect."""
+    async with asyncio.timeout(15):
+        while not condition():  # noqa: ASYNC110 - polling a fake Telegram session, no event to wait on
+            await asyncio.sleep(0.05)
+
+
+async def test_client_through_forced_command(fake_host: Path, remote: Path) -> None:
+    c = client(b"-----BEGIN KEY-----")
+    archive = archive_of(remote)
+
+    ok = await c.deploy("hello", "abc1234", archive)
+    restart = await c.restart("hello")
+    bad = await c.run("deploy", "hello;", "rm")
+
+    assert ok.ok and ok.summary.startswith("deployed hello abc1234")
+    assert restart.ok
+    assert (bad.ok, bad.code) == (False, 2)
+    assert "tgit-svc-hello" in docker_state(fake_host)["containers"]  # type: ignore[operator]
+
+
+class Recorder:
+    def __init__(self) -> None:
+        self.outcomes: list[Outcome] = []
+
+    async def __call__(self, o: Outcome) -> None:
+        self.outcomes.append(o)
+
+
+async def test_deployer_deploys_and_removes(fake_host: Path, repo: UploadsRepo, remote: Path, tmp_path: Path) -> None:
+    await repo.ensure()
+    sha = await repo.head()
+    rec = Recorder()
+    deployer = Deployer(repo, client())
+
+    await deployer.deploy(sha, ["hello", "gone"], rec)
+
+    hello, gone = rec.outcomes
+    assert hello.result.ok and not hello.removed
+    assert gone.removed and gone.result.ok
+    calls = [json.loads(line) for line in (fake_host / "calls.log").read_text().splitlines()]
+    assert any(c[0] == "run" for c in calls)
+
+
+async def test_refused_deploy_raises_alert(
+    fake_host: Path, repo: UploadsRepo, sessions: SessionMaker, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    await repo.ensure()
+    monkeypatch.setenv("FAKE_BUILD_FAIL", "1")
+    events: list[Alert] = []
+
+    class Notes:
+        async def opened(self, alert: Alert, admins_only: bool) -> None:
+            events.append(alert)
+
+        async def repeated(self, alert: Alert) -> None: ...
+
+        async def resolved(self, alert: Alert) -> None: ...
+
+    alerts = AlertService(sessions, Notes(), load_alerts_config(ROOT / "config"))
+    rec = Recorder()
+    await Deployer(repo, client(), alerts).deploy(await repo.head(), ["hello"], rec)
+
+    [o] = rec.outcomes
+    assert (o.result.ok, o.result.code, o.result.summary) == (False, 5, "build failed")
+    [alert] = events
+    assert alert.kind is AlertKind.DEPLOY_REFUSED and alert.service == "tgit-svc-hello"
+    assert "could not install" in alert.payload["log"]
+
+
+async def test_one_broken_service_does_not_stop_others(repo: UploadsRepo) -> None:
+    class Flaky(RunnerClient):
+        async def run(self, *args: str, stdin: bytes = b"") -> RunnerResult:
+            if args[1] == "aaa":
+                raise RuntimeError("ssh exploded")
+            return RunnerResult(True, 0, f"deployed {args[1]}", "OK")
+
+    await repo.ensure()
+    rec = Recorder()
+    await Deployer(repo, Flaky(RunnerSettings(host="h"))).deploy(await repo.head(), ["aaa", "hello"], rec)
+
+    assert [(o.service, o.result.ok) for o in rec.outcomes] == [("aaa", False), ("hello", True)]
+
+
+# ---- in the bot -------------------------------------------------------------------------------------------------
+
+
+@pytest.fixture
+def tg(texts: Texts, sessions: SessionMaker, access: AccessService, repo: UploadsRepo, fake_host: Path) -> TgClient:
+    def dispatcher(bot: Bot) -> Dispatcher:
+        alerts = AlertService(
+            sessions, TelegramNotifier(bot, access, sessions, texts), load_alerts_config(ROOT / "config")
+        )
+        return build_dispatcher(texts, sessions, access, repo, alerts=alerts, runner=client())
+
+    return TgClient(dispatcher)
+
+
+async def test_push_deploys_touched_service(tg: TgClient, texts: Texts, fake_host: Path, remote: Path) -> None:
+    await tg.send(ADMIN, texts.menu.commit)
+    await tg.press(ADMIN, CommitCb(a=Act.OPEN, i=2).pack())  # services
+    await tg.press(ADMIN, CommitCb(a=Act.OPEN, i=0).pack())  # services/hello
+    await tg.press(ADMIN, CommitCb(a=Act.NEW_FILE).pack())
+    await tg.send(ADMIN, "util.py")
+    await tg.send(ADMIN, "X = 1\n")
+    await tg.press(ADMIN, CommitCb(a=Act.MESSAGE).pack())
+    await tg.send(ADMIN, "add util")
+    await tg.press(ADMIN, CommitCb(a=Act.PUSH).pack())
+    await tg.press(ADMIN, CommitCb(a=Act.PUSH_GO).pack())
+    await until(lambda: any("работает" in (m.text or "") for m in tg.session.sent(SendMessage)))
+
+    pushed = [e for e in tg.session.sent(EditMessageText) if "Запушено" in (e.text or "")]
+    assert pushed and "🚀 Разворачиваю: hello…" in pushed[-1].text
+    [done] = [m for m in tg.session.sent(SendMessage) if "работает" in (m.text or "")]
+    assert done.text.startswith("🚀 <b>hello</b> работает · ")
+    assert "tgit-svc-hello" in docker_state(fake_host)["containers"]  # type: ignore[operator]
+
+
+def buttons(markup: InlineKeyboardMarkup) -> list[str]:
+    return [b.text for row in markup.inline_keyboard for b in row]
+
+
+def test_restart_button_only_for_runner_services(texts: Texts) -> None:
+    service = replace(RUNNING, labels={"tgit.service": "hello"})
+    now = datetime.now(UTC)
+
+    assert texts.status.restart in buttons(render_card(texts, service, None, now, can_restart=True)[1])
+    assert texts.status.restart not in buttons(render_card(texts, service, None, now)[1])  # not an admin
+    assert texts.status.restart not in buttons(render_card(texts, RUNNING, None, now, can_restart=True)[1])  # the bot
+    assert runner_service(service) == "hello" and runner_service(RUNNING) is None
+
+
+async def test_restart_through_status(
+    texts: Texts, sessions: SessionMaker, access: AccessService, repo: UploadsRepo, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    from tests.runtime.fake_docker import FakeDocker
+
+    fake = FakeDocker()
+    svc = fake.add("tgit-svc-hello", labels={"tgit.service": "hello"})
+    calls: list[tuple[str, ...]] = []
+
+    async def run(self: RunnerClient, *args: str, stdin: bytes = b"") -> RunnerResult:
+        calls.append(args)
+        return RunnerResult(True, 0, "restarted hello", "OK restarted hello")
+
+    monkeypatch.setattr(RunnerClient, "run", run)
+    async with fake.client() as docker:
+        tg = TgClient(build_dispatcher(texts, sessions, access, repo, docker=docker, runner=client()))
+        await tg.press(ADMIN, StatusCb(a=StatusAct.RESTART, id=svc.id[:12]).pack())
+
+    assert calls == [("restart", "hello")]
+    assert tg.session.sent(AnswerCallbackQuery)[0].text == texts.status.restarted

tests/runner/test_tgit_runner.py

@@ -0,0 +1,250 @@
+"""tgit-runner against a fake docker: argument validation, archive checks, sandbox flags, rollback."""
+
+import io
+import json
+import shutil
+import subprocess
+import tarfile
+from collections.abc import Callable
+from pathlib import Path
+
+import pytest
+
+from tests.conftest import ROOT
+
+RUNNER = ROOT / "infra" / "runner" / "tgit-runner"
+FAKE = Path(__file__).with_name("fake_docker.py")
+
+pytestmark = pytest.mark.skipif(shutil.which("bash") is None, reason="needs bash")
+
+
+def tar_of(files: dict[str, str | bytes], extra: Callable[[tarfile.TarFile], None] | None = None) -> bytes:
+    buf = io.BytesIO()
+    with tarfile.open(fileobj=buf, mode="w") as tar:
+        for name, content in files.items():
+            data = content.encode() if isinstance(content, str) else content
+            info = tarfile.TarInfo(name)
+            info.size = len(data)
+            tar.addfile(info, io.BytesIO(data))
+        if extra:
+            extra(tar)
+    return buf.getvalue()
+
+
+HELLO = tar_of({"main.py": "print('hi')\n"})
+
+
+class Runner:
+    def __init__(self, tmp: Path) -> None:
+        self.tmp = tmp
+        bin_dir = tmp / "bin"
+        bin_dir.mkdir()
+        (bin_dir / "docker").symlink_to(FAKE)
+        self.env = {
+            "PATH": f"{bin_dir}:/usr/bin:/bin",
+            "FAKE_DOCKER_STATE": str(tmp / "state.json"),
+            "FAKE_DOCKER_LOG": str(tmp / "calls.log"),
+            "TGIT_RUNNER_STATE": str(tmp / "runner"),
+            "TGIT_HEALTH_TIMEOUT": "4",
+            "TGIT_SETTLE": "0",
+        }
+
+    def __call__(
+        self, *args: str, stdin: bytes = b"", ssh: str | None = None, **env: str
+    ) -> subprocess.CompletedProcess[str]:
+        e = {**self.env, **env}
+        if ssh is not None:
+            e["SSH_ORIGINAL_COMMAND"] = ssh
+        return subprocess.run(
+            ["bash", str(RUNNER), *args],
+            input=stdin.decode("latin-1"),
+            env=e,
+            capture_output=True,
+            text=True,
+            timeout=30,
+            encoding="latin-1",
+        )
+
+    def calls(self) -> list[list[str]]:
+        path = self.tmp / "calls.log"
+        return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else []
+
+    def state(self) -> dict[str, object]:
+        return json.loads((self.tmp / "state.json").read_text())
+
+
+@pytest.fixture
+def runner(tmp_path: Path) -> Runner:
+    return Runner(tmp_path)
+
+
+def last(out: subprocess.CompletedProcess[str]) -> str:
+    return out.stdout.strip().splitlines()[-1]
+
+
+@pytest.mark.parametrize(
+    ("ssh", "reason"),
+    [
+        ("", "usage"),
+        ("rm -rf /", "unknown command"),
+        ("deploy hello; rm -rf /", "wrong number of arguments"),
+        ("deploy Hello abc1234", "bad service name"),
+        ("deploy ../etc abc1234", "bad service name"),
+        ("deploy hello $(id)", "bad sha"),
+        ("deploy hello abc1234 extra", "wrong number of arguments"),
+        ("restart *", "bad service name"),
+        ("list now", "list takes no arguments"),
+    ],
+)
+def test_bad_requests_rejected_before_docker(runner: Runner, ssh: str, reason: str) -> None:
+    out = runner(ssh=ssh)
+
+    assert out.returncode == 2
+    assert reason in last(out)
+    assert runner.calls() == []
+
+
+def test_deploy_runs_in_sandbox(runner: Runner) -> None:
+    out = runner(ssh="deploy hello abc1234", stdin=tar_of({"main.py": "x", "tgit.yaml": "network: true\n"}))
+
+    assert out.returncode == 0, out.stdout + out.stderr
+    assert last(out) == "OK deployed hello abc1234 network=none healthcheck=no"  # network not allowed by admin
+    [run] = [c for c in runner.calls() if c[0] == "run"]
+    flags = " ".join(run)
+    for flag in [
+        "--user 10001:10001",
+        "--read-only",
+        "--cap-drop ALL",
+        "--security-opt no-new-privileges",
+        "--memory 128m --memory-swap 128m",
+        "--cpus 0.25",
+        "--pids-limit 64",
+        "--network none",
+        "--label tgit.managed=true",
+        "--label tgit.sha=abc1234",
+    ]:
+        assert flag in flags
+    assert "/var/run/docker.sock" not in flags
+    assert list(runner.state()["containers"]) == ["tgit-svc-hello"]  # -next renamed into place
+
+
+def test_egress_only_when_admin_allows(runner: Runner) -> None:
+    (runner.tmp / "runner").mkdir()
+    (runner.tmp / "runner" / "egress-allow").write_text("weather\n")
+
+    out = runner(ssh="deploy weather abc1234", stdin=tar_of({"main.py": "x", "tgit.yaml": "network: true\n"}))
+
+    assert "network=tgit-svc-egress" in last(out)
+
+
+def test_healthcheck_from_tgit_yaml_without_a_shell(runner: Runner) -> None:
+    yaml = "health_url: http://localhost:8080/health   # probe\n"
+    out = runner(ssh="deploy web abc1234", stdin=tar_of({"main.py": "x", "tgit.yaml": yaml}))
+
+    assert out.returncode == 0
+    dockerfile = runner.state()["dockerfiles"]["tgit-svc-web:abc1234"]  # type: ignore[index]
+    assert 'CMD ["python", "-c", "import sys, urllib.request' in dockerfile
+    assert '"http://localhost:8080/health"]' in dockerfile
+
+
+@pytest.mark.parametrize(
+    "url", ["http://evil.com/x", "http://localhost:80/$(id)", 'http://localhost:80/"]; RUN id #', "file:///etc/passwd"]
+)
+def test_health_url_must_be_local(runner: Runner, url: str) -> None:
+    out = runner(ssh="deploy web abc1234", stdin=tar_of({"main.py": "x", "tgit.yaml": f"health_url: {url}\n"}))
+
+    assert out.returncode == 2 and "health_url" in last(out)
+    assert not [c for c in runner.calls() if c[0] == "build"]
+
+
+def symlink_to_etc(tar: tarfile.TarFile) -> None:
+    info = tarfile.TarInfo("link")
+    info.type = tarfile.SYMTYPE
+    info.linkname = "/etc/passwd"
+    tar.addfile(info)
+
+
+@pytest.mark.parametrize(
+    ("archive", "reason"),
+    [
+        (b"not a tar", "not a tar archive"),
+        (tar_of({"main.py": "x"}, symlink_to_etc), "only files and directories"),
+        (tar_of({"main.py": "x", "../escape.py": "x"}), "unsafe path"),
+        (tar_of({"lib.py": "x"}), "main.py is missing"),
+    ],
+)
+def test_bad_archives(runner: Runner, archive: bytes, reason: str) -> None:
+    out = runner(ssh="deploy hello abc1234", stdin=archive)
+
+    assert out.returncode == 2 and reason in last(out)
+
+
+def test_archive_size_limit(runner: Runner) -> None:
+    out = runner(ssh="deploy hello abc1234", stdin=HELLO, TGIT_MAX_TAR="100")
+
+    assert out.returncode == 2 and "larger than" in last(out)
+
+
+def test_unhealthy_new_version_keeps_the_old_one(runner: Runner) -> None:
+    assert runner(ssh="deploy hello abc1234", stdin=HELLO).returncode == 0
+
+    out = runner(ssh="deploy hello def5678", stdin=HELLO, FAKE_HEALTH="unhealthy", FAKE_STATUS="exited")
+
+    assert out.returncode == 3
+    assert last(out) == "FAIL not healthy within 4s, previous version kept"
+    assert "Traceback: boom" in out.stdout  # the new version's log tail for the user
+    assert list(runner.state()["containers"]) == ["tgit-svc-hello"]
+    assert runner.state()["containers"]["tgit-svc-hello"]["image"] == "tgit-svc-hello:abc1234"  # type: ignore[index]
+
+
+def test_restarting_without_healthcheck_is_not_ready(runner: Runner) -> None:
+    out = runner(ssh="deploy hello abc1234", stdin=HELLO, FAKE_RESTARTS="2")
+
+    assert out.returncode == 3 and last(out) == "FAIL not healthy within 4s"
+
+
+def test_build_failure(runner: Runner) -> None:
+    out = runner(ssh="deploy hello abc1234", stdin=HELLO, FAKE_BUILD_FAIL="1")
+
+    assert out.returncode == 5 and "could not install requirements" in out.stdout
+
+
+def test_service_limit(runner: Runner) -> None:
+    for i in range(3):
+        assert runner(ssh=f"deploy s{i}x abc1234", stdin=HELLO, TGIT_MAX_SERVICES="3").returncode == 0
+
+    out = runner(ssh="deploy extra abc1234", stdin=HELLO, TGIT_MAX_SERVICES="3")
+    redeploy = runner(ssh="deploy s0x def5678", stdin=HELLO, TGIT_MAX_SERVICES="3")
+
+    assert out.returncode == 4 and "limit 3" in last(out)
+    assert redeploy.returncode == 0  # an existing service may always be updated
+
+
+def test_old_images_pruned(runner: Runner) -> None:
+    for sha in ("aaa1111", "bbb2222", "ccc3333"):
+        assert runner(ssh=f"deploy hello {sha}", stdin=HELLO).returncode == 0
+
+    assert sorted(runner.state()["images"]) == ["tgit-svc-hello:bbb2222", "tgit-svc-hello:ccc3333"]  # type: ignore[type-var]
+
+
+def test_restart_remove_list(runner: Runner) -> None:
+    assert last(runner(ssh="restart hello")) == "FAIL no such service"
+    runner(ssh="deploy hello abc1234", stdin=HELLO)
+
+    assert last(runner(ssh="restart hello")) == "OK restarted hello"
+    assert last(runner(ssh="list")) == "OK list"
+    assert last(runner(ssh="remove hello")) == "OK removed hello"
+    assert runner.state()["containers"] == {}
+    assert runner.state()["images"] == []
+
+
+def test_direct_invocation_without_ssh(runner: Runner) -> None:
+    out = runner("list")
+    assert out.returncode == 0
+
+
+def test_glob_in_ssh_command_is_not_expanded(runner: Runner, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.chdir(tmp_path)
+    (tmp_path / "abc1234").write_text("")
+    out = runner(ssh="deploy hello abc12*")
+    assert out.returncode == 2 and "bad sha" in last(out)