feat(status): раздел «Статус» — контейнеры, карточка, проверки (#14)
CI / ruff · mypy · pytest (pull_request) Successful in 1m11s
CI / ruff · mypy · pytest (pull_request) Successful in 1m11s
- docker-socket-proxy в Compose (только GET /containers/*), сокет только у него - app/runtime/docker.py: read-only клиент; видны только контейнеры с меткой tgit.managed=true, чужие id отклоняются до запроса - app/runtime/checks.py: running, health, рестарты за окно, CPU/RAM, ошибки в логе, HTTP-проба; у каждой проверки свой таймаут, отчёт всегда полный - app/runtime/snapshots.py: снапшоты раз в 15 с в container_snapshots - бот: список с 🟢🟡🔴⚪, карточка, «🩺 Запустить проверки», «🔄 Обновить» - деплой ставит метки tgit.sha / tgit.deployed_at на контейнер бота - имя джоба CD без неотрисованного выражения Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,203 @@
|
||||
import asyncio
|
||||
import contextlib
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app.db.models import ContainerSnapshot
|
||||
from app.db.session import SessionMaker
|
||||
from app.runtime import checks
|
||||
from app.runtime.checks import Checker, CheckName, ChecksConfig, Verdict, count_errors, load_checks
|
||||
from app.runtime.docker import DockerClient, Light
|
||||
from app.runtime.snapshots import SnapshotStore, collect_once, snapshot_loop
|
||||
from tests.conftest import ROOT
|
||||
from tests.runtime.fake_docker import FakeDocker, now_minus
|
||||
|
||||
CFG = load_checks(ROOT / "config")
|
||||
|
||||
|
||||
def probe(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.host == "broken":
|
||||
raise httpx.ConnectError("refused", request=request)
|
||||
return httpx.Response(200 if request.url.path == "/health" else 503)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake() -> FakeDocker:
|
||||
fake = FakeDocker()
|
||||
fake.add("hello", service="hello", labels={"tgit.health_url": "http://hello:8000/health"})
|
||||
fake.add(
|
||||
"unhealthy", service="unhealthy", health="unhealthy", labels={"tgit.health_url": "http://unhealthy:8000/x"}
|
||||
)
|
||||
fake.add("crashed", state="exited", health=None, exit_code=137, oom=True)
|
||||
return fake
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def docker(fake: FakeDocker) -> AsyncIterator[DockerClient]:
|
||||
async with fake.client() as client:
|
||||
await client.containers() # the bot always lists before it inspects
|
||||
yield client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(sessions: SessionMaker) -> SnapshotStore:
|
||||
return SnapshotStore(sessions)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def checker(docker: DockerClient, store: SnapshotStore) -> Checker:
|
||||
return Checker(docker, store, CFG, httpx.AsyncClient(transport=httpx.MockTransport(probe)))
|
||||
|
||||
|
||||
def verdicts(report: checks.ContainerReport) -> dict[CheckName, Verdict]:
|
||||
return {r.name: r.verdict for r in report.results}
|
||||
|
||||
|
||||
async def test_config_loads_and_bad_pattern_rejected() -> None:
|
||||
assert CFG.window == 300
|
||||
with pytest.raises(ValueError):
|
||||
ChecksConfig(error_pattern="(unclosed")
|
||||
|
||||
|
||||
async def test_healthy_vs_unhealthy_vs_crashed(checker: Checker) -> None:
|
||||
reports = {r.details.name: r for r in await checker.check_all()}
|
||||
|
||||
hello = verdicts(reports["hello"])
|
||||
assert hello[CheckName.RUNNING] is Verdict.OK
|
||||
assert hello[CheckName.HEALTH] is Verdict.OK
|
||||
assert hello[CheckName.HTTP] is Verdict.OK
|
||||
assert hello[CheckName.RESTARTS] is Verdict.SKIP # no history yet
|
||||
assert reports["hello"].ok
|
||||
|
||||
bad = verdicts(reports["unhealthy"])
|
||||
assert bad[CheckName.HEALTH] is Verdict.FAIL
|
||||
assert bad[CheckName.HTTP] is Verdict.FAIL
|
||||
assert not reports["unhealthy"].ok
|
||||
|
||||
crashed = reports["crashed"]
|
||||
assert verdicts(crashed)[CheckName.RUNNING] is Verdict.FAIL
|
||||
assert "exit 137, OOM" in crashed.results[0].detail
|
||||
assert verdicts(crashed)[CheckName.CPU] is Verdict.SKIP
|
||||
assert crashed.details.light is Light.RED
|
||||
|
||||
|
||||
async def test_restarts_compared_with_history(
|
||||
checker: Checker, fake: FakeDocker, docker: DockerClient, store: SnapshotStore
|
||||
) -> None:
|
||||
hello = next(c for c in fake.containers.values() if c.name == "hello")
|
||||
assert await collect_once(docker, store, now=datetime.now(UTC) - timedelta(minutes=3)) == 2 # running ones only
|
||||
hello.restart_count = 2
|
||||
|
||||
report = await checker.check(hello.id)
|
||||
|
||||
result = next(r for r in report.results if r.name is CheckName.RESTARTS)
|
||||
assert result.verdict is Verdict.FAIL
|
||||
assert result.detail == "0 → 2"
|
||||
|
||||
|
||||
async def test_old_history_outside_window_is_ignored(
|
||||
checker: Checker, fake: FakeDocker, sessions: SessionMaker
|
||||
) -> None:
|
||||
hello = next(c for c in fake.containers.values() if c.name == "hello")
|
||||
async with sessions.begin() as s:
|
||||
s.add(
|
||||
ContainerSnapshot(container="hello", at=now_minus(30), restart_count=0, cpu_pct=1, mem_pct=1, health=None)
|
||||
)
|
||||
hello.restart_count = 5
|
||||
|
||||
report = await checker.check(hello.id)
|
||||
|
||||
assert verdicts(report)[CheckName.RESTARTS] is Verdict.SKIP
|
||||
|
||||
|
||||
async def test_thresholds_and_log_errors(checker: Checker, fake: FakeDocker) -> None:
|
||||
hello = next(c for c in fake.containers.values() if c.name == "hello")
|
||||
hello.cpu_pct, hello.mem_bytes = 150.0, 250 * 1024 * 1024
|
||||
hello.logs = [
|
||||
(now_minus(20), "Traceback from long ago"),
|
||||
(now_minus(1), "all good"),
|
||||
(now_minus(1), '{"level": "ERROR", "msg": "boom"}'),
|
||||
]
|
||||
|
||||
report = await checker.check(hello.id)
|
||||
|
||||
v = verdicts(report)
|
||||
assert v[CheckName.CPU] is Verdict.FAIL
|
||||
assert v[CheckName.MEM] is Verdict.FAIL
|
||||
log_result = next(r for r in report.results if r.name is CheckName.LOG_ERRORS)
|
||||
assert log_result.verdict is Verdict.FAIL
|
||||
assert log_result.detail.startswith("1: ") # the old Traceback is outside the window
|
||||
|
||||
|
||||
async def test_broken_or_slow_check_does_not_spoil_report(
|
||||
checker: Checker, fake: FakeDocker, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
async def boom(ctx: checks.Context) -> checks.CheckResult:
|
||||
raise RuntimeError("kaputt")
|
||||
|
||||
async def slow(ctx: checks.Context) -> checks.CheckResult:
|
||||
await asyncio.sleep(10)
|
||||
raise AssertionError
|
||||
|
||||
monkeypatch.setattr(checks, "CHECKS", [(CheckName.HEALTH, boom), (CheckName.HTTP, slow), *checks.CHECKS[:1]])
|
||||
fast = Checker(checker._docker, checker._store, CFG.model_copy(update={"check_timeout": 0.05}), checker._http)
|
||||
hello = next(c for c in fake.containers.values() if c.name == "hello")
|
||||
|
||||
report = await fast.check(hello.id)
|
||||
|
||||
by_name = {r.name: r for r in report.results}
|
||||
assert by_name[CheckName.HEALTH].detail == "RuntimeError: kaputt"
|
||||
assert by_name[CheckName.HTTP].detail == "timeout"
|
||||
assert by_name[CheckName.RUNNING].verdict is Verdict.OK
|
||||
|
||||
|
||||
async def test_stats_failure_skips_cpu_and_mem(checker: Checker, fake: FakeDocker) -> None:
|
||||
hello = next(c for c in fake.containers.values() if c.name == "hello")
|
||||
hello.stats_status = 500
|
||||
|
||||
report = await checker.check(hello.id)
|
||||
|
||||
assert verdicts(report)[CheckName.CPU] is Verdict.SKIP
|
||||
assert verdicts(report)[CheckName.RUNNING] is Verdict.OK
|
||||
|
||||
|
||||
async def test_http_probe_connection_error(fake: FakeDocker, checker: Checker) -> None:
|
||||
hello = next(c for c in fake.containers.values() if c.name == "hello")
|
||||
hello.labels["tgit.health_url"] = "http://broken/health"
|
||||
|
||||
report = await checker.check(hello.id)
|
||||
|
||||
result = next(r for r in report.results if r.name is CheckName.HTTP)
|
||||
assert (result.verdict, result.detail) == (Verdict.FAIL, "ConnectError")
|
||||
|
||||
|
||||
async def test_snapshot_store_latest(docker: DockerClient, store: SnapshotStore) -> None:
|
||||
await collect_once(docker, store, now=now_minus(1))
|
||||
await collect_once(docker, store)
|
||||
|
||||
latest = await store.latest("hello")
|
||||
|
||||
assert latest is not None and latest.cpu_pct == 5.0 and latest.health == "healthy"
|
||||
|
||||
|
||||
def test_count_errors_reports_first_hit() -> None:
|
||||
hits, first = count_errors("ok\nERROR one\nERROR two\n", r"ERROR")
|
||||
assert (hits, first) == (2, "ERROR one")
|
||||
|
||||
|
||||
async def test_snapshot_loop_survives_docker_outage(
|
||||
fake: FakeDocker, docker: DockerClient, store: SnapshotStore
|
||||
) -> None:
|
||||
fake.down = True
|
||||
task = asyncio.create_task(snapshot_loop(docker, store, interval=0.01))
|
||||
await asyncio.sleep(0.05)
|
||||
fake.down = False
|
||||
await asyncio.sleep(0.05)
|
||||
task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
assert await store.latest("hello") is not None
|
||||
Reference in New Issue
Block a user