import asyncio import contextlib from collections.abc import AsyncIterator from datetime import UTC, datetime, timedelta import httpx import pytest from app.db.models import ContainerSnapshot from app.db.session import SessionMaker from app.runtime import checks from app.runtime.checks import Checker, CheckName, ChecksConfig, Verdict, count_errors, load_checks from app.runtime.docker import DockerClient, Light from app.runtime.snapshots import SnapshotStore, collect_once, snapshot_loop from tests.conftest import ROOT from tests.runtime.fake_docker import FakeDocker, now_minus CFG = load_checks(ROOT / "config") def probe(request: httpx.Request) -> httpx.Response: if request.url.host == "broken": raise httpx.ConnectError("refused", request=request) return httpx.Response(200 if request.url.path == "/health" else 503) @pytest.fixture def fake() -> FakeDocker: fake = FakeDocker() fake.add("hello", service="hello", labels={"tgit.health_url": "http://hello:8000/health"}) fake.add( "unhealthy", service="unhealthy", health="unhealthy", labels={"tgit.health_url": "http://unhealthy:8000/x"} ) fake.add("crashed", state="exited", health=None, exit_code=137, oom=True) return fake @pytest.fixture async def docker(fake: FakeDocker) -> AsyncIterator[DockerClient]: async with fake.client() as client: await client.containers() # the bot always lists before it inspects yield client @pytest.fixture def store(sessions: SessionMaker) -> SnapshotStore: return SnapshotStore(sessions) @pytest.fixture def checker(docker: DockerClient, store: SnapshotStore) -> Checker: return Checker(docker, store, CFG, httpx.AsyncClient(transport=httpx.MockTransport(probe))) def verdicts(report: checks.ContainerReport) -> dict[CheckName, Verdict]: return {r.name: r.verdict for r in report.results} async def test_config_loads_and_bad_pattern_rejected() -> None: assert CFG.window == 300 with pytest.raises(ValueError): ChecksConfig(error_pattern="(unclosed") async def test_healthy_vs_unhealthy_vs_crashed(checker: Checker) -> None: reports = {r.details.name: r for r in await checker.check_all()} hello = verdicts(reports["hello"]) assert hello[CheckName.RUNNING] is Verdict.OK assert hello[CheckName.HEALTH] is Verdict.OK assert hello[CheckName.HTTP] is Verdict.OK assert hello[CheckName.RESTARTS] is Verdict.SKIP # no history yet assert reports["hello"].ok bad = verdicts(reports["unhealthy"]) assert bad[CheckName.HEALTH] is Verdict.FAIL assert bad[CheckName.HTTP] is Verdict.FAIL assert not reports["unhealthy"].ok crashed = reports["crashed"] assert verdicts(crashed)[CheckName.RUNNING] is Verdict.FAIL assert "exit 137, OOM" in crashed.results[0].detail assert verdicts(crashed)[CheckName.CPU] is Verdict.SKIP assert crashed.details.light is Light.RED async def test_restarts_compared_with_history( checker: Checker, fake: FakeDocker, docker: DockerClient, store: SnapshotStore ) -> None: hello = next(c for c in fake.containers.values() if c.name == "hello") observed = await collect_once(docker, store, now=datetime.now(UTC) - timedelta(minutes=3)) assert len(observed) == 3 and len([o for o in observed if o.stats]) == 2 # stats for running ones only hello.restart_count = 2 report = await checker.check(hello.id) result = next(r for r in report.results if r.name is CheckName.RESTARTS) assert result.verdict is Verdict.FAIL assert result.detail == "0 → 2" async def test_old_history_outside_window_is_ignored( checker: Checker, fake: FakeDocker, sessions: SessionMaker ) -> None: hello = next(c for c in fake.containers.values() if c.name == "hello") async with sessions.begin() as s: s.add( ContainerSnapshot(container="hello", at=now_minus(30), restart_count=0, cpu_pct=1, mem_pct=1, health=None) ) hello.restart_count = 5 report = await checker.check(hello.id) assert verdicts(report)[CheckName.RESTARTS] is Verdict.SKIP async def test_thresholds_and_log_errors(checker: Checker, fake: FakeDocker) -> None: hello = next(c for c in fake.containers.values() if c.name == "hello") hello.cpu_pct, hello.mem_bytes = 150.0, 250 * 1024 * 1024 hello.logs = [ (now_minus(20), "Traceback from long ago"), (now_minus(1), "all good"), (now_minus(1), '{"level": "ERROR", "msg": "boom"}'), ] report = await checker.check(hello.id) v = verdicts(report) assert v[CheckName.CPU] is Verdict.FAIL assert v[CheckName.MEM] is Verdict.FAIL log_result = next(r for r in report.results if r.name is CheckName.LOG_ERRORS) assert log_result.verdict is Verdict.FAIL assert log_result.detail.startswith("1: ") # the old Traceback is outside the window async def test_broken_or_slow_check_does_not_spoil_report( checker: Checker, fake: FakeDocker, monkeypatch: pytest.MonkeyPatch ) -> None: async def boom(ctx: checks.Context) -> checks.CheckResult: raise RuntimeError("kaputt") async def slow(ctx: checks.Context) -> checks.CheckResult: await asyncio.sleep(10) raise AssertionError monkeypatch.setattr(checks, "CHECKS", [(CheckName.HEALTH, boom), (CheckName.HTTP, slow), *checks.CHECKS[:1]]) fast = Checker(checker._docker, checker._store, CFG.model_copy(update={"check_timeout": 0.05}), checker._http) hello = next(c for c in fake.containers.values() if c.name == "hello") report = await fast.check(hello.id) by_name = {r.name: r for r in report.results} assert by_name[CheckName.HEALTH].detail == "RuntimeError: kaputt" assert by_name[CheckName.HTTP].detail == "timeout" assert by_name[CheckName.RUNNING].verdict is Verdict.OK async def test_stats_failure_skips_cpu_and_mem(checker: Checker, fake: FakeDocker) -> None: hello = next(c for c in fake.containers.values() if c.name == "hello") hello.stats_status = 500 report = await checker.check(hello.id) assert verdicts(report)[CheckName.CPU] is Verdict.SKIP assert verdicts(report)[CheckName.RUNNING] is Verdict.OK async def test_http_probe_connection_error(fake: FakeDocker, checker: Checker) -> None: hello = next(c for c in fake.containers.values() if c.name == "hello") hello.labels["tgit.health_url"] = "http://broken/health" report = await checker.check(hello.id) result = next(r for r in report.results if r.name is CheckName.HTTP) assert (result.verdict, result.detail) == (Verdict.FAIL, "ConnectError") async def test_snapshot_store_latest(docker: DockerClient, store: SnapshotStore) -> None: await collect_once(docker, store, now=now_minus(1)) await collect_once(docker, store) latest = await store.latest("hello") assert latest is not None and latest.cpu_pct == 5.0 and latest.health == "healthy" def test_count_errors_reports_first_hit() -> None: hits, first = count_errors("ok\nERROR one\nERROR two\n", r"ERROR") assert (hits, first) == (2, "ERROR one") async def test_snapshot_loop_survives_docker_outage( fake: FakeDocker, docker: DockerClient, store: SnapshotStore ) -> None: fake.down = True task = asyncio.create_task(snapshot_loop(docker, store, interval=0.01)) await asyncio.sleep(0.05) fake.down = False await asyncio.sleep(0.05) task.cancel() with contextlib.suppress(asyncio.CancelledError): await task assert await store.latest("hello") is not None