tgit/tests/runtime/test_checks.py

import asyncio
import contextlib
from collections.abc import AsyncIterator
from datetime import UTC, datetime, timedelta

import httpx
import pytest

from app.db.models import ContainerSnapshot
from app.db.session import SessionMaker
from app.runtime import checks
from app.runtime.checks import Checker, CheckName, ChecksConfig, Verdict, count_errors, load_checks
from app.runtime.docker import DockerClient, Light
from app.runtime.snapshots import SnapshotStore, collect_once, snapshot_loop
from tests.conftest import ROOT
from tests.runtime.fake_docker import FakeDocker, now_minus

CFG = load_checks(ROOT / "config")


def probe(request: httpx.Request) -> httpx.Response:
    if request.url.host == "broken":
        raise httpx.ConnectError("refused", request=request)
    return httpx.Response(200 if request.url.path == "/health" else 503)


@pytest.fixture
def fake() -> FakeDocker:
    fake = FakeDocker()
    fake.add("hello", service="hello", labels={"tgit.health_url": "http://hello:8000/health"})
    fake.add(
        "unhealthy", service="unhealthy", health="unhealthy", labels={"tgit.health_url": "http://unhealthy:8000/x"}
    )
    fake.add("crashed", state="exited", health=None, exit_code=137, oom=True)
    return fake


@pytest.fixture
async def docker(fake: FakeDocker) -> AsyncIterator[DockerClient]:
    async with fake.client() as client:
        await client.containers()  # the bot always lists before it inspects
        yield client


@pytest.fixture
def store(sessions: SessionMaker) -> SnapshotStore:
    return SnapshotStore(sessions)


@pytest.fixture
def checker(docker: DockerClient, store: SnapshotStore) -> Checker:
    return Checker(docker, store, CFG, httpx.AsyncClient(transport=httpx.MockTransport(probe)))


def verdicts(report: checks.ContainerReport) -> dict[CheckName, Verdict]:
    return {r.name: r.verdict for r in report.results}


async def test_config_loads_and_bad_pattern_rejected() -> None:
    assert CFG.window == 300
    with pytest.raises(ValueError):
        ChecksConfig(error_pattern="(unclosed")


async def test_healthy_vs_unhealthy_vs_crashed(checker: Checker) -> None:
    reports = {r.details.name: r for r in await checker.check_all()}

    hello = verdicts(reports["hello"])
    assert hello[CheckName.RUNNING] is Verdict.OK
    assert hello[CheckName.HEALTH] is Verdict.OK
    assert hello[CheckName.HTTP] is Verdict.OK
    assert hello[CheckName.RESTARTS] is Verdict.SKIP  # no history yet
    assert reports["hello"].ok

    bad = verdicts(reports["unhealthy"])
    assert bad[CheckName.HEALTH] is Verdict.FAIL
    assert bad[CheckName.HTTP] is Verdict.FAIL
    assert not reports["unhealthy"].ok

    crashed = reports["crashed"]
    assert verdicts(crashed)[CheckName.RUNNING] is Verdict.FAIL
    assert "exit 137, OOM" in crashed.results[0].detail
    assert verdicts(crashed)[CheckName.CPU] is Verdict.SKIP
    assert crashed.details.light is Light.RED


async def test_restarts_compared_with_history(
    checker: Checker, fake: FakeDocker, docker: DockerClient, store: SnapshotStore
) -> None:
    hello = next(c for c in fake.containers.values() if c.name == "hello")
    observed = await collect_once(docker, store, now=datetime.now(UTC) - timedelta(minutes=3))
    assert len(observed) == 3 and len([o for o in observed if o.stats]) == 2  # stats for running ones only
    hello.restart_count = 2

    report = await checker.check(hello.id)

    result = next(r for r in report.results if r.name is CheckName.RESTARTS)
    assert result.verdict is Verdict.FAIL
    assert result.detail == "0 → 2"


async def test_old_history_outside_window_is_ignored(
    checker: Checker, fake: FakeDocker, sessions: SessionMaker
) -> None:
    hello = next(c for c in fake.containers.values() if c.name == "hello")
    async with sessions.begin() as s:
        s.add(
            ContainerSnapshot(container="hello", at=now_minus(30), restart_count=0, cpu_pct=1, mem_pct=1, health=None)
        )
    hello.restart_count = 5

    report = await checker.check(hello.id)

    assert verdicts(report)[CheckName.RESTARTS] is Verdict.SKIP


async def test_thresholds_and_log_errors(checker: Checker, fake: FakeDocker) -> None:
    hello = next(c for c in fake.containers.values() if c.name == "hello")
    hello.cpu_pct, hello.mem_bytes = 150.0, 250 * 1024 * 1024
    hello.logs = [
        (now_minus(20), "Traceback from long ago"),
        (now_minus(1), "all good"),
        (now_minus(1), '{"level": "ERROR", "msg": "boom"}'),
    ]

    report = await checker.check(hello.id)

    v = verdicts(report)
    assert v[CheckName.CPU] is Verdict.FAIL
    assert v[CheckName.MEM] is Verdict.FAIL
    log_result = next(r for r in report.results if r.name is CheckName.LOG_ERRORS)
    assert log_result.verdict is Verdict.FAIL
    assert log_result.detail.startswith("1: ")  # the old Traceback is outside the window


async def test_broken_or_slow_check_does_not_spoil_report(
    checker: Checker, fake: FakeDocker, monkeypatch: pytest.MonkeyPatch
) -> None:
    async def boom(ctx: checks.Context) -> checks.CheckResult:
        raise RuntimeError("kaputt")

    async def slow(ctx: checks.Context) -> checks.CheckResult:
        await asyncio.sleep(10)
        raise AssertionError

    monkeypatch.setattr(checks, "CHECKS", [(CheckName.HEALTH, boom), (CheckName.HTTP, slow), *checks.CHECKS[:1]])
    fast = Checker(checker._docker, checker._store, CFG.model_copy(update={"check_timeout": 0.05}), checker._http)
    hello = next(c for c in fake.containers.values() if c.name == "hello")

    report = await fast.check(hello.id)

    by_name = {r.name: r for r in report.results}
    assert by_name[CheckName.HEALTH].detail == "RuntimeError: kaputt"
    assert by_name[CheckName.HTTP].detail == "timeout"
    assert by_name[CheckName.RUNNING].verdict is Verdict.OK


async def test_stats_failure_skips_cpu_and_mem(checker: Checker, fake: FakeDocker) -> None:
    hello = next(c for c in fake.containers.values() if c.name == "hello")
    hello.stats_status = 500

    report = await checker.check(hello.id)

    assert verdicts(report)[CheckName.CPU] is Verdict.SKIP
    assert verdicts(report)[CheckName.RUNNING] is Verdict.OK


async def test_http_probe_connection_error(fake: FakeDocker, checker: Checker) -> None:
    hello = next(c for c in fake.containers.values() if c.name == "hello")
    hello.labels["tgit.health_url"] = "http://broken/health"

    report = await checker.check(hello.id)

    result = next(r for r in report.results if r.name is CheckName.HTTP)
    assert (result.verdict, result.detail) == (Verdict.FAIL, "ConnectError")


async def test_snapshot_store_latest(docker: DockerClient, store: SnapshotStore) -> None:
    await collect_once(docker, store, now=now_minus(1))
    await collect_once(docker, store)

    latest = await store.latest("hello")

    assert latest is not None and latest.cpu_pct == 5.0 and latest.health == "healthy"


def test_count_errors_reports_first_hit() -> None:
    hits, first = count_errors("ok\nERROR one\nERROR two\n", r"ERROR")
    assert (hits, first) == (2, "ERROR one")


async def test_snapshot_loop_survives_docker_outage(
    fake: FakeDocker, docker: DockerClient, store: SnapshotStore
) -> None:
    fake.down = True
    task = asyncio.create_task(snapshot_loop(docker, store, interval=0.01))
    await asyncio.sleep(0.05)
    fake.down = False
    await asyncio.sleep(0.05)
    task.cancel()
    with contextlib.suppress(asyncio.CancelledError):
        await task

    assert await store.latest("hello") is not None