feat(alerts): алерты всей команде — детекторы, дедупликация, список (#15)

- детекторы по состоянию после каждого раунда снапшотов: краш, OOM, цикл рестартов, unhealthy, перегрузка ≥ 1 мин, мало места, Docker недоступен; пороги в config/alerts.yaml - AlertService: ключ kind+service, повтор в 10 мин — счётчик в исходном сообщении, «✅ восстановлено» ответом; разовые события (🛡) со счётчиком - рассылка всем активным (🛡 — только админам), темп ~25/с, RetryAfter, заблокировавший бота пропускается; alert_deliveries для правок и ответов - «🚨 Алерты»: активные + 7 дней по 10, карточка; кнопки «📜 Логи», «🩺 Статус» под алертом открывают новое сообщение, а не затирают алерт - 🛡 атаки через агента идут через общий сервис алертов - убраны заглушки «Раздел в разработке» — у всех разделов свои роутеры Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-17 15:36:09 +03:00
parent 65acf3abf1
commit 6c6f766abc
24 changed files with 1355 additions and 107 deletions
@@ -0,0 +1,96 @@
+from dataclasses import replace
+from datetime import UTC, datetime, timedelta
+
+from app.alerts.detectors import (
+    AlertsConfig,
+    container_conditions,
+    disk_condition,
+    docker_condition,
+    load_alerts_config,
+)
+from app.db.models import AlertKind, ContainerSnapshot
+from app.runtime.docker import Details, Stats
+from app.runtime.snapshots import Observation
+from tests.conftest import ROOT
+
+CFG = load_alerts_config(ROOT / "config")
+NOW = datetime(2026, 10, 17, 12, 0, tzinfo=UTC)
+RUNNING = Details(
+    id="a" * 64,
+    name="svc",
+    service="svc",
+    image="svc:latest",
+    state="running",
+    health="healthy",
+    exit_code=0,
+    restart_count=5,
+    started_at=NOW - timedelta(hours=1),
+    finished_at=None,
+    oom_killed=False,
+    labels={},
+)
+CALM = Stats(cpu_pct=5, mem_bytes=10, mem_limit=100)
+
+
+def snap(seconds_ago: int, restarts: int = 5, cpu: float = 5, mem: float = 10) -> ContainerSnapshot:
+    return ContainerSnapshot(
+        container="svc", at=NOW - timedelta(seconds=seconds_ago), restart_count=restarts, cpu_pct=cpu, mem_pct=mem
+    )
+
+
+def kinds(o: Observation, history: list[ContainerSnapshot] | None = None) -> set[AlertKind]:
+    return {c.kind for c in container_conditions(o, history or [], CFG, NOW)}
+
+
+def test_config_defaults() -> None:
+    assert AlertsConfig() == CFG
+
+
+def test_healthy_container_is_quiet() -> None:
+    assert kinds(Observation(RUNNING, CALM), [snap(300), snap(15)]) == set()
+
+
+def test_crash_oom_and_clean_stop() -> None:
+    crashed = replace(RUNNING, state="exited", exit_code=1, health=None)
+    oom = replace(crashed, exit_code=137, oom_killed=True)
+    stopped = replace(crashed, exit_code=0)
+
+    assert kinds(Observation(crashed, None)) == {AlertKind.CRASH}
+    assert kinds(Observation(oom, None)) == {AlertKind.OOM}
+    assert kinds(Observation(stopped, None)) == set()  # docker stop is not an incident
+
+
+def test_restart_loop_counts_within_window() -> None:
+    history = [snap(900, restarts=0), snap(500, restarts=2), snap(60, restarts=4)]
+    assert kinds(Observation(RUNNING, CALM), history) == {AlertKind.RESTART_LOOP}  # 5 - 2 ≥ 3 within 10 min
+    assert kinds(Observation(replace(RUNNING, restart_count=4), CALM), history) == set()
+
+
+def test_unhealthy() -> None:
+    assert kinds(Observation(replace(RUNNING, health="unhealthy"), CALM)) == {AlertKind.UNHEALTHY}
+    assert kinds(Observation(replace(RUNNING, health="starting"), CALM)) == set()
+
+
+def test_overload_needs_a_full_minute() -> None:
+    hot = Stats(cpu_pct=150, mem_bytes=10, mem_limit=100)
+    held = [snap(60, cpu=120), snap(45, cpu=130), snap(30, cpu=140), snap(15, cpu=150)]
+    blip = [snap(15, cpu=150)]
+    dipped = [snap(60, cpu=120), snap(30, cpu=40), snap(15, cpu=150)]
+
+    assert kinds(Observation(RUNNING, hot), held) == {AlertKind.OVERLOAD}
+    assert kinds(Observation(RUNNING, hot), blip) == set()
+    assert kinds(Observation(RUNNING, hot), dipped) == set()
+    mem_hot = Stats(cpu_pct=1, mem_bytes=95, mem_limit=100)
+    assert kinds(Observation(RUNNING, mem_hot), [snap(60, mem=95), snap(15, mem=96)]) == {AlertKind.OVERLOAD}
+
+
+def test_disk_and_docker() -> None:
+    assert disk_condition(None, CFG) == []
+    assert disk_condition(50, CFG) == []
+    [low] = disk_condition(4.2, CFG)
+    assert low.kind is AlertKind.DISK_LOW and low.service is None
+
+    assert docker_condition(None, CFG, NOW) == []
+    assert docker_condition(NOW - timedelta(minutes=2), CFG, NOW) == []
+    [down] = docker_condition(NOW - timedelta(minutes=6), CFG, NOW)
+    assert down.key == (AlertKind.DEPENDENCY_DOWN, "docker")
@@ -0,0 +1,116 @@
+from dataclasses import dataclass, field
+from datetime import UTC, datetime, timedelta
+
+import pytest
+
+from app.alerts.detectors import MONITORED, Condition, load_alerts_config
+from app.alerts.service import AlertService
+from app.db.models import Alert, AlertKind
+from app.db.session import SessionMaker
+from tests.conftest import ROOT
+
+CFG = load_alerts_config(ROOT / "config")
+T0 = datetime(2026, 10, 17, 12, 0, tzinfo=UTC)
+CRASH = Condition(AlertKind.CRASH, "svc", {"exit_code": 1}, container_id="c" * 64)
+
+
+@dataclass
+class Recorder:
+    events: list[tuple[str, int, int]] = field(default_factory=list)  # (what, alert id, count)
+
+    async def opened(self, alert: Alert, admins_only: bool) -> None:
+        self.events.append(("admins" if admins_only else "opened", alert.id, alert.count))
+
+    async def repeated(self, alert: Alert) -> None:
+        self.events.append(("repeated", alert.id, alert.count))
+
+    async def resolved(self, alert: Alert) -> None:
+        self.events.append(("resolved", alert.id, alert.count))
+
+
+@pytest.fixture
+def rec() -> Recorder:
+    return Recorder()
+
+
+@pytest.fixture
+def service(sessions: SessionMaker, rec: Recorder) -> AlertService:
+    async def tail(container_id: str, lines: int) -> str:
+        return f"last {lines} lines of {container_id[:4]}"
+
+    return AlertService(sessions, rec, CFG, log_source=tail)
+
+
+def at(minutes: float) -> datetime:
+    return T0 + timedelta(minutes=minutes)
+
+
+async def test_storm_is_one_alert_then_recovery(service: AlertService, rec: Recorder) -> None:
+    for minute in range(5):  # the condition holds for five rounds
+        await service.sync([CRASH], MONITORED, at(minute))
+    await service.sync([], MONITORED, at(6))
+
+    assert [e[0] for e in rec.events] == ["opened", "resolved"]
+    alert = await service.get(rec.events[0][1])
+    assert alert is not None and alert.payload["log"] == "last 30 lines of cccc"
+    assert alert.payload["container_id"] == "c" * 64
+
+
+async def test_flapping_within_window_bumps_counter(service: AlertService, rec: Recorder) -> None:
+    await service.sync([CRASH], MONITORED, at(0))
+    await service.sync([], MONITORED, at(1))
+    await service.sync([CRASH], MONITORED, at(3))  # back within 10 minutes
+    await service.sync([], MONITORED, at(4))
+    await service.sync([CRASH], MONITORED, at(20))  # long after: a new alert
+
+    what = [(e[0], e[2]) for e in rec.events]
+    assert what == [("opened", 1), ("resolved", 1), ("repeated", 2), ("resolved", 2), ("opened", 1)]
+    assert rec.events[0][1] == rec.events[2][1] != rec.events[4][1]
+
+
+async def test_scope_protects_other_kinds(service: AlertService, rec: Recorder) -> None:
+    await service.sync([CRASH], MONITORED, at(0))
+    # Docker down: only these kinds are judged — the crash stays open
+    await service.sync([], frozenset({AlertKind.DEPENDENCY_DOWN, AlertKind.DISK_LOW}), at(1))
+
+    assert [e[0] for e in rec.events] == ["opened"]
+    assert await service.active_count() == 1
+
+
+async def test_event_alerts_repeat_with_counter(service: AlertService, rec: Recorder) -> None:
+    attack = Condition(AlertKind.AGENT_ATTACK, "user 7", {"who": "u7", "stage": "guard", "reason": "x"})
+
+    first = await service.event(attack, admins_only=True, now=at(0))
+    second = await service.event(attack, admins_only=True, now=at(2))
+    third = await service.event(attack, admins_only=True, now=at(30))
+
+    assert first.id == second.id != third.id
+    assert [(e[0], e[2]) for e in rec.events] == [("admins", 1), ("repeated", 2), ("admins", 1)]
+    _, total = await service.page(0, now=at(31))
+    assert total == 0  # 🛡 never shows in the team's list
+
+
+async def test_list_active_first_then_history(service: AlertService) -> None:
+    other = Condition(AlertKind.UNHEALTHY, "web")
+    await service.sync([CRASH], MONITORED, at(0))
+    await service.sync([], MONITORED, at(1))
+    await service.sync([other], MONITORED, at(2))
+    old = Condition(AlertKind.OOM, "ancient")
+    await service.sync([other, old], MONITORED, at(-60 * 24 * 8))  # eight days ago, resolved right after
+    await service.sync([other], MONITORED, at(-60 * 24 * 8 + 1))
+
+    rows, total = await service.page(0, now=at(3))
+
+    assert [(a.kind, a.resolved_at is None) for a in rows] == [(AlertKind.UNHEALTHY, True), (AlertKind.CRASH, False)]
+    assert total == 2
+
+
+async def test_broken_log_source_still_alerts(sessions: SessionMaker, rec: Recorder) -> None:
+    async def broken(container_id: str, lines: int) -> str:
+        raise RuntimeError("docker gone")
+
+    service = AlertService(sessions, rec, CFG, log_source=broken)
+    await service.sync([CRASH], MONITORED, at(0))
+
+    alert = await service.get(rec.events[0][1])
+    assert alert is not None and "log" not in alert.payload