diff --git a/app/alerts/detectors.py b/app/alerts/detectors.py index d24a4a9..9c7c9b6 100644 --- a/app/alerts/detectors.py +++ b/app/alerts/detectors.py @@ -28,6 +28,7 @@ class AlertsConfig(BaseModel): disk_path: Path = Path("/data") dependency_down_for: int = Field(default=300, gt=0) repeat_window: int = Field(default=600, ge=0) + resolve_after: int = Field(default=60, ge=0) log_tail: int = Field(default=30, ge=0, le=200) @@ -56,6 +57,7 @@ class Condition: service: str | None payload: dict[str, Any] = field(default_factory=dict) container_id: str | None = None # for the log tail and the «📜 Логи» / «🩺 Статус» buttons + bump: bool = False # a fresh incident this round: if the alert is still open, its counter grows @property def key(self) -> tuple[AlertKind, str | None]: @@ -69,13 +71,19 @@ def container_conditions( d = o.details found: list[Condition] = [] - def add(kind: AlertKind, **payload: Any) -> None: - found.append(Condition(kind, d.name, payload, d.id)) + def add(kind: AlertKind, bump: bool = False, **payload: Any) -> None: + found.append(Condition(kind, d.name, payload, d.id, bump)) + # A restart policy brings a crashed container back within a second — between two rounds it was already + # restarted, so a grown restart count is a crash too (the exit code of that run is gone by then). + before = [s for s in history if o.at is None or as_utc(s.at) < o.at] # not this round's own row + restarted = bool(before) and d.restart_count > before[-1].restart_count if d.state in ("exited", "dead") and d.oom_killed: add(AlertKind.OOM, exit_code=d.exit_code) elif d.state in ("exited", "dead") and d.exit_code != 0: add(AlertKind.CRASH, exit_code=d.exit_code) + elif restarted or d.state == "restarting": + add(AlertKind.CRASH, bump=restarted, exit_code=d.exit_code or "≠ 0") window = [s for s in history if as_utc(s.at) >= now - timedelta(seconds=cfg.restart_loop_window)] if window: diff --git a/app/alerts/service.py b/app/alerts/service.py index a9cc3d4..520e5c1 100644 --- a/app/alerts/service.py +++ b/app/alerts/service.py @@ -15,6 +15,7 @@ from app.db.session import SessionMaker log = logging.getLogger(__name__) HISTORY = timedelta(days=7) # AL-6 +EDIT_EVERY = timedelta(minutes=1) LOG_CHARS = 3000 LogSource = Callable[[str, int], Awaitable[str]] # (container id, lines) -> tail @@ -68,6 +69,11 @@ class AlertService: known = by_key.get(key) if known is not None and known.resolved_at is None: known.last_at = now + if c.bump: # one more incident while still open (a crash that Docker already restarted) + known.count += 1 + known.payload = {**known.payload, **c.payload} + if self._may_edit(known, now): + repeated.append(known) continue if ( known is not None @@ -76,7 +82,8 @@ class AlertService: ): known.resolved_at, known.last_at, known.count = None, now, known.count + 1 # back within the window known.payload = {**known.payload, **c.payload} - repeated.append(known) + if self._may_edit(known, now): + repeated.append(known) continue fresh = Alert( kind=c.kind, service=c.service, payload=await self._payload(c), opened_at=now, last_at=now @@ -84,12 +91,12 @@ class AlertService: session.add(fresh) opened.append(fresh) for key, a in by_key.items(): - if a.resolved_at is None and key not in wanted: + # hysteresis: gone for a while, not just for one round — otherwise a flapping service + # would get «✅ восстановлено» after every blip + if a.resolved_at is None and key not in wanted and now - as_utc(a.last_at) >= self._settle: a.resolved_at = now resolved.append(a) await session.flush() - for a in opened: - await session.refresh(a, ["deliveries"]) for a in opened: await self._notifier.opened(a, admins_only=False) for a in repeated: @@ -112,18 +119,17 @@ class AlertService: recent.count += 1 recent.last_at = now recent.payload = {**recent.payload, **c.payload} - alert, new = recent, False + alert, new, edit = recent, False, self._may_edit(recent, now) else: alert = Alert( kind=c.kind, service=c.service, payload=dict(c.payload), opened_at=now, last_at=now, resolved_at=now ) session.add(alert) await session.flush() - await session.refresh(alert, ["deliveries"]) - new = True + new, edit = True, False if new: await self._notifier.opened(alert, admins_only=admins_only) - else: + elif edit: await self._notifier.repeated(alert) return alert @@ -140,6 +146,18 @@ class AlertService: def _window(self) -> timedelta: return timedelta(seconds=self._cfg.repeat_window) + @property + def _settle(self) -> timedelta: + return timedelta(seconds=self._cfg.resolve_after) + + def _may_edit(self, alert: Alert, now: datetime) -> bool: + """Counter edits at most once a minute per alert; the count itself is always exact in the DB.""" + last = alert.payload.get("edited_at") + if isinstance(last, str) and now - as_utc(datetime.fromisoformat(last)) < EDIT_EVERY: + return False + alert.payload = {**alert.payload, "edited_at": now.isoformat()} + return True + async def page(self, page: int, per_page: int = 10, now: datetime | None = None) -> tuple[list[Alert], int]: """Active first, then the last 7 days, newest first (AL-6). Admin-only kinds are left out.""" now = now or utcnow() diff --git a/app/runtime/snapshots.py b/app/runtime/snapshots.py index 5f6667f..f51d073 100644 --- a/app/runtime/snapshots.py +++ b/app/runtime/snapshots.py @@ -7,7 +7,7 @@ Old rows are removed by `app.db.maintenance`. import asyncio import logging from collections.abc import Awaitable, Callable -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import datetime from sqlalchemy import select @@ -62,6 +62,7 @@ class Observation: details: Details stats: Stats | None # None when not running or stats failed + at: datetime | None = None # the round's time; its own snapshot row carries the same `at` async def _observe(docker: DockerClient, container_id: str) -> Observation | None: @@ -76,7 +77,7 @@ async def _observe(docker: DockerClient, container_id: str) -> Observation | Non stats = await docker.stats(container_id) except DockerError as e: log.warning("stats %s: %s", container_id[:12], e) - return Observation(details, stats) + return Observation(details, stats) # `at` is set by collect_once def _row(o: Observation, at: datetime) -> ContainerSnapshot | None: @@ -96,7 +97,7 @@ async def collect_once(docker: DockerClient, store: SnapshotStore, now: datetime """Inspect every managed container, stats for the running ones; a snapshot row per running container.""" at = now or utcnow() found = await asyncio.gather(*(_observe(docker, c.id) for c in await docker.containers())) - observations = [o for o in found if o is not None] + observations = [replace(o, at=at) for o in found if o is not None] await store.add([row for o in observations if (row := _row(o, at)) is not None]) return observations diff --git a/config/alerts.yaml b/config/alerts.yaml index 38ace05..57207ae 100644 --- a/config/alerts.yaml +++ b/config/alerts.yaml @@ -14,5 +14,7 @@ disk_path: /data dependency_down_for: 300 # Повтор того же алерта (тип + сервис) в этом окне, секунды, — счётчик в исходном сообщении, а не новое (AL-5) repeat_window: 600 +# Закрывать алерт («✅ восстановлено»), когда условия нет уже столько секунд, — не после одного удачного раунда +resolve_after: 60 # Строк лога в сообщении алерта (AL-3) log_tail: 30 diff --git a/tests/alerts/test_detectors.py b/tests/alerts/test_detectors.py index 5117488..e9fe9d4 100644 --- a/tests/alerts/test_detectors.py +++ b/tests/alerts/test_detectors.py @@ -61,9 +61,16 @@ def test_crash_oom_and_clean_stop() -> None: def test_restart_loop_counts_within_window() -> None: - history = [snap(900, restarts=0), snap(500, restarts=2), snap(60, restarts=4)] + history = [snap(900, restarts=0), snap(500, restarts=2), snap(60, restarts=5)] assert kinds(Observation(RUNNING, CALM), history) == {AlertKind.RESTART_LOOP} # 5 - 2 ≥ 3 within 10 min - assert kinds(Observation(replace(RUNNING, restart_count=4), CALM), history) == set() + assert kinds(Observation(replace(RUNNING, restart_count=4), CALM), history[:2]) == {AlertKind.CRASH} # 2 → 4 + + +def test_crash_seen_through_restart_policy() -> None: + """Restarted between two rounds: the container is running again, but the restart count gave it away.""" + [crash] = container_conditions(Observation(replace(RUNNING, restart_count=6), CALM), [snap(15)], CFG, NOW) + assert crash.kind is AlertKind.CRASH and crash.bump and crash.payload["exit_code"] == "≠ 0" + assert kinds(Observation(replace(RUNNING, state="restarting"), None), [snap(15)]) == {AlertKind.CRASH} def test_unhealthy() -> None: diff --git a/tests/alerts/test_service.py b/tests/alerts/test_service.py index 1827513..735832b 100644 --- a/tests/alerts/test_service.py +++ b/tests/alerts/test_service.py @@ -9,7 +9,8 @@ from app.db.models import Alert, AlertKind from app.db.session import SessionMaker from tests.conftest import ROOT -CFG = load_alerts_config(ROOT / "config") +CFG = load_alerts_config(ROOT / "config").model_copy(update={"resolve_after": 0}) +SETTLING = CFG.model_copy(update={"resolve_after": 60}) T0 = datetime(2026, 10, 17, 12, 0, tzinfo=UTC) CRASH = Condition(AlertKind.CRASH, "svc", {"exit_code": 1}, container_id="c" * 64) @@ -114,3 +115,26 @@ async def test_broken_log_source_still_alerts(sessions: SessionMaker, rec: Recor alert = await service.get(rec.events[0][1]) assert alert is not None and "log" not in alert.payload + + +async def test_resolution_waits_for_the_condition_to_settle(sessions: SessionMaker, rec: Recorder) -> None: + service = AlertService(sessions, rec, SETTLING) + await service.sync([CRASH], MONITORED, at(0)) + await service.sync([], MONITORED, at(0.5)) # one good round is not recovery + await service.sync([CRASH], MONITORED, at(0.75)) + await service.sync([], MONITORED, at(1)) + await service.sync([], MONITORED, at(1.9)) + + assert [e[0] for e in rec.events] == ["opened", "resolved"] + + +async def test_bumps_while_open_throttle_edits(sessions: SessionMaker, rec: Recorder) -> None: + service = AlertService(sessions, rec, SETTLING) + bumped = Condition(CRASH.kind, CRASH.service, {}, CRASH.container_id, bump=True) + await service.sync([CRASH], MONITORED, at(0)) + for second in (15, 30, 45, 75): + await service.sync([bumped], MONITORED, at(second / 60)) + + assert [(e[0], e[2]) for e in rec.events] == [("opened", 1), ("repeated", 2), ("repeated", 5)] + alert = await service.get(rec.events[0][1]) + assert alert is not None and alert.count == 5 diff --git a/tests/bot/test_alerts.py b/tests/bot/test_alerts.py index c3857c4..d6d1880 100644 --- a/tests/bot/test_alerts.py +++ b/tests/bot/test_alerts.py @@ -27,7 +27,7 @@ from tests.fakes import TgClient from tests.runtime.fake_docker import FakeDocker, now_minus MEMBER, BLOCKED = 5005, 6006 -CFG = load_alerts_config(ROOT / "config") +CFG = load_alerts_config(ROOT / "config").model_copy(update={"resolve_after": 0}) # hysteresis: tests/alerts @pytest.fixture(autouse=True)