fix(alerts): краш при restart-политике, гистерезис закрытия, редкие правки

Найдено смоуком на живых контейнерах (examples/, #19):
- контейнер с restart: on-failure поднимается за секунду — между раундами
  его не видно в exited; краш теперь распознаётся и по росту RestartCount
  (сравнение со снапшотами до текущего раунда), повторы — счётчик
- алерт закрывается, только когда условия нет resolve_after (60 с): иначе
  флапающий сервис получал бы «✅ восстановлено» после каждого раунда
- правки сообщений со счётчиком — не чаще раза в минуту на алерт

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/alerts/detectors.py

@@ -28,6 +28,7 @@ class AlertsConfig(BaseModel):
     disk_path: Path = Path("/data")
     dependency_down_for: int = Field(default=300, gt=0)
     repeat_window: int = Field(default=600, ge=0)
+    resolve_after: int = Field(default=60, ge=0)
     log_tail: int = Field(default=30, ge=0, le=200)
 
 
@@ -56,6 +57,7 @@ class Condition:
     service: str | None
     payload: dict[str, Any] = field(default_factory=dict)
     container_id: str | None = None  # for the log tail and the «📜 Логи» / «🩺 Статус» buttons
+    bump: bool = False  # a fresh incident this round: if the alert is still open, its counter grows
 
     @property
     def key(self) -> tuple[AlertKind, str | None]:
@@ -69,13 +71,19 @@ def container_conditions(
     d = o.details
     found: list[Condition] = []
 
-    def add(kind: AlertKind, **payload: Any) -> None:
-        found.append(Condition(kind, d.name, payload, d.id))
+    def add(kind: AlertKind, bump: bool = False, **payload: Any) -> None:
+        found.append(Condition(kind, d.name, payload, d.id, bump))
 
+    # A restart policy brings a crashed container back within a second — between two rounds it was already
+    # restarted, so a grown restart count is a crash too (the exit code of that run is gone by then).
+    before = [s for s in history if o.at is None or as_utc(s.at) < o.at]  # not this round's own row
+    restarted = bool(before) and d.restart_count > before[-1].restart_count
     if d.state in ("exited", "dead") and d.oom_killed:
         add(AlertKind.OOM, exit_code=d.exit_code)
     elif d.state in ("exited", "dead") and d.exit_code != 0:
         add(AlertKind.CRASH, exit_code=d.exit_code)
+    elif restarted or d.state == "restarting":
+        add(AlertKind.CRASH, bump=restarted, exit_code=d.exit_code or "≠ 0")
 
     window = [s for s in history if as_utc(s.at) >= now - timedelta(seconds=cfg.restart_loop_window)]
     if window:

app/alerts/service.py

@@ -15,6 +15,7 @@ from app.db.session import SessionMaker
 log = logging.getLogger(__name__)
 
 HISTORY = timedelta(days=7)  # AL-6
+EDIT_EVERY = timedelta(minutes=1)
 LOG_CHARS = 3000
 
 LogSource = Callable[[str, int], Awaitable[str]]  # (container id, lines) -> tail
@@ -68,6 +69,11 @@ class AlertService:
                 known = by_key.get(key)
                 if known is not None and known.resolved_at is None:
                     known.last_at = now
+                    if c.bump:  # one more incident while still open (a crash that Docker already restarted)
+                        known.count += 1
+                        known.payload = {**known.payload, **c.payload}
+                        if self._may_edit(known, now):
+                            repeated.append(known)
                     continue
                 if (
                     known is not None
@@ -76,6 +82,7 @@ class AlertService:
                 ):
                     known.resolved_at, known.last_at, known.count = None, now, known.count + 1  # back within the window
                     known.payload = {**known.payload, **c.payload}
+                    if self._may_edit(known, now):
                         repeated.append(known)
                     continue
                 fresh = Alert(
@@ -84,12 +91,12 @@ class AlertService:
                 session.add(fresh)
                 opened.append(fresh)
             for key, a in by_key.items():
-                if a.resolved_at is None and key not in wanted:
+                # hysteresis: gone for a while, not just for one round — otherwise a flapping service
+                # would get «✅ восстановлено» after every blip
+                if a.resolved_at is None and key not in wanted and now - as_utc(a.last_at) >= self._settle:
                     a.resolved_at = now
                     resolved.append(a)
             await session.flush()
-            for a in opened:
-                await session.refresh(a, ["deliveries"])
         for a in opened:
             await self._notifier.opened(a, admins_only=False)
         for a in repeated:
@@ -112,18 +119,17 @@ class AlertService:
                 recent.count += 1
                 recent.last_at = now
                 recent.payload = {**recent.payload, **c.payload}
-                alert, new = recent, False
+                alert, new, edit = recent, False, self._may_edit(recent, now)
             else:
                 alert = Alert(
                     kind=c.kind, service=c.service, payload=dict(c.payload), opened_at=now, last_at=now, resolved_at=now
                 )
                 session.add(alert)
                 await session.flush()
-                await session.refresh(alert, ["deliveries"])
-                new = True
+                new, edit = True, False
         if new:
             await self._notifier.opened(alert, admins_only=admins_only)
-        else:
+        elif edit:
             await self._notifier.repeated(alert)
         return alert
 
@@ -140,6 +146,18 @@ class AlertService:
     def _window(self) -> timedelta:
         return timedelta(seconds=self._cfg.repeat_window)
 
+    @property
+    def _settle(self) -> timedelta:
+        return timedelta(seconds=self._cfg.resolve_after)
+
+    def _may_edit(self, alert: Alert, now: datetime) -> bool:
+        """Counter edits at most once a minute per alert; the count itself is always exact in the DB."""
+        last = alert.payload.get("edited_at")
+        if isinstance(last, str) and now - as_utc(datetime.fromisoformat(last)) < EDIT_EVERY:
+            return False
+        alert.payload = {**alert.payload, "edited_at": now.isoformat()}
+        return True
+
     async def page(self, page: int, per_page: int = 10, now: datetime | None = None) -> tuple[list[Alert], int]:
         """Active first, then the last 7 days, newest first (AL-6). Admin-only kinds are left out."""
         now = now or utcnow()

app/runtime/snapshots.py

@@ -7,7 +7,7 @@ Old rows are removed by `app.db.maintenance`.
 import asyncio
 import logging
 from collections.abc import Awaitable, Callable
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 from datetime import datetime
 
 from sqlalchemy import select
@@ -62,6 +62,7 @@ class Observation:
 
     details: Details
     stats: Stats | None  # None when not running or stats failed
+    at: datetime | None = None  # the round's time; its own snapshot row carries the same `at`
 
 
 async def _observe(docker: DockerClient, container_id: str) -> Observation | None:
@@ -76,7 +77,7 @@ async def _observe(docker: DockerClient, container_id: str) -> Observation | Non
             stats = await docker.stats(container_id)
         except DockerError as e:
             log.warning("stats %s: %s", container_id[:12], e)
-    return Observation(details, stats)
+    return Observation(details, stats)  # `at` is set by collect_once
 
 
 def _row(o: Observation, at: datetime) -> ContainerSnapshot | None:
@@ -96,7 +97,7 @@ async def collect_once(docker: DockerClient, store: SnapshotStore, now: datetime
     """Inspect every managed container, stats for the running ones; a snapshot row per running container."""
     at = now or utcnow()
     found = await asyncio.gather(*(_observe(docker, c.id) for c in await docker.containers()))
-    observations = [o for o in found if o is not None]
+    observations = [replace(o, at=at) for o in found if o is not None]
     await store.add([row for o in observations if (row := _row(o, at)) is not None])
     return observations
 

config/alerts.yaml

@@ -14,5 +14,7 @@ disk_path: /data
 dependency_down_for: 300
 # Повтор того же алерта (тип + сервис) в этом окне, секунды, — счётчик в исходном сообщении, а не новое (AL-5)
 repeat_window: 600
+# Закрывать алерт («✅ восстановлено»), когда условия нет уже столько секунд, — не после одного удачного раунда
+resolve_after: 60
 # Строк лога в сообщении алерта (AL-3)
 log_tail: 30

tests/alerts/test_detectors.py

@@ -61,9 +61,16 @@ def test_crash_oom_and_clean_stop() -> None:
 
 
 def test_restart_loop_counts_within_window() -> None:
-    history = [snap(900, restarts=0), snap(500, restarts=2), snap(60, restarts=4)]
+    history = [snap(900, restarts=0), snap(500, restarts=2), snap(60, restarts=5)]
     assert kinds(Observation(RUNNING, CALM), history) == {AlertKind.RESTART_LOOP}  # 5 - 2 ≥ 3 within 10 min
-    assert kinds(Observation(replace(RUNNING, restart_count=4), CALM), history) == set()
+    assert kinds(Observation(replace(RUNNING, restart_count=4), CALM), history[:2]) == {AlertKind.CRASH}  # 2 → 4
+
+
+def test_crash_seen_through_restart_policy() -> None:
+    """Restarted between two rounds: the container is running again, but the restart count gave it away."""
+    [crash] = container_conditions(Observation(replace(RUNNING, restart_count=6), CALM), [snap(15)], CFG, NOW)
+    assert crash.kind is AlertKind.CRASH and crash.bump and crash.payload["exit_code"] == "≠ 0"
+    assert kinds(Observation(replace(RUNNING, state="restarting"), None), [snap(15)]) == {AlertKind.CRASH}
 
 
 def test_unhealthy() -> None:

tests/alerts/test_service.py

@@ -9,7 +9,8 @@ from app.db.models import Alert, AlertKind
 from app.db.session import SessionMaker
 from tests.conftest import ROOT
 
-CFG = load_alerts_config(ROOT / "config")
+CFG = load_alerts_config(ROOT / "config").model_copy(update={"resolve_after": 0})
+SETTLING = CFG.model_copy(update={"resolve_after": 60})
 T0 = datetime(2026, 10, 17, 12, 0, tzinfo=UTC)
 CRASH = Condition(AlertKind.CRASH, "svc", {"exit_code": 1}, container_id="c" * 64)
 
@@ -114,3 +115,26 @@ async def test_broken_log_source_still_alerts(sessions: SessionMaker, rec: Recor
 
     alert = await service.get(rec.events[0][1])
     assert alert is not None and "log" not in alert.payload
+
+
+async def test_resolution_waits_for_the_condition_to_settle(sessions: SessionMaker, rec: Recorder) -> None:
+    service = AlertService(sessions, rec, SETTLING)
+    await service.sync([CRASH], MONITORED, at(0))
+    await service.sync([], MONITORED, at(0.5))  # one good round is not recovery
+    await service.sync([CRASH], MONITORED, at(0.75))
+    await service.sync([], MONITORED, at(1))
+    await service.sync([], MONITORED, at(1.9))
+
+    assert [e[0] for e in rec.events] == ["opened", "resolved"]
+
+
+async def test_bumps_while_open_throttle_edits(sessions: SessionMaker, rec: Recorder) -> None:
+    service = AlertService(sessions, rec, SETTLING)
+    bumped = Condition(CRASH.kind, CRASH.service, {}, CRASH.container_id, bump=True)
+    await service.sync([CRASH], MONITORED, at(0))
+    for second in (15, 30, 45, 75):
+        await service.sync([bumped], MONITORED, at(second / 60))
+
+    assert [(e[0], e[2]) for e in rec.events] == [("opened", 1), ("repeated", 2), ("repeated", 5)]
+    alert = await service.get(rec.events[0][1])
+    assert alert is not None and alert.count == 5

tests/bot/test_alerts.py

@@ -27,7 +27,7 @@ from tests.fakes import TgClient
 from tests.runtime.fake_docker import FakeDocker, now_minus
 
 MEMBER, BLOCKED = 5005, 6006
-CFG = load_alerts_config(ROOT / "config")
+CFG = load_alerts_config(ROOT / "config").model_copy(update={"resolve_after": 0})  # hysteresis: tests/alerts
 
 
 @pytest.fixture(autouse=True)