fix(alerts): краш при restart-политике, гистерезис закрытия, редкие правки
CI / ruff · mypy · pytest (pull_request) Successful in 24m47s

Найдено смоуком на живых контейнерах (examples/, #19):
- контейнер с restart: on-failure поднимается за секунду — между раундами
  его не видно в exited; краш теперь распознаётся и по росту RestartCount
  (сравнение со снапшотами до текущего раунда), повторы — счётчик
- алерт закрывается, только когда условия нет resolve_after (60 с): иначе
  флапающий сервис получал бы «✅ восстановлено» после каждого раунда
- правки сообщений со счётчиком — не чаще раза в минуту на алерт

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-17 15:52:54 +03:00
co-authored by Claude Opus 5.5
parent 6c6f766abc
commit 79ea5ce2a5
7 changed files with 77 additions and 17 deletions
+26 -8
View File
@@ -15,6 +15,7 @@ from app.db.session import SessionMaker
log = logging.getLogger(__name__)
HISTORY = timedelta(days=7) # AL-6
EDIT_EVERY = timedelta(minutes=1)
LOG_CHARS = 3000
LogSource = Callable[[str, int], Awaitable[str]] # (container id, lines) -> tail
@@ -68,6 +69,11 @@ class AlertService:
known = by_key.get(key)
if known is not None and known.resolved_at is None:
known.last_at = now
if c.bump: # one more incident while still open (a crash that Docker already restarted)
known.count += 1
known.payload = {**known.payload, **c.payload}
if self._may_edit(known, now):
repeated.append(known)
continue
if (
known is not None
@@ -76,7 +82,8 @@ class AlertService:
):
known.resolved_at, known.last_at, known.count = None, now, known.count + 1 # back within the window
known.payload = {**known.payload, **c.payload}
repeated.append(known)
if self._may_edit(known, now):
repeated.append(known)
continue
fresh = Alert(
kind=c.kind, service=c.service, payload=await self._payload(c), opened_at=now, last_at=now
@@ -84,12 +91,12 @@ class AlertService:
session.add(fresh)
opened.append(fresh)
for key, a in by_key.items():
if a.resolved_at is None and key not in wanted:
# hysteresis: gone for a while, not just for one round — otherwise a flapping service
# would get «✅ восстановлено» after every blip
if a.resolved_at is None and key not in wanted and now - as_utc(a.last_at) >= self._settle:
a.resolved_at = now
resolved.append(a)
await session.flush()
for a in opened:
await session.refresh(a, ["deliveries"])
for a in opened:
await self._notifier.opened(a, admins_only=False)
for a in repeated:
@@ -112,18 +119,17 @@ class AlertService:
recent.count += 1
recent.last_at = now
recent.payload = {**recent.payload, **c.payload}
alert, new = recent, False
alert, new, edit = recent, False, self._may_edit(recent, now)
else:
alert = Alert(
kind=c.kind, service=c.service, payload=dict(c.payload), opened_at=now, last_at=now, resolved_at=now
)
session.add(alert)
await session.flush()
await session.refresh(alert, ["deliveries"])
new = True
new, edit = True, False
if new:
await self._notifier.opened(alert, admins_only=admins_only)
else:
elif edit:
await self._notifier.repeated(alert)
return alert
@@ -140,6 +146,18 @@ class AlertService:
def _window(self) -> timedelta:
return timedelta(seconds=self._cfg.repeat_window)
@property
def _settle(self) -> timedelta:
return timedelta(seconds=self._cfg.resolve_after)
def _may_edit(self, alert: Alert, now: datetime) -> bool:
"""Counter edits at most once a minute per alert; the count itself is always exact in the DB."""
last = alert.payload.get("edited_at")
if isinstance(last, str) and now - as_utc(datetime.fromisoformat(last)) < EDIT_EVERY:
return False
alert.payload = {**alert.payload, "edited_at": now.isoformat()}
return True
async def page(self, page: int, per_page: int = 10, now: datetime | None = None) -> tuple[list[Alert], int]:
"""Active first, then the last 7 days, newest first (AL-6). Admin-only kinds are left out."""
now = now or utcnow()