fix(alerts): краш при restart-политике, гистерезис закрытия, редкие правки
CI / ruff · mypy · pytest (pull_request) Successful in 24m47s
CI / ruff · mypy · pytest (pull_request) Successful in 24m47s
Найдено смоуком на живых контейнерах (examples/, #19): - контейнер с restart: on-failure поднимается за секунду — между раундами его не видно в exited; краш теперь распознаётся и по росту RestartCount (сравнение со снапшотами до текущего раунда), повторы — счётчик - алерт закрывается, только когда условия нет resolve_after (60 с): иначе флапающий сервис получал бы «✅ восстановлено» после каждого раунда - правки сообщений со счётчиком — не чаще раза в минуту на алерт Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+26
-8
@@ -15,6 +15,7 @@ from app.db.session import SessionMaker
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
HISTORY = timedelta(days=7) # AL-6
|
||||
EDIT_EVERY = timedelta(minutes=1)
|
||||
LOG_CHARS = 3000
|
||||
|
||||
LogSource = Callable[[str, int], Awaitable[str]] # (container id, lines) -> tail
|
||||
@@ -68,6 +69,11 @@ class AlertService:
|
||||
known = by_key.get(key)
|
||||
if known is not None and known.resolved_at is None:
|
||||
known.last_at = now
|
||||
if c.bump: # one more incident while still open (a crash that Docker already restarted)
|
||||
known.count += 1
|
||||
known.payload = {**known.payload, **c.payload}
|
||||
if self._may_edit(known, now):
|
||||
repeated.append(known)
|
||||
continue
|
||||
if (
|
||||
known is not None
|
||||
@@ -76,7 +82,8 @@ class AlertService:
|
||||
):
|
||||
known.resolved_at, known.last_at, known.count = None, now, known.count + 1 # back within the window
|
||||
known.payload = {**known.payload, **c.payload}
|
||||
repeated.append(known)
|
||||
if self._may_edit(known, now):
|
||||
repeated.append(known)
|
||||
continue
|
||||
fresh = Alert(
|
||||
kind=c.kind, service=c.service, payload=await self._payload(c), opened_at=now, last_at=now
|
||||
@@ -84,12 +91,12 @@ class AlertService:
|
||||
session.add(fresh)
|
||||
opened.append(fresh)
|
||||
for key, a in by_key.items():
|
||||
if a.resolved_at is None and key not in wanted:
|
||||
# hysteresis: gone for a while, not just for one round — otherwise a flapping service
|
||||
# would get «✅ восстановлено» after every blip
|
||||
if a.resolved_at is None and key not in wanted and now - as_utc(a.last_at) >= self._settle:
|
||||
a.resolved_at = now
|
||||
resolved.append(a)
|
||||
await session.flush()
|
||||
for a in opened:
|
||||
await session.refresh(a, ["deliveries"])
|
||||
for a in opened:
|
||||
await self._notifier.opened(a, admins_only=False)
|
||||
for a in repeated:
|
||||
@@ -112,18 +119,17 @@ class AlertService:
|
||||
recent.count += 1
|
||||
recent.last_at = now
|
||||
recent.payload = {**recent.payload, **c.payload}
|
||||
alert, new = recent, False
|
||||
alert, new, edit = recent, False, self._may_edit(recent, now)
|
||||
else:
|
||||
alert = Alert(
|
||||
kind=c.kind, service=c.service, payload=dict(c.payload), opened_at=now, last_at=now, resolved_at=now
|
||||
)
|
||||
session.add(alert)
|
||||
await session.flush()
|
||||
await session.refresh(alert, ["deliveries"])
|
||||
new = True
|
||||
new, edit = True, False
|
||||
if new:
|
||||
await self._notifier.opened(alert, admins_only=admins_only)
|
||||
else:
|
||||
elif edit:
|
||||
await self._notifier.repeated(alert)
|
||||
return alert
|
||||
|
||||
@@ -140,6 +146,18 @@ class AlertService:
|
||||
def _window(self) -> timedelta:
|
||||
return timedelta(seconds=self._cfg.repeat_window)
|
||||
|
||||
@property
|
||||
def _settle(self) -> timedelta:
|
||||
return timedelta(seconds=self._cfg.resolve_after)
|
||||
|
||||
def _may_edit(self, alert: Alert, now: datetime) -> bool:
|
||||
"""Counter edits at most once a minute per alert; the count itself is always exact in the DB."""
|
||||
last = alert.payload.get("edited_at")
|
||||
if isinstance(last, str) and now - as_utc(datetime.fromisoformat(last)) < EDIT_EVERY:
|
||||
return False
|
||||
alert.payload = {**alert.payload, "edited_at": now.isoformat()}
|
||||
return True
|
||||
|
||||
async def page(self, page: int, per_page: int = 10, now: datetime | None = None) -> tuple[list[Alert], int]:
|
||||
"""Active first, then the last 7 days, newest first (AL-6). Admin-only kinds are left out."""
|
||||
now = now or utcnow()
|
||||
|
||||
Reference in New Issue
Block a user