fix(alerts): краш при restart-политике, гистерезис закрытия, редкие правки
CI / ruff · mypy · pytest (pull_request) Successful in 24m47s

Найдено смоуком на живых контейнерах (examples/, #19):
- контейнер с restart: on-failure поднимается за секунду — между раундами
  его не видно в exited; краш теперь распознаётся и по росту RestartCount
  (сравнение со снапшотами до текущего раунда), повторы — счётчик
- алерт закрывается, только когда условия нет resolve_after (60 с): иначе
  флапающий сервис получал бы «✅ восстановлено» после каждого раунда
- правки сообщений со счётчиком — не чаще раза в минуту на алерт

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-17 15:52:54 +03:00
co-authored by Claude Opus 5.5
parent 6c6f766abc
commit 79ea5ce2a5
7 changed files with 77 additions and 17 deletions
+25 -1
View File
@@ -9,7 +9,8 @@ from app.db.models import Alert, AlertKind
from app.db.session import SessionMaker
from tests.conftest import ROOT
CFG = load_alerts_config(ROOT / "config")
CFG = load_alerts_config(ROOT / "config").model_copy(update={"resolve_after": 0})
SETTLING = CFG.model_copy(update={"resolve_after": 60})
T0 = datetime(2026, 10, 17, 12, 0, tzinfo=UTC)
CRASH = Condition(AlertKind.CRASH, "svc", {"exit_code": 1}, container_id="c" * 64)
@@ -114,3 +115,26 @@ async def test_broken_log_source_still_alerts(sessions: SessionMaker, rec: Recor
alert = await service.get(rec.events[0][1])
assert alert is not None and "log" not in alert.payload
async def test_resolution_waits_for_the_condition_to_settle(sessions: SessionMaker, rec: Recorder) -> None:
service = AlertService(sessions, rec, SETTLING)
await service.sync([CRASH], MONITORED, at(0))
await service.sync([], MONITORED, at(0.5)) # one good round is not recovery
await service.sync([CRASH], MONITORED, at(0.75))
await service.sync([], MONITORED, at(1))
await service.sync([], MONITORED, at(1.9))
assert [e[0] for e in rec.events] == ["opened", "resolved"]
async def test_bumps_while_open_throttle_edits(sessions: SessionMaker, rec: Recorder) -> None:
service = AlertService(sessions, rec, SETTLING)
bumped = Condition(CRASH.kind, CRASH.service, {}, CRASH.container_id, bump=True)
await service.sync([CRASH], MONITORED, at(0))
for second in (15, 30, 45, 75):
await service.sync([bumped], MONITORED, at(second / 60))
assert [(e[0], e[2]) for e in rec.events] == [("opened", 1), ("repeated", 2), ("repeated", 5)]
alert = await service.get(rec.events[0][1])
assert alert is not None and alert.count == 5