fix(alerts): краш при restart-политике, гистерезис закрытия, редкие правки
CI / ruff · mypy · pytest (pull_request) Successful in 24m47s

Найдено смоуком на живых контейнерах (examples/, #19):
- контейнер с restart: on-failure поднимается за секунду — между раундами
  его не видно в exited; краш теперь распознаётся и по росту RestartCount
  (сравнение со снапшотами до текущего раунда), повторы — счётчик
- алерт закрывается, только когда условия нет resolve_after (60 с): иначе
  флапающий сервис получал бы «✅ восстановлено» после каждого раунда
- правки сообщений со счётчиком — не чаще раза в минуту на алерт

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-17 15:52:54 +03:00
co-authored by Claude Opus 5.5
parent 6c6f766abc
commit 79ea5ce2a5
7 changed files with 77 additions and 17 deletions
+9 -2
View File
@@ -61,9 +61,16 @@ def test_crash_oom_and_clean_stop() -> None:
def test_restart_loop_counts_within_window() -> None:
history = [snap(900, restarts=0), snap(500, restarts=2), snap(60, restarts=4)]
history = [snap(900, restarts=0), snap(500, restarts=2), snap(60, restarts=5)]
assert kinds(Observation(RUNNING, CALM), history) == {AlertKind.RESTART_LOOP} # 5 - 2 ≥ 3 within 10 min
assert kinds(Observation(replace(RUNNING, restart_count=4), CALM), history) == set()
assert kinds(Observation(replace(RUNNING, restart_count=4), CALM), history[:2]) == {AlertKind.CRASH} # 2 → 4
def test_crash_seen_through_restart_policy() -> None:
"""Restarted between two rounds: the container is running again, but the restart count gave it away."""
[crash] = container_conditions(Observation(replace(RUNNING, restart_count=6), CALM), [snap(15)], CFG, NOW)
assert crash.kind is AlertKind.CRASH and crash.bump and crash.payload["exit_code"] == "≠ 0"
assert kinds(Observation(replace(RUNNING, state="restarting"), None), [snap(15)]) == {AlertKind.CRASH}
def test_unhealthy() -> None:
+25 -1
View File
@@ -9,7 +9,8 @@ from app.db.models import Alert, AlertKind
from app.db.session import SessionMaker
from tests.conftest import ROOT
CFG = load_alerts_config(ROOT / "config")
CFG = load_alerts_config(ROOT / "config").model_copy(update={"resolve_after": 0})
SETTLING = CFG.model_copy(update={"resolve_after": 60})
T0 = datetime(2026, 10, 17, 12, 0, tzinfo=UTC)
CRASH = Condition(AlertKind.CRASH, "svc", {"exit_code": 1}, container_id="c" * 64)
@@ -114,3 +115,26 @@ async def test_broken_log_source_still_alerts(sessions: SessionMaker, rec: Recor
alert = await service.get(rec.events[0][1])
assert alert is not None and "log" not in alert.payload
async def test_resolution_waits_for_the_condition_to_settle(sessions: SessionMaker, rec: Recorder) -> None:
service = AlertService(sessions, rec, SETTLING)
await service.sync([CRASH], MONITORED, at(0))
await service.sync([], MONITORED, at(0.5)) # one good round is not recovery
await service.sync([CRASH], MONITORED, at(0.75))
await service.sync([], MONITORED, at(1))
await service.sync([], MONITORED, at(1.9))
assert [e[0] for e in rec.events] == ["opened", "resolved"]
async def test_bumps_while_open_throttle_edits(sessions: SessionMaker, rec: Recorder) -> None:
service = AlertService(sessions, rec, SETTLING)
bumped = Condition(CRASH.kind, CRASH.service, {}, CRASH.container_id, bump=True)
await service.sync([CRASH], MONITORED, at(0))
for second in (15, 30, 45, 75):
await service.sync([bumped], MONITORED, at(second / 60))
assert [(e[0], e[2]) for e in rec.events] == [("opened", 1), ("repeated", 2), ("repeated", 5)]
alert = await service.get(rec.events[0][1])
assert alert is not None and alert.count == 5