fix(alerts): краш при restart-политике, гистерезис закрытия, редкие правки
CI / ruff · mypy · pytest (pull_request) Successful in 24m47s
CI / ruff · mypy · pytest (pull_request) Successful in 24m47s
Найдено смоуком на живых контейнерах (examples/, #19): - контейнер с restart: on-failure поднимается за секунду — между раундами его не видно в exited; краш теперь распознаётся и по росту RestartCount (сравнение со снапшотами до текущего раунда), повторы — счётчик - алерт закрывается, только когда условия нет resolve_after (60 с): иначе флапающий сервис получал бы «✅ восстановлено» после каждого раунда - правки сообщений со счётчиком — не чаще раза в минуту на алерт Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -9,7 +9,8 @@ from app.db.models import Alert, AlertKind
|
||||
from app.db.session import SessionMaker
|
||||
from tests.conftest import ROOT
|
||||
|
||||
CFG = load_alerts_config(ROOT / "config")
|
||||
CFG = load_alerts_config(ROOT / "config").model_copy(update={"resolve_after": 0})
|
||||
SETTLING = CFG.model_copy(update={"resolve_after": 60})
|
||||
T0 = datetime(2026, 10, 17, 12, 0, tzinfo=UTC)
|
||||
CRASH = Condition(AlertKind.CRASH, "svc", {"exit_code": 1}, container_id="c" * 64)
|
||||
|
||||
@@ -114,3 +115,26 @@ async def test_broken_log_source_still_alerts(sessions: SessionMaker, rec: Recor
|
||||
|
||||
alert = await service.get(rec.events[0][1])
|
||||
assert alert is not None and "log" not in alert.payload
|
||||
|
||||
|
||||
async def test_resolution_waits_for_the_condition_to_settle(sessions: SessionMaker, rec: Recorder) -> None:
|
||||
service = AlertService(sessions, rec, SETTLING)
|
||||
await service.sync([CRASH], MONITORED, at(0))
|
||||
await service.sync([], MONITORED, at(0.5)) # one good round is not recovery
|
||||
await service.sync([CRASH], MONITORED, at(0.75))
|
||||
await service.sync([], MONITORED, at(1))
|
||||
await service.sync([], MONITORED, at(1.9))
|
||||
|
||||
assert [e[0] for e in rec.events] == ["opened", "resolved"]
|
||||
|
||||
|
||||
async def test_bumps_while_open_throttle_edits(sessions: SessionMaker, rec: Recorder) -> None:
|
||||
service = AlertService(sessions, rec, SETTLING)
|
||||
bumped = Condition(CRASH.kind, CRASH.service, {}, CRASH.container_id, bump=True)
|
||||
await service.sync([CRASH], MONITORED, at(0))
|
||||
for second in (15, 30, 45, 75):
|
||||
await service.sync([bumped], MONITORED, at(second / 60))
|
||||
|
||||
assert [(e[0], e[2]) for e in rec.events] == [("opened", 1), ("repeated", 2), ("repeated", 5)]
|
||||
alert = await service.get(rec.events[0][1])
|
||||
assert alert is not None and alert.count == 5
|
||||
|
||||
Reference in New Issue
Block a user