fix(alerts): краш при restart-политике, гистерезис закрытия, редкие правки
CI / ruff · mypy · pytest (pull_request) Successful in 24m47s
CI / ruff · mypy · pytest (pull_request) Successful in 24m47s
Найдено смоуком на живых контейнерах (examples/, #19): - контейнер с restart: on-failure поднимается за секунду — между раундами его не видно в exited; краш теперь распознаётся и по росту RestartCount (сравнение со снапшотами до текущего раунда), повторы — счётчик - алерт закрывается, только когда условия нет resolve_after (60 с): иначе флапающий сервис получал бы «✅ восстановлено» после каждого раунда - правки сообщений со счётчиком — не чаще раза в минуту на алерт Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -61,9 +61,16 @@ def test_crash_oom_and_clean_stop() -> None:
|
||||
|
||||
|
||||
def test_restart_loop_counts_within_window() -> None:
|
||||
history = [snap(900, restarts=0), snap(500, restarts=2), snap(60, restarts=4)]
|
||||
history = [snap(900, restarts=0), snap(500, restarts=2), snap(60, restarts=5)]
|
||||
assert kinds(Observation(RUNNING, CALM), history) == {AlertKind.RESTART_LOOP} # 5 - 2 ≥ 3 within 10 min
|
||||
assert kinds(Observation(replace(RUNNING, restart_count=4), CALM), history) == set()
|
||||
assert kinds(Observation(replace(RUNNING, restart_count=4), CALM), history[:2]) == {AlertKind.CRASH} # 2 → 4
|
||||
|
||||
|
||||
def test_crash_seen_through_restart_policy() -> None:
|
||||
"""Restarted between two rounds: the container is running again, but the restart count gave it away."""
|
||||
[crash] = container_conditions(Observation(replace(RUNNING, restart_count=6), CALM), [snap(15)], CFG, NOW)
|
||||
assert crash.kind is AlertKind.CRASH and crash.bump and crash.payload["exit_code"] == "≠ 0"
|
||||
assert kinds(Observation(replace(RUNNING, state="restarting"), None), [snap(15)]) == {AlertKind.CRASH}
|
||||
|
||||
|
||||
def test_unhealthy() -> None:
|
||||
|
||||
@@ -9,7 +9,8 @@ from app.db.models import Alert, AlertKind
|
||||
from app.db.session import SessionMaker
|
||||
from tests.conftest import ROOT
|
||||
|
||||
CFG = load_alerts_config(ROOT / "config")
|
||||
CFG = load_alerts_config(ROOT / "config").model_copy(update={"resolve_after": 0})
|
||||
SETTLING = CFG.model_copy(update={"resolve_after": 60})
|
||||
T0 = datetime(2026, 10, 17, 12, 0, tzinfo=UTC)
|
||||
CRASH = Condition(AlertKind.CRASH, "svc", {"exit_code": 1}, container_id="c" * 64)
|
||||
|
||||
@@ -114,3 +115,26 @@ async def test_broken_log_source_still_alerts(sessions: SessionMaker, rec: Recor
|
||||
|
||||
alert = await service.get(rec.events[0][1])
|
||||
assert alert is not None and "log" not in alert.payload
|
||||
|
||||
|
||||
async def test_resolution_waits_for_the_condition_to_settle(sessions: SessionMaker, rec: Recorder) -> None:
|
||||
service = AlertService(sessions, rec, SETTLING)
|
||||
await service.sync([CRASH], MONITORED, at(0))
|
||||
await service.sync([], MONITORED, at(0.5)) # one good round is not recovery
|
||||
await service.sync([CRASH], MONITORED, at(0.75))
|
||||
await service.sync([], MONITORED, at(1))
|
||||
await service.sync([], MONITORED, at(1.9))
|
||||
|
||||
assert [e[0] for e in rec.events] == ["opened", "resolved"]
|
||||
|
||||
|
||||
async def test_bumps_while_open_throttle_edits(sessions: SessionMaker, rec: Recorder) -> None:
|
||||
service = AlertService(sessions, rec, SETTLING)
|
||||
bumped = Condition(CRASH.kind, CRASH.service, {}, CRASH.container_id, bump=True)
|
||||
await service.sync([CRASH], MONITORED, at(0))
|
||||
for second in (15, 30, 45, 75):
|
||||
await service.sync([bumped], MONITORED, at(second / 60))
|
||||
|
||||
assert [(e[0], e[2]) for e in rec.events] == [("opened", 1), ("repeated", 2), ("repeated", 5)]
|
||||
alert = await service.get(rec.events[0][1])
|
||||
assert alert is not None and alert.count == 5
|
||||
|
||||
@@ -27,7 +27,7 @@ from tests.fakes import TgClient
|
||||
from tests.runtime.fake_docker import FakeDocker, now_minus
|
||||
|
||||
MEMBER, BLOCKED = 5005, 6006
|
||||
CFG = load_alerts_config(ROOT / "config")
|
||||
CFG = load_alerts_config(ROOT / "config").model_copy(update={"resolve_after": 0}) # hysteresis: tests/alerts
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
||||
Reference in New Issue
Block a user