fix(alerts): краш при restart-политике, гистерезис закрытия, редкие правки
CI / ruff · mypy · pytest (pull_request) Successful in 24m47s

Найдено смоуком на живых контейнерах (examples/, #19):
- контейнер с restart: on-failure поднимается за секунду — между раундами
  его не видно в exited; краш теперь распознаётся и по росту RestartCount
  (сравнение со снапшотами до текущего раунда), повторы — счётчик
- алерт закрывается, только когда условия нет resolve_after (60 с): иначе
  флапающий сервис получал бы «✅ восстановлено» после каждого раунда
- правки сообщений со счётчиком — не чаще раза в минуту на алерт

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-17 15:52:54 +03:00
co-authored by Claude Opus 5.5
parent 6c6f766abc
commit 79ea5ce2a5
7 changed files with 77 additions and 17 deletions
+10 -2
View File
@@ -28,6 +28,7 @@ class AlertsConfig(BaseModel):
disk_path: Path = Path("/data")
dependency_down_for: int = Field(default=300, gt=0)
repeat_window: int = Field(default=600, ge=0)
resolve_after: int = Field(default=60, ge=0)
log_tail: int = Field(default=30, ge=0, le=200)
@@ -56,6 +57,7 @@ class Condition:
service: str | None
payload: dict[str, Any] = field(default_factory=dict)
container_id: str | None = None # for the log tail and the «📜 Логи» / «🩺 Статус» buttons
bump: bool = False # a fresh incident this round: if the alert is still open, its counter grows
@property
def key(self) -> tuple[AlertKind, str | None]:
@@ -69,13 +71,19 @@ def container_conditions(
d = o.details
found: list[Condition] = []
def add(kind: AlertKind, **payload: Any) -> None:
found.append(Condition(kind, d.name, payload, d.id))
def add(kind: AlertKind, bump: bool = False, **payload: Any) -> None:
found.append(Condition(kind, d.name, payload, d.id, bump))
# A restart policy brings a crashed container back within a second — between two rounds it was already
# restarted, so a grown restart count is a crash too (the exit code of that run is gone by then).
before = [s for s in history if o.at is None or as_utc(s.at) < o.at] # not this round's own row
restarted = bool(before) and d.restart_count > before[-1].restart_count
if d.state in ("exited", "dead") and d.oom_killed:
add(AlertKind.OOM, exit_code=d.exit_code)
elif d.state in ("exited", "dead") and d.exit_code != 0:
add(AlertKind.CRASH, exit_code=d.exit_code)
elif restarted or d.state == "restarting":
add(AlertKind.CRASH, bump=restarted, exit_code=d.exit_code or "≠ 0")
window = [s for s in history if as_utc(s.at) >= now - timedelta(seconds=cfg.restart_loop_window)]
if window: