feat(alerts): алерты всей команде — детекторы, дедупликация, список (#15)
CI / ruff · mypy · pytest (pull_request) Successful in 4m49s

- детекторы по состоянию после каждого раунда снапшотов: краш, OOM,
  цикл рестартов, unhealthy, перегрузка ≥ 1 мин, мало места, Docker
  недоступен; пороги в config/alerts.yaml
- AlertService: ключ kind+service, повтор в 10 мин — счётчик в исходном
  сообщении, «✅ восстановлено» ответом; разовые события (🛡) со счётчиком
- рассылка всем активным (🛡 — только админам), темп ~25/с, RetryAfter,
  заблокировавший бота пропускается; alert_deliveries для правок и ответов
- «🚨 Алерты»: активные + 7 дней по 10, карточка; кнопки «📜 Логи»,
  «🩺 Статус» под алертом открывают новое сообщение, а не затирают алерт
- 🛡 атаки через агента идут через общий сервис алертов
- убраны заглушки «Раздел в разработке» — у всех разделов свои роутеры

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-17 15:36:09 +03:00
co-authored by Claude Opus 5.5
parent 65acf3abf1
commit 6c6f766abc
24 changed files with 1355 additions and 107 deletions
+109
View File
@@ -0,0 +1,109 @@
"""Pure functions: one round of observations (+ snapshot history) → the conditions that hold right now.
The alert service compares them with open alerts: a new condition opens an alert, a vanished one resolves it.
Being state-based, a detector can't flap on a missed event, and a crash loop stays one alert with a counter.
"""
from dataclasses import dataclass, field
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any
import yaml
from pydantic import BaseModel, ConfigDict, Field
from app.db.models import AlertKind, ContainerSnapshot, as_utc
from app.runtime.snapshots import Observation
class AlertsConfig(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
restart_loop_count: int = Field(default=3, ge=1)
restart_loop_window: int = Field(default=600, gt=0)
overload_cpu_pct: float = Field(default=80, gt=0)
overload_mem_pct: float = Field(default=90, gt=0, le=100)
overload_for: int = Field(default=60, gt=0)
disk_low_pct: float = Field(default=10, gt=0, lt=100)
disk_path: Path = Path("/data")
dependency_down_for: int = Field(default=300, gt=0)
repeat_window: int = Field(default=600, ge=0)
log_tail: int = Field(default=30, ge=0, le=200)
def load_alerts_config(config_dir: Path) -> AlertsConfig:
with (config_dir / "alerts.yaml").open(encoding="utf-8") as f:
return AlertsConfig.model_validate(yaml.safe_load(f))
# Conditions the monitor owns: it may resolve these when they stop holding. Others (agent attack, CD) are events.
MONITORED = frozenset(
{
AlertKind.CRASH,
AlertKind.OOM,
AlertKind.RESTART_LOOP,
AlertKind.UNHEALTHY,
AlertKind.OVERLOAD,
AlertKind.DISK_LOW,
AlertKind.DEPENDENCY_DOWN,
}
)
@dataclass(frozen=True)
class Condition:
kind: AlertKind
service: str | None
payload: dict[str, Any] = field(default_factory=dict)
container_id: str | None = None # for the log tail and the «📜 Логи» / «🩺 Статус» buttons
@property
def key(self) -> tuple[AlertKind, str | None]:
return (self.kind, self.service)
def container_conditions(
o: Observation, history: list[ContainerSnapshot], cfg: AlertsConfig, now: datetime
) -> list[Condition]:
"""history: this container's snapshots, oldest first, covering at least the longest window."""
d = o.details
found: list[Condition] = []
def add(kind: AlertKind, **payload: Any) -> None:
found.append(Condition(kind, d.name, payload, d.id))
if d.state in ("exited", "dead") and d.oom_killed:
add(AlertKind.OOM, exit_code=d.exit_code)
elif d.state in ("exited", "dead") and d.exit_code != 0:
add(AlertKind.CRASH, exit_code=d.exit_code)
window = [s for s in history if as_utc(s.at) >= now - timedelta(seconds=cfg.restart_loop_window)]
if window:
grown = d.restart_count - window[0].restart_count
if grown >= cfg.restart_loop_count:
add(AlertKind.RESTART_LOOP, restarts=grown, window_min=cfg.restart_loop_window // 60)
if d.state == "running" and d.health == "unhealthy":
add(AlertKind.UNHEALTHY)
if o.stats is not None:
recent = [s for s in history if as_utc(s.at) >= now - timedelta(seconds=cfg.overload_for)]
# held long enough: the samples span the whole period and every one of them (and the current) is over
spans = bool(recent) and (now - as_utc(recent[0].at)) >= timedelta(seconds=cfg.overload_for * 0.75)
cpu_over = o.stats.cpu_pct > cfg.overload_cpu_pct and all(s.cpu_pct > cfg.overload_cpu_pct for s in recent)
mem_over = o.stats.mem_pct > cfg.overload_mem_pct and all(s.mem_pct > cfg.overload_mem_pct for s in recent)
if spans and (cpu_over or mem_over):
add(AlertKind.OVERLOAD, cpu_pct=o.stats.cpu_pct, mem_pct=round(o.stats.mem_pct, 1))
return found
def disk_condition(free_pct: float | None, cfg: AlertsConfig) -> list[Condition]:
if free_pct is None or free_pct >= cfg.disk_low_pct:
return []
return [Condition(AlertKind.DISK_LOW, None, {"free_pct": round(free_pct, 1)})]
def docker_condition(down_since: datetime | None, cfg: AlertsConfig, now: datetime) -> list[Condition]:
if down_since is None or now - down_since < timedelta(seconds=cfg.dependency_down_for):
return []
return [Condition(AlertKind.DEPENDENCY_DOWN, "docker", {"since": down_since.isoformat()})]