"""Pure functions: one round of observations (+ snapshot history) → the conditions that hold right now. The alert service compares them with open alerts: a new condition opens an alert, a vanished one resolves it. Being state-based, a detector can't flap on a missed event, and a crash loop stays one alert with a counter. """ from dataclasses import dataclass, field from datetime import datetime, timedelta from pathlib import Path from typing import Any import yaml from pydantic import BaseModel, ConfigDict, Field from app.db.models import AlertKind, ContainerSnapshot, as_utc from app.runtime.snapshots import Observation class AlertsConfig(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True) restart_loop_count: int = Field(default=3, ge=1) restart_loop_window: int = Field(default=600, gt=0) overload_cpu_pct: float = Field(default=80, gt=0) overload_mem_pct: float = Field(default=90, gt=0, le=100) overload_for: int = Field(default=60, gt=0) disk_low_pct: float = Field(default=10, gt=0, lt=100) disk_path: Path = Path("/data") dependency_down_for: int = Field(default=300, gt=0) repeat_window: int = Field(default=600, ge=0) log_tail: int = Field(default=30, ge=0, le=200) def load_alerts_config(config_dir: Path) -> AlertsConfig: with (config_dir / "alerts.yaml").open(encoding="utf-8") as f: return AlertsConfig.model_validate(yaml.safe_load(f)) # Conditions the monitor owns: it may resolve these when they stop holding. Others (agent attack, CD) are events. MONITORED = frozenset( { AlertKind.CRASH, AlertKind.OOM, AlertKind.RESTART_LOOP, AlertKind.UNHEALTHY, AlertKind.OVERLOAD, AlertKind.DISK_LOW, AlertKind.DEPENDENCY_DOWN, } ) @dataclass(frozen=True) class Condition: kind: AlertKind service: str | None payload: dict[str, Any] = field(default_factory=dict) container_id: str | None = None # for the log tail and the «📜 Логи» / «🩺 Статус» buttons @property def key(self) -> tuple[AlertKind, str | None]: return (self.kind, self.service) def container_conditions( o: Observation, history: list[ContainerSnapshot], cfg: AlertsConfig, now: datetime ) -> list[Condition]: """history: this container's snapshots, oldest first, covering at least the longest window.""" d = o.details found: list[Condition] = [] def add(kind: AlertKind, **payload: Any) -> None: found.append(Condition(kind, d.name, payload, d.id)) if d.state in ("exited", "dead") and d.oom_killed: add(AlertKind.OOM, exit_code=d.exit_code) elif d.state in ("exited", "dead") and d.exit_code != 0: add(AlertKind.CRASH, exit_code=d.exit_code) window = [s for s in history if as_utc(s.at) >= now - timedelta(seconds=cfg.restart_loop_window)] if window: grown = d.restart_count - window[0].restart_count if grown >= cfg.restart_loop_count: add(AlertKind.RESTART_LOOP, restarts=grown, window_min=cfg.restart_loop_window // 60) if d.state == "running" and d.health == "unhealthy": add(AlertKind.UNHEALTHY) if o.stats is not None: recent = [s for s in history if as_utc(s.at) >= now - timedelta(seconds=cfg.overload_for)] # held long enough: the samples span the whole period and every one of them (and the current) is over spans = bool(recent) and (now - as_utc(recent[0].at)) >= timedelta(seconds=cfg.overload_for * 0.75) cpu_over = o.stats.cpu_pct > cfg.overload_cpu_pct and all(s.cpu_pct > cfg.overload_cpu_pct for s in recent) mem_over = o.stats.mem_pct > cfg.overload_mem_pct and all(s.mem_pct > cfg.overload_mem_pct for s in recent) if spans and (cpu_over or mem_over): add(AlertKind.OVERLOAD, cpu_pct=o.stats.cpu_pct, mem_pct=round(o.stats.mem_pct, 1)) return found def disk_condition(free_pct: float | None, cfg: AlertsConfig) -> list[Condition]: if free_pct is None or free_pct >= cfg.disk_low_pct: return [] return [Condition(AlertKind.DISK_LOW, None, {"free_pct": round(free_pct, 1)})] def docker_condition(down_since: datetime | None, cfg: AlertsConfig, now: datetime) -> list[Condition]: if down_since is None or now - down_since < timedelta(seconds=cfg.dependency_down_for): return [] return [Condition(AlertKind.DEPENDENCY_DOWN, "docker", {"since": down_since.isoformat()})]