tgit/app/alerts/detectors.py

"""Pure functions: one round of observations (+ snapshot history) → the conditions that hold right now.

The alert service compares them with open alerts: a new condition opens an alert, a vanished one resolves it.
Being state-based, a detector can't flap on a missed event, and a crash loop stays one alert with a counter.
"""

from dataclasses import dataclass, field
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any

import yaml
from pydantic import BaseModel, ConfigDict, Field

from app.db.models import AlertKind, ContainerSnapshot, as_utc
from app.runtime.snapshots import Observation


class AlertsConfig(BaseModel):
    model_config = ConfigDict(extra="forbid", frozen=True)

    restart_loop_count: int = Field(default=3, ge=1)
    restart_loop_window: int = Field(default=600, gt=0)
    overload_cpu_pct: float = Field(default=80, gt=0)
    overload_mem_pct: float = Field(default=90, gt=0, le=100)
    overload_for: int = Field(default=60, gt=0)
    disk_low_pct: float = Field(default=10, gt=0, lt=100)
    disk_path: Path = Path("/data")
    dependency_down_for: int = Field(default=300, gt=0)
    repeat_window: int = Field(default=600, ge=0)
    resolve_after: int = Field(default=60, ge=0)
    log_tail: int = Field(default=30, ge=0, le=200)


def load_alerts_config(config_dir: Path) -> AlertsConfig:
    with (config_dir / "alerts.yaml").open(encoding="utf-8") as f:
        return AlertsConfig.model_validate(yaml.safe_load(f))


# Conditions the monitor owns: it may resolve these when they stop holding. Others (agent attack, CD) are events.
MONITORED = frozenset(
    {
        AlertKind.CRASH,
        AlertKind.OOM,
        AlertKind.RESTART_LOOP,
        AlertKind.UNHEALTHY,
        AlertKind.OVERLOAD,
        AlertKind.DISK_LOW,
        AlertKind.DEPENDENCY_DOWN,
    }
)


@dataclass(frozen=True)
class Condition:
    kind: AlertKind
    service: str | None
    payload: dict[str, Any] = field(default_factory=dict)
    container_id: str | None = None  # for the log tail and the «📜 Логи» / «🩺 Статус» buttons
    bump: bool = False  # a fresh incident this round: if the alert is still open, its counter grows

    @property
    def key(self) -> tuple[AlertKind, str | None]:
        return (self.kind, self.service)


def container_conditions(
    o: Observation, history: list[ContainerSnapshot], cfg: AlertsConfig, now: datetime
) -> list[Condition]:
    """history: this container's snapshots, oldest first, covering at least the longest window."""
    d = o.details
    found: list[Condition] = []

    def add(kind: AlertKind, bump: bool = False, **payload: Any) -> None:
        found.append(Condition(kind, d.name, payload, d.id, bump))

    # A restart policy brings a crashed container back within a second — between two rounds it was already
    # restarted, so a grown restart count is a crash too (the exit code of that run is gone by then).
    before = [s for s in history if o.at is None or as_utc(s.at) < o.at]  # not this round's own row
    restarted = bool(before) and d.restart_count > before[-1].restart_count
    if d.state in ("exited", "dead") and d.oom_killed:
        add(AlertKind.OOM, exit_code=d.exit_code)
    elif d.state in ("exited", "dead") and d.exit_code != 0:
        add(AlertKind.CRASH, exit_code=d.exit_code)
    elif restarted or d.state == "restarting":
        add(AlertKind.CRASH, bump=restarted, exit_code=d.exit_code or "≠ 0")

    window = [s for s in history if as_utc(s.at) >= now - timedelta(seconds=cfg.restart_loop_window)]
    if window:
        grown = d.restart_count - window[0].restart_count
        if grown >= cfg.restart_loop_count:
            add(AlertKind.RESTART_LOOP, restarts=grown, window_min=cfg.restart_loop_window // 60)

    if d.state == "running" and d.health == "unhealthy":
        add(AlertKind.UNHEALTHY)

    if o.stats is not None:
        recent = [s for s in history if as_utc(s.at) >= now - timedelta(seconds=cfg.overload_for)]
        # held long enough: the samples span the whole period and every one of them (and the current) is over
        spans = bool(recent) and (now - as_utc(recent[0].at)) >= timedelta(seconds=cfg.overload_for * 0.75)
        cpu_over = o.stats.cpu_pct > cfg.overload_cpu_pct and all(s.cpu_pct > cfg.overload_cpu_pct for s in recent)
        mem_over = o.stats.mem_pct > cfg.overload_mem_pct and all(s.mem_pct > cfg.overload_mem_pct for s in recent)
        if spans and (cpu_over or mem_over):
            add(AlertKind.OVERLOAD, cpu_pct=o.stats.cpu_pct, mem_pct=round(o.stats.mem_pct, 1))
    return found


def disk_condition(free_pct: float | None, cfg: AlertsConfig) -> list[Condition]:
    if free_pct is None or free_pct >= cfg.disk_low_pct:
        return []
    return [Condition(AlertKind.DISK_LOW, None, {"free_pct": round(free_pct, 1)})]


def docker_condition(down_since: datetime | None, cfg: AlertsConfig, now: datetime) -> list[Condition]:
    if down_since is None or now - down_since < timedelta(seconds=cfg.dependency_down_for):
        return []
    return [Condition(AlertKind.DEPENDENCY_DOWN, "docker", {"since": down_since.isoformat()})]