feat(alerts): алерты всей команде — детекторы, дедупликация, список (#15)

- детекторы по состоянию после каждого раунда снапшотов: краш, OOM,
  цикл рестартов, unhealthy, перегрузка ≥ 1 мин, мало места, Docker
  недоступен; пороги в config/alerts.yaml
- AlertService: ключ kind+service, повтор в 10 мин — счётчик в исходном
  сообщении, «✅ восстановлено» ответом; разовые события (🛡) со счётчиком
- рассылка всем активным (🛡 — только админам), темп ~25/с, RetryAfter,
  заблокировавший бота пропускается; alert_deliveries для правок и ответов
- «🚨 Алерты»: активные + 7 дней по 10, карточка; кнопки «📜 Логи»,
  «🩺 Статус» под алертом открывают новое сообщение, а не затирают алерт
- 🛡 атаки через агента идут через общий сервис алертов
- убраны заглушки «Раздел в разработке» — у всех разделов свои роутеры

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/alerts/__init__.py

@@ -0,0 +1 @@
+"""Alerts: detectors over container observations, dedup with counters, broadcast to the team (ARCHITECTURE §2.9)."""

app/alerts/broadcast.py

@@ -0,0 +1,110 @@
+"""Alerts → Telegram: one message per member (🛡 — admins only), edited on repeats, answered on recovery.
+
+Sends are paced under Telegram's ~30 msg/s; a member who blocked the bot is skipped, not an error (AL-1, AL-3…AL-5).
+"""
+
+import asyncio
+import logging
+from typing import Any
+
+from aiogram import Bot
+from aiogram.enums import ParseMode
+from aiogram.exceptions import TelegramAPIError, TelegramBadRequest, TelegramForbiddenError, TelegramRetryAfter
+from aiogram.types import ReplyParameters
+from sqlalchemy import select
+
+from app.access.service import AccessService
+from app.bot.routers.alerts import keyboard, render, service, title
+from app.bot.routers.status import duration
+from app.db.models import Alert, AlertDelivery, Role, as_utc, utcnow
+from app.db.session import SessionMaker
+from app.texts import Texts
+
+log = logging.getLogger(__name__)
+
+PACE = 1 / 25  # seconds between sends
+
+
+class TelegramNotifier:
+    def __init__(self, bot: Bot, access: AccessService, sessions: SessionMaker, texts: Texts) -> None:
+        self._bot = bot
+        self._access = access
+        self._sessions = sessions
+        self._texts = texts
+
+    async def _call(self, coro_factory: Any) -> Any:
+        """One Telegram call; flood control is waited out once, a blocked chat is just skipped."""
+        for attempt in range(2):
+            try:
+                return await coro_factory()
+            except TelegramRetryAfter as e:
+                if attempt:
+                    raise
+                await asyncio.sleep(e.retry_after)
+            except TelegramForbiddenError:
+                return None
+        return None
+
+    async def opened(self, alert: Alert, admins_only: bool) -> None:
+        members = await self._access.list_members()
+        recipients = [m.tg_id for m in members if not admins_only or m.role is Role.ADMIN]
+        text = render(self._texts, alert)
+        kb = keyboard(self._texts, alert)
+        deliveries = []
+        for chat_id in recipients:
+            try:
+                msg = await self._call(
+                    lambda chat_id=chat_id: self._bot.send_message(
+                        chat_id, text, parse_mode=ParseMode.HTML, reply_markup=kb
+                    )
+                )
+            except TelegramAPIError as e:
+                log.warning("alert %s to %s: %s", alert.id, chat_id, e)
+                continue
+            if msg is not None:
+                deliveries.append(AlertDelivery(alert_id=alert.id, chat_id=chat_id, message_id=msg.message_id))
+            await asyncio.sleep(PACE)
+        async with self._sessions.begin() as session:
+            session.add_all(deliveries)
+
+    async def _deliveries(self, alert: Alert) -> list[AlertDelivery]:
+        async with self._sessions() as session:
+            return list(await session.scalars(select(AlertDelivery).where(AlertDelivery.alert_id == alert.id)))
+
+    async def repeated(self, alert: Alert) -> None:
+        text = render(self._texts, alert)
+        kb = keyboard(self._texts, alert)
+        for d in await self._deliveries(alert):
+            try:
+                await self._call(
+                    lambda d=d: self._bot.edit_message_text(
+                        text, chat_id=d.chat_id, message_id=d.message_id, parse_mode=ParseMode.HTML, reply_markup=kb
+                    )
+                )
+            except TelegramBadRequest as e:  # deleted by the user, too old, not modified
+                log.info("alert %s edit in %s: %s", alert.id, d.chat_id, e)
+            except TelegramAPIError as e:
+                log.warning("alert %s edit in %s: %s", alert.id, d.chat_id, e)
+            await asyncio.sleep(PACE)
+
+    async def resolved(self, alert: Alert) -> None:
+        t = self._texts.alerts
+        took = (as_utc(alert.resolved_at or utcnow()) - as_utc(alert.opened_at)).total_seconds()
+        text = t.resolved.format(
+            title=title(self._texts, alert),
+            service=service(self._texts, alert),
+            duration=duration(self._texts.status, took),
+        )
+        for d in await self._deliveries(alert):
+            try:
+                await self._call(
+                    lambda d=d: self._bot.send_message(
+                        d.chat_id,
+                        text,
+                        parse_mode=ParseMode.HTML,
+                        reply_parameters=ReplyParameters(message_id=d.message_id, allow_sending_without_reply=True),
+                    )
+                )
+            except TelegramAPIError as e:
+                log.warning("alert %s recovery to %s: %s", alert.id, d.chat_id, e)
+            await asyncio.sleep(PACE)

app/alerts/detectors.py

@@ -0,0 +1,109 @@
+"""Pure functions: one round of observations (+ snapshot history) → the conditions that hold right now.
+
+The alert service compares them with open alerts: a new condition opens an alert, a vanished one resolves it.
+Being state-based, a detector can't flap on a missed event, and a crash loop stays one alert with a counter.
+"""
+
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field
+
+from app.db.models import AlertKind, ContainerSnapshot, as_utc
+from app.runtime.snapshots import Observation
+
+
+class AlertsConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    restart_loop_count: int = Field(default=3, ge=1)
+    restart_loop_window: int = Field(default=600, gt=0)
+    overload_cpu_pct: float = Field(default=80, gt=0)
+    overload_mem_pct: float = Field(default=90, gt=0, le=100)
+    overload_for: int = Field(default=60, gt=0)
+    disk_low_pct: float = Field(default=10, gt=0, lt=100)
+    disk_path: Path = Path("/data")
+    dependency_down_for: int = Field(default=300, gt=0)
+    repeat_window: int = Field(default=600, ge=0)
+    log_tail: int = Field(default=30, ge=0, le=200)
+
+
+def load_alerts_config(config_dir: Path) -> AlertsConfig:
+    with (config_dir / "alerts.yaml").open(encoding="utf-8") as f:
+        return AlertsConfig.model_validate(yaml.safe_load(f))
+
+
+# Conditions the monitor owns: it may resolve these when they stop holding. Others (agent attack, CD) are events.
+MONITORED = frozenset(
+    {
+        AlertKind.CRASH,
+        AlertKind.OOM,
+        AlertKind.RESTART_LOOP,
+        AlertKind.UNHEALTHY,
+        AlertKind.OVERLOAD,
+        AlertKind.DISK_LOW,
+        AlertKind.DEPENDENCY_DOWN,
+    }
+)
+
+
+@dataclass(frozen=True)
+class Condition:
+    kind: AlertKind
+    service: str | None
+    payload: dict[str, Any] = field(default_factory=dict)
+    container_id: str | None = None  # for the log tail and the «📜 Логи» / «🩺 Статус» buttons
+
+    @property
+    def key(self) -> tuple[AlertKind, str | None]:
+        return (self.kind, self.service)
+
+
+def container_conditions(
+    o: Observation, history: list[ContainerSnapshot], cfg: AlertsConfig, now: datetime
+) -> list[Condition]:
+    """history: this container's snapshots, oldest first, covering at least the longest window."""
+    d = o.details
+    found: list[Condition] = []
+
+    def add(kind: AlertKind, **payload: Any) -> None:
+        found.append(Condition(kind, d.name, payload, d.id))
+
+    if d.state in ("exited", "dead") and d.oom_killed:
+        add(AlertKind.OOM, exit_code=d.exit_code)
+    elif d.state in ("exited", "dead") and d.exit_code != 0:
+        add(AlertKind.CRASH, exit_code=d.exit_code)
+
+    window = [s for s in history if as_utc(s.at) >= now - timedelta(seconds=cfg.restart_loop_window)]
+    if window:
+        grown = d.restart_count - window[0].restart_count
+        if grown >= cfg.restart_loop_count:
+            add(AlertKind.RESTART_LOOP, restarts=grown, window_min=cfg.restart_loop_window // 60)
+
+    if d.state == "running" and d.health == "unhealthy":
+        add(AlertKind.UNHEALTHY)
+
+    if o.stats is not None:
+        recent = [s for s in history if as_utc(s.at) >= now - timedelta(seconds=cfg.overload_for)]
+        # held long enough: the samples span the whole period and every one of them (and the current) is over
+        spans = bool(recent) and (now - as_utc(recent[0].at)) >= timedelta(seconds=cfg.overload_for * 0.75)
+        cpu_over = o.stats.cpu_pct > cfg.overload_cpu_pct and all(s.cpu_pct > cfg.overload_cpu_pct for s in recent)
+        mem_over = o.stats.mem_pct > cfg.overload_mem_pct and all(s.mem_pct > cfg.overload_mem_pct for s in recent)
+        if spans and (cpu_over or mem_over):
+            add(AlertKind.OVERLOAD, cpu_pct=o.stats.cpu_pct, mem_pct=round(o.stats.mem_pct, 1))
+    return found
+
+
+def disk_condition(free_pct: float | None, cfg: AlertsConfig) -> list[Condition]:
+    if free_pct is None or free_pct >= cfg.disk_low_pct:
+        return []
+    return [Condition(AlertKind.DISK_LOW, None, {"free_pct": round(free_pct, 1)})]
+
+
+def docker_condition(down_since: datetime | None, cfg: AlertsConfig, now: datetime) -> list[Condition]:
+    if down_since is None or now - down_since < timedelta(seconds=cfg.dependency_down_for):
+        return []
+    return [Condition(AlertKind.DEPENDENCY_DOWN, "docker", {"since": down_since.isoformat()})]

app/alerts/monitor.py

@@ -0,0 +1,64 @@
+"""Glue: after each snapshot round run the detectors and sync open alerts (hooked into `snapshot_loop`)."""
+
+import os
+from collections.abc import Callable
+from datetime import datetime, timedelta
+from pathlib import Path
+
+from app.alerts.detectors import (
+    MONITORED,
+    AlertsConfig,
+    Condition,
+    container_conditions,
+    disk_condition,
+    docker_condition,
+)
+from app.alerts.service import AlertService
+from app.db.models import AlertKind, utcnow
+from app.runtime.snapshots import Observation, SnapshotStore
+
+WITHOUT_DOCKER = frozenset({AlertKind.DEPENDENCY_DOWN, AlertKind.DISK_LOW})
+
+
+def free_disk_pct(path: Path) -> float | None:
+    try:
+        st = os.statvfs(path)
+    except OSError:
+        return None
+    return 100.0 * st.f_bavail / st.f_blocks if st.f_blocks else None
+
+
+class AlertMonitor:
+    def __init__(
+        self,
+        alerts: AlertService,
+        store: SnapshotStore,
+        cfg: AlertsConfig,
+        disk: Callable[[Path], float | None] = free_disk_pct,
+        clock: Callable[[], datetime] = utcnow,
+    ) -> None:
+        self._alerts = alerts
+        self._store = store
+        self._cfg = cfg
+        self._disk = disk
+        self._clock = clock
+        self._docker_down_since: datetime | None = None
+
+    @property
+    def _history(self) -> timedelta:
+        return timedelta(seconds=max(self._cfg.restart_loop_window, self._cfg.overload_for) + 60)
+
+    async def on_round(self, observations: list[Observation] | None) -> None:
+        now = self._clock()
+        conditions: list[Condition] = disk_condition(self._disk(self._cfg.disk_path), self._cfg)
+        if observations is None:
+            # Docker is unreachable: container alerts can't be judged — leave them as they are
+            self._docker_down_since = self._docker_down_since or now
+            conditions += docker_condition(self._docker_down_since, self._cfg, now)
+            await self._alerts.sync(conditions, WITHOUT_DOCKER, now)
+            return
+        self._docker_down_since = None
+        for o in observations:
+            history = await self._store.since(o.details.name, now - self._history)
+            conditions += container_conditions(o, history, self._cfg, now)
+        await self._alerts.sync(conditions, MONITORED, now)

app/alerts/service.py

@@ -0,0 +1,166 @@
+"""Open / repeat / resolve alerts (SPEC AL-4, AL-5). Messages go through a `Notifier` (the Telegram broadcast)."""
+
+import logging
+from collections.abc import Awaitable, Callable, Iterable
+from datetime import datetime, timedelta
+from typing import Protocol
+
+from sqlalchemy import or_, select
+from sqlalchemy.orm import selectinload
+
+from app.alerts.detectors import AlertsConfig, Condition
+from app.db.models import Alert, AlertKind, as_utc, utcnow
+from app.db.session import SessionMaker
+
+log = logging.getLogger(__name__)
+
+HISTORY = timedelta(days=7)  # AL-6
+LOG_CHARS = 3000
+
+LogSource = Callable[[str, int], Awaitable[str]]  # (container id, lines) -> tail
+
+
+class Notifier(Protocol):
+    async def opened(self, alert: Alert, admins_only: bool) -> None: ...
+
+    async def repeated(self, alert: Alert) -> None: ...
+
+    async def resolved(self, alert: Alert) -> None: ...
+
+
+class AlertService:
+    def __init__(
+        self, sessions: SessionMaker, notifier: Notifier, cfg: AlertsConfig, log_source: LogSource | None = None
+    ) -> None:
+        self._sessions = sessions
+        self._notifier = notifier
+        self._cfg = cfg
+        self._log_source = log_source
+
+    async def _tail(self, c: Condition) -> str:
+        if self._log_source is None or c.container_id is None or not self._cfg.log_tail:
+            return ""
+        try:
+            text = await self._log_source(c.container_id, self._cfg.log_tail)
+        except Exception as e:  # an alert without a log tail is still an alert
+            log.warning("alert log tail for %s: %s", c.service, e)
+            return ""
+        return text[-LOG_CHARS:]
+
+    async def sync(
+        self, conditions: Iterable[Condition], scope: frozenset[AlertKind], now: datetime | None = None
+    ) -> None:
+        """Make open alerts of `scope` kinds match `conditions` exactly: open the new, resolve the gone."""
+        now = now or utcnow()
+        wanted = {c.key: c for c in conditions if c.kind in scope}
+        async with self._sessions.begin() as session:
+            rows = await session.scalars(
+                select(Alert)
+                .options(selectinload(Alert.deliveries))
+                .where(Alert.kind.in_(scope), or_(Alert.resolved_at.is_(None), Alert.last_at >= now - self._window))
+            )
+            by_key: dict[tuple[AlertKind, str | None], Alert] = {}
+            for a in rows:  # the open one wins over a recently resolved one with the same key
+                if (a.kind, a.service) not in by_key or a.resolved_at is None:
+                    by_key[(a.kind, a.service)] = a
+            opened, repeated, resolved = [], [], []
+            for key, c in wanted.items():
+                known = by_key.get(key)
+                if known is not None and known.resolved_at is None:
+                    known.last_at = now
+                    continue
+                if (
+                    known is not None
+                    and known.resolved_at is not None
+                    and now - as_utc(known.resolved_at) <= self._window
+                ):
+                    known.resolved_at, known.last_at, known.count = None, now, known.count + 1  # back within the window
+                    known.payload = {**known.payload, **c.payload}
+                    repeated.append(known)
+                    continue
+                fresh = Alert(
+                    kind=c.kind, service=c.service, payload=await self._payload(c), opened_at=now, last_at=now
+                )
+                session.add(fresh)
+                opened.append(fresh)
+            for key, a in by_key.items():
+                if a.resolved_at is None and key not in wanted:
+                    a.resolved_at = now
+                    resolved.append(a)
+            await session.flush()
+            for a in opened:
+                await session.refresh(a, ["deliveries"])
+        for a in opened:
+            await self._notifier.opened(a, admins_only=False)
+        for a in repeated:
+            await self._notifier.repeated(a)
+        for a in resolved:
+            await self._notifier.resolved(a)
+
+    async def event(self, c: Condition, admins_only: bool = False, now: datetime | None = None) -> Alert:
+        """A one-shot alert (🛡 attack, CD failure): opened already resolved; repeats bump the counter (AL-5)."""
+        now = now or utcnow()
+        async with self._sessions.begin() as session:
+            recent = await session.scalar(
+                select(Alert)
+                .options(selectinload(Alert.deliveries))
+                .where(Alert.kind == c.kind, Alert.service == c.service, Alert.last_at >= now - self._window)
+                .order_by(Alert.last_at.desc())
+                .limit(1)
+            )
+            if recent is not None:
+                recent.count += 1
+                recent.last_at = now
+                recent.payload = {**recent.payload, **c.payload}
+                alert, new = recent, False
+            else:
+                alert = Alert(
+                    kind=c.kind, service=c.service, payload=dict(c.payload), opened_at=now, last_at=now, resolved_at=now
+                )
+                session.add(alert)
+                await session.flush()
+                await session.refresh(alert, ["deliveries"])
+                new = True
+        if new:
+            await self._notifier.opened(alert, admins_only=admins_only)
+        else:
+            await self._notifier.repeated(alert)
+        return alert
+
+    async def _payload(self, c: Condition) -> dict[str, object]:
+        payload: dict[str, object] = dict(c.payload)
+        if c.container_id:
+            payload["container_id"] = c.container_id
+        tail = await self._tail(c)
+        if tail:
+            payload["log"] = tail
+        return payload
+
+    @property
+    def _window(self) -> timedelta:
+        return timedelta(seconds=self._cfg.repeat_window)
+
+    async def page(self, page: int, per_page: int = 10, now: datetime | None = None) -> tuple[list[Alert], int]:
+        """Active first, then the last 7 days, newest first (AL-6). Admin-only kinds are left out."""
+        now = now or utcnow()
+        async with self._sessions() as session:
+            rows = list(
+                await session.scalars(
+                    select(Alert)
+                    .where(Alert.kind != AlertKind.AGENT_ATTACK)
+                    .where(or_(Alert.resolved_at.is_(None), Alert.last_at >= now - HISTORY))
+                )
+            )
+        rows.sort(key=lambda a: (a.resolved_at is not None, -as_utc(a.last_at).timestamp()))
+        return rows[page * per_page : (page + 1) * per_page], len(rows)
+
+    async def get(self, alert_id: int) -> Alert | None:
+        async with self._sessions() as session:
+            return await session.get(Alert, alert_id)
+
+    async def active_count(self) -> int:
+        async with self._sessions() as session:
+            rows = await session.scalars(
+                select(Alert.id).where(Alert.resolved_at.is_(None), Alert.kind != AlertKind.AGENT_ATTACK)
+            )
+            return len(list(rows))