feat(alerts): алерты всей команде — детекторы, дедупликация, список (#15)

- детекторы по состоянию после каждого раунда снапшотов: краш, OOM,
  цикл рестартов, unhealthy, перегрузка ≥ 1 мин, мало места, Docker
  недоступен; пороги в config/alerts.yaml
- AlertService: ключ kind+service, повтор в 10 мин — счётчик в исходном
  сообщении, «✅ восстановлено» ответом; разовые события (🛡) со счётчиком
- рассылка всем активным (🛡 — только админам), темп ~25/с, RetryAfter,
  заблокировавший бота пропускается; alert_deliveries для правок и ответов
- «🚨 Алерты»: активные + 7 дней по 10, карточка; кнопки «📜 Логи»,
  «🩺 Статус» под алертом открывают новое сообщение, а не затирают алерт
- 🛡 атаки через агента идут через общий сервис алертов
- убраны заглушки «Раздел в разработке» — у всех разделов свои роутеры

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/alerts/__init__.py

@@ -0,0 +1 @@
+"""Alerts: detectors over container observations, dedup with counters, broadcast to the team (ARCHITECTURE §2.9)."""

app/alerts/broadcast.py

@@ -0,0 +1,110 @@
+"""Alerts → Telegram: one message per member (🛡 — admins only), edited on repeats, answered on recovery.
+
+Sends are paced under Telegram's ~30 msg/s; a member who blocked the bot is skipped, not an error (AL-1, AL-3…AL-5).
+"""
+
+import asyncio
+import logging
+from typing import Any
+
+from aiogram import Bot
+from aiogram.enums import ParseMode
+from aiogram.exceptions import TelegramAPIError, TelegramBadRequest, TelegramForbiddenError, TelegramRetryAfter
+from aiogram.types import ReplyParameters
+from sqlalchemy import select
+
+from app.access.service import AccessService
+from app.bot.routers.alerts import keyboard, render, service, title
+from app.bot.routers.status import duration
+from app.db.models import Alert, AlertDelivery, Role, as_utc, utcnow
+from app.db.session import SessionMaker
+from app.texts import Texts
+
+log = logging.getLogger(__name__)
+
+PACE = 1 / 25  # seconds between sends
+
+
+class TelegramNotifier:
+    def __init__(self, bot: Bot, access: AccessService, sessions: SessionMaker, texts: Texts) -> None:
+        self._bot = bot
+        self._access = access
+        self._sessions = sessions
+        self._texts = texts
+
+    async def _call(self, coro_factory: Any) -> Any:
+        """One Telegram call; flood control is waited out once, a blocked chat is just skipped."""
+        for attempt in range(2):
+            try:
+                return await coro_factory()
+            except TelegramRetryAfter as e:
+                if attempt:
+                    raise
+                await asyncio.sleep(e.retry_after)
+            except TelegramForbiddenError:
+                return None
+        return None
+
+    async def opened(self, alert: Alert, admins_only: bool) -> None:
+        members = await self._access.list_members()
+        recipients = [m.tg_id for m in members if not admins_only or m.role is Role.ADMIN]
+        text = render(self._texts, alert)
+        kb = keyboard(self._texts, alert)
+        deliveries = []
+        for chat_id in recipients:
+            try:
+                msg = await self._call(
+                    lambda chat_id=chat_id: self._bot.send_message(
+                        chat_id, text, parse_mode=ParseMode.HTML, reply_markup=kb
+                    )
+                )
+            except TelegramAPIError as e:
+                log.warning("alert %s to %s: %s", alert.id, chat_id, e)
+                continue
+            if msg is not None:
+                deliveries.append(AlertDelivery(alert_id=alert.id, chat_id=chat_id, message_id=msg.message_id))
+            await asyncio.sleep(PACE)
+        async with self._sessions.begin() as session:
+            session.add_all(deliveries)
+
+    async def _deliveries(self, alert: Alert) -> list[AlertDelivery]:
+        async with self._sessions() as session:
+            return list(await session.scalars(select(AlertDelivery).where(AlertDelivery.alert_id == alert.id)))
+
+    async def repeated(self, alert: Alert) -> None:
+        text = render(self._texts, alert)
+        kb = keyboard(self._texts, alert)
+        for d in await self._deliveries(alert):
+            try:
+                await self._call(
+                    lambda d=d: self._bot.edit_message_text(
+                        text, chat_id=d.chat_id, message_id=d.message_id, parse_mode=ParseMode.HTML, reply_markup=kb
+                    )
+                )
+            except TelegramBadRequest as e:  # deleted by the user, too old, not modified
+                log.info("alert %s edit in %s: %s", alert.id, d.chat_id, e)
+            except TelegramAPIError as e:
+                log.warning("alert %s edit in %s: %s", alert.id, d.chat_id, e)
+            await asyncio.sleep(PACE)
+
+    async def resolved(self, alert: Alert) -> None:
+        t = self._texts.alerts
+        took = (as_utc(alert.resolved_at or utcnow()) - as_utc(alert.opened_at)).total_seconds()
+        text = t.resolved.format(
+            title=title(self._texts, alert),
+            service=service(self._texts, alert),
+            duration=duration(self._texts.status, took),
+        )
+        for d in await self._deliveries(alert):
+            try:
+                await self._call(
+                    lambda d=d: self._bot.send_message(
+                        d.chat_id,
+                        text,
+                        parse_mode=ParseMode.HTML,
+                        reply_parameters=ReplyParameters(message_id=d.message_id, allow_sending_without_reply=True),
+                    )
+                )
+            except TelegramAPIError as e:
+                log.warning("alert %s recovery to %s: %s", alert.id, d.chat_id, e)
+            await asyncio.sleep(PACE)

app/alerts/detectors.py

@@ -0,0 +1,109 @@
+"""Pure functions: one round of observations (+ snapshot history) → the conditions that hold right now.
+
+The alert service compares them with open alerts: a new condition opens an alert, a vanished one resolves it.
+Being state-based, a detector can't flap on a missed event, and a crash loop stays one alert with a counter.
+"""
+
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field
+
+from app.db.models import AlertKind, ContainerSnapshot, as_utc
+from app.runtime.snapshots import Observation
+
+
+class AlertsConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    restart_loop_count: int = Field(default=3, ge=1)
+    restart_loop_window: int = Field(default=600, gt=0)
+    overload_cpu_pct: float = Field(default=80, gt=0)
+    overload_mem_pct: float = Field(default=90, gt=0, le=100)
+    overload_for: int = Field(default=60, gt=0)
+    disk_low_pct: float = Field(default=10, gt=0, lt=100)
+    disk_path: Path = Path("/data")
+    dependency_down_for: int = Field(default=300, gt=0)
+    repeat_window: int = Field(default=600, ge=0)
+    log_tail: int = Field(default=30, ge=0, le=200)
+
+
+def load_alerts_config(config_dir: Path) -> AlertsConfig:
+    with (config_dir / "alerts.yaml").open(encoding="utf-8") as f:
+        return AlertsConfig.model_validate(yaml.safe_load(f))
+
+
+# Conditions the monitor owns: it may resolve these when they stop holding. Others (agent attack, CD) are events.
+MONITORED = frozenset(
+    {
+        AlertKind.CRASH,
+        AlertKind.OOM,
+        AlertKind.RESTART_LOOP,
+        AlertKind.UNHEALTHY,
+        AlertKind.OVERLOAD,
+        AlertKind.DISK_LOW,
+        AlertKind.DEPENDENCY_DOWN,
+    }
+)
+
+
+@dataclass(frozen=True)
+class Condition:
+    kind: AlertKind
+    service: str | None
+    payload: dict[str, Any] = field(default_factory=dict)
+    container_id: str | None = None  # for the log tail and the «📜 Логи» / «🩺 Статус» buttons
+
+    @property
+    def key(self) -> tuple[AlertKind, str | None]:
+        return (self.kind, self.service)
+
+
+def container_conditions(
+    o: Observation, history: list[ContainerSnapshot], cfg: AlertsConfig, now: datetime
+) -> list[Condition]:
+    """history: this container's snapshots, oldest first, covering at least the longest window."""
+    d = o.details
+    found: list[Condition] = []
+
+    def add(kind: AlertKind, **payload: Any) -> None:
+        found.append(Condition(kind, d.name, payload, d.id))
+
+    if d.state in ("exited", "dead") and d.oom_killed:
+        add(AlertKind.OOM, exit_code=d.exit_code)
+    elif d.state in ("exited", "dead") and d.exit_code != 0:
+        add(AlertKind.CRASH, exit_code=d.exit_code)
+
+    window = [s for s in history if as_utc(s.at) >= now - timedelta(seconds=cfg.restart_loop_window)]
+    if window:
+        grown = d.restart_count - window[0].restart_count
+        if grown >= cfg.restart_loop_count:
+            add(AlertKind.RESTART_LOOP, restarts=grown, window_min=cfg.restart_loop_window // 60)
+
+    if d.state == "running" and d.health == "unhealthy":
+        add(AlertKind.UNHEALTHY)
+
+    if o.stats is not None:
+        recent = [s for s in history if as_utc(s.at) >= now - timedelta(seconds=cfg.overload_for)]
+        # held long enough: the samples span the whole period and every one of them (and the current) is over
+        spans = bool(recent) and (now - as_utc(recent[0].at)) >= timedelta(seconds=cfg.overload_for * 0.75)
+        cpu_over = o.stats.cpu_pct > cfg.overload_cpu_pct and all(s.cpu_pct > cfg.overload_cpu_pct for s in recent)
+        mem_over = o.stats.mem_pct > cfg.overload_mem_pct and all(s.mem_pct > cfg.overload_mem_pct for s in recent)
+        if spans and (cpu_over or mem_over):
+            add(AlertKind.OVERLOAD, cpu_pct=o.stats.cpu_pct, mem_pct=round(o.stats.mem_pct, 1))
+    return found
+
+
+def disk_condition(free_pct: float | None, cfg: AlertsConfig) -> list[Condition]:
+    if free_pct is None or free_pct >= cfg.disk_low_pct:
+        return []
+    return [Condition(AlertKind.DISK_LOW, None, {"free_pct": round(free_pct, 1)})]
+
+
+def docker_condition(down_since: datetime | None, cfg: AlertsConfig, now: datetime) -> list[Condition]:
+    if down_since is None or now - down_since < timedelta(seconds=cfg.dependency_down_for):
+        return []
+    return [Condition(AlertKind.DEPENDENCY_DOWN, "docker", {"since": down_since.isoformat()})]

app/alerts/monitor.py

@@ -0,0 +1,64 @@
+"""Glue: after each snapshot round run the detectors and sync open alerts (hooked into `snapshot_loop`)."""
+
+import os
+from collections.abc import Callable
+from datetime import datetime, timedelta
+from pathlib import Path
+
+from app.alerts.detectors import (
+    MONITORED,
+    AlertsConfig,
+    Condition,
+    container_conditions,
+    disk_condition,
+    docker_condition,
+)
+from app.alerts.service import AlertService
+from app.db.models import AlertKind, utcnow
+from app.runtime.snapshots import Observation, SnapshotStore
+
+WITHOUT_DOCKER = frozenset({AlertKind.DEPENDENCY_DOWN, AlertKind.DISK_LOW})
+
+
+def free_disk_pct(path: Path) -> float | None:
+    try:
+        st = os.statvfs(path)
+    except OSError:
+        return None
+    return 100.0 * st.f_bavail / st.f_blocks if st.f_blocks else None
+
+
+class AlertMonitor:
+    def __init__(
+        self,
+        alerts: AlertService,
+        store: SnapshotStore,
+        cfg: AlertsConfig,
+        disk: Callable[[Path], float | None] = free_disk_pct,
+        clock: Callable[[], datetime] = utcnow,
+    ) -> None:
+        self._alerts = alerts
+        self._store = store
+        self._cfg = cfg
+        self._disk = disk
+        self._clock = clock
+        self._docker_down_since: datetime | None = None
+
+    @property
+    def _history(self) -> timedelta:
+        return timedelta(seconds=max(self._cfg.restart_loop_window, self._cfg.overload_for) + 60)
+
+    async def on_round(self, observations: list[Observation] | None) -> None:
+        now = self._clock()
+        conditions: list[Condition] = disk_condition(self._disk(self._cfg.disk_path), self._cfg)
+        if observations is None:
+            # Docker is unreachable: container alerts can't be judged — leave them as they are
+            self._docker_down_since = self._docker_down_since or now
+            conditions += docker_condition(self._docker_down_since, self._cfg, now)
+            await self._alerts.sync(conditions, WITHOUT_DOCKER, now)
+            return
+        self._docker_down_since = None
+        for o in observations:
+            history = await self._store.since(o.details.name, now - self._history)
+            conditions += container_conditions(o, history, self._cfg, now)
+        await self._alerts.sync(conditions, MONITORED, now)

app/alerts/service.py

@@ -0,0 +1,166 @@
+"""Open / repeat / resolve alerts (SPEC AL-4, AL-5). Messages go through a `Notifier` (the Telegram broadcast)."""
+
+import logging
+from collections.abc import Awaitable, Callable, Iterable
+from datetime import datetime, timedelta
+from typing import Protocol
+
+from sqlalchemy import or_, select
+from sqlalchemy.orm import selectinload
+
+from app.alerts.detectors import AlertsConfig, Condition
+from app.db.models import Alert, AlertKind, as_utc, utcnow
+from app.db.session import SessionMaker
+
+log = logging.getLogger(__name__)
+
+HISTORY = timedelta(days=7)  # AL-6
+LOG_CHARS = 3000
+
+LogSource = Callable[[str, int], Awaitable[str]]  # (container id, lines) -> tail
+
+
+class Notifier(Protocol):
+    async def opened(self, alert: Alert, admins_only: bool) -> None: ...
+
+    async def repeated(self, alert: Alert) -> None: ...
+
+    async def resolved(self, alert: Alert) -> None: ...
+
+
+class AlertService:
+    def __init__(
+        self, sessions: SessionMaker, notifier: Notifier, cfg: AlertsConfig, log_source: LogSource | None = None
+    ) -> None:
+        self._sessions = sessions
+        self._notifier = notifier
+        self._cfg = cfg
+        self._log_source = log_source
+
+    async def _tail(self, c: Condition) -> str:
+        if self._log_source is None or c.container_id is None or not self._cfg.log_tail:
+            return ""
+        try:
+            text = await self._log_source(c.container_id, self._cfg.log_tail)
+        except Exception as e:  # an alert without a log tail is still an alert
+            log.warning("alert log tail for %s: %s", c.service, e)
+            return ""
+        return text[-LOG_CHARS:]
+
+    async def sync(
+        self, conditions: Iterable[Condition], scope: frozenset[AlertKind], now: datetime | None = None
+    ) -> None:
+        """Make open alerts of `scope` kinds match `conditions` exactly: open the new, resolve the gone."""
+        now = now or utcnow()
+        wanted = {c.key: c for c in conditions if c.kind in scope}
+        async with self._sessions.begin() as session:
+            rows = await session.scalars(
+                select(Alert)
+                .options(selectinload(Alert.deliveries))
+                .where(Alert.kind.in_(scope), or_(Alert.resolved_at.is_(None), Alert.last_at >= now - self._window))
+            )
+            by_key: dict[tuple[AlertKind, str | None], Alert] = {}
+            for a in rows:  # the open one wins over a recently resolved one with the same key
+                if (a.kind, a.service) not in by_key or a.resolved_at is None:
+                    by_key[(a.kind, a.service)] = a
+            opened, repeated, resolved = [], [], []
+            for key, c in wanted.items():
+                known = by_key.get(key)
+                if known is not None and known.resolved_at is None:
+                    known.last_at = now
+                    continue
+                if (
+                    known is not None
+                    and known.resolved_at is not None
+                    and now - as_utc(known.resolved_at) <= self._window
+                ):
+                    known.resolved_at, known.last_at, known.count = None, now, known.count + 1  # back within the window
+                    known.payload = {**known.payload, **c.payload}
+                    repeated.append(known)
+                    continue
+                fresh = Alert(
+                    kind=c.kind, service=c.service, payload=await self._payload(c), opened_at=now, last_at=now
+                )
+                session.add(fresh)
+                opened.append(fresh)
+            for key, a in by_key.items():
+                if a.resolved_at is None and key not in wanted:
+                    a.resolved_at = now
+                    resolved.append(a)
+            await session.flush()
+            for a in opened:
+                await session.refresh(a, ["deliveries"])
+        for a in opened:
+            await self._notifier.opened(a, admins_only=False)
+        for a in repeated:
+            await self._notifier.repeated(a)
+        for a in resolved:
+            await self._notifier.resolved(a)
+
+    async def event(self, c: Condition, admins_only: bool = False, now: datetime | None = None) -> Alert:
+        """A one-shot alert (🛡 attack, CD failure): opened already resolved; repeats bump the counter (AL-5)."""
+        now = now or utcnow()
+        async with self._sessions.begin() as session:
+            recent = await session.scalar(
+                select(Alert)
+                .options(selectinload(Alert.deliveries))
+                .where(Alert.kind == c.kind, Alert.service == c.service, Alert.last_at >= now - self._window)
+                .order_by(Alert.last_at.desc())
+                .limit(1)
+            )
+            if recent is not None:
+                recent.count += 1
+                recent.last_at = now
+                recent.payload = {**recent.payload, **c.payload}
+                alert, new = recent, False
+            else:
+                alert = Alert(
+                    kind=c.kind, service=c.service, payload=dict(c.payload), opened_at=now, last_at=now, resolved_at=now
+                )
+                session.add(alert)
+                await session.flush()
+                await session.refresh(alert, ["deliveries"])
+                new = True
+        if new:
+            await self._notifier.opened(alert, admins_only=admins_only)
+        else:
+            await self._notifier.repeated(alert)
+        return alert
+
+    async def _payload(self, c: Condition) -> dict[str, object]:
+        payload: dict[str, object] = dict(c.payload)
+        if c.container_id:
+            payload["container_id"] = c.container_id
+        tail = await self._tail(c)
+        if tail:
+            payload["log"] = tail
+        return payload
+
+    @property
+    def _window(self) -> timedelta:
+        return timedelta(seconds=self._cfg.repeat_window)
+
+    async def page(self, page: int, per_page: int = 10, now: datetime | None = None) -> tuple[list[Alert], int]:
+        """Active first, then the last 7 days, newest first (AL-6). Admin-only kinds are left out."""
+        now = now or utcnow()
+        async with self._sessions() as session:
+            rows = list(
+                await session.scalars(
+                    select(Alert)
+                    .where(Alert.kind != AlertKind.AGENT_ATTACK)
+                    .where(or_(Alert.resolved_at.is_(None), Alert.last_at >= now - HISTORY))
+                )
+            )
+        rows.sort(key=lambda a: (a.resolved_at is not None, -as_utc(a.last_at).timestamp()))
+        return rows[page * per_page : (page + 1) * per_page], len(rows)
+
+    async def get(self, alert_id: int) -> Alert | None:
+        async with self._sessions() as session:
+            return await session.get(Alert, alert_id)
+
+    async def active_count(self) -> int:
+        async with self._sessions() as session:
+            rows = await session.scalars(
+                select(Alert.id).where(Alert.resolved_at.is_(None), Alert.kind != AlertKind.AGENT_ATTACK)
+            )
+            return len(list(rows))

app/bot/menu.py

@@ -4,8 +4,8 @@ from enum import StrEnum
 
 from aiogram import F
 from aiogram.filters.callback_data import CallbackData
-from aiogram.types import InlineKeyboardMarkup, ReplyKeyboardMarkup
-from aiogram.utils.keyboard import InlineKeyboardBuilder, ReplyKeyboardBuilder
+from aiogram.types import ReplyKeyboardMarkup
+from aiogram.utils.keyboard import ReplyKeyboardBuilder
 from magic_filter import MagicFilter
 
 from app.texts import Texts
@@ -57,10 +57,3 @@ def main_menu(texts: Texts, *, is_admin: bool) -> ReplyKeyboardMarkup:
         kb.button(text=m.members)
     kb.adjust(2, 2, 1)
     return kb.as_markup(resize_keyboard=True, is_persistent=True, input_field_placeholder=m.placeholder)
-
-
-def section_nav(texts: Texts) -> InlineKeyboardMarkup:
-    kb = InlineKeyboardBuilder()
-    kb.button(text=texts.menu.back, callback_data=Nav(action=NavAction.BACK))
-    kb.button(text=texts.menu.cancel, callback_data=Nav(action=NavAction.CANCEL))
-    return kb.as_markup()

app/bot/routers/alerts.py

@@ -0,0 +1,205 @@
+"""«🚨 Алерты»: the alert message itself, the list (active first, then 7 days, 10 per page) and cards (SPEC AL-3, AL-6).
+
+Buttons under an alert open «📜 Логи» / «🩺 Статус» as a new message: editing would wipe the alert out of the chat.
+"""
+
+import html
+import logging
+from collections import defaultdict
+from enum import StrEnum
+from typing import Any
+
+from aiogram import F, Router
+from aiogram.enums import ParseMode
+from aiogram.exceptions import TelegramBadRequest
+from aiogram.filters.callback_data import CallbackData
+from aiogram.fsm.context import FSMContext
+from aiogram.types import CallbackQuery, InlineKeyboardMarkup, Message
+from aiogram.utils.keyboard import InlineKeyboardBuilder
+
+from app.alerts.service import AlertService
+from app.bot.menu import Nav, NavAction
+from app.bot.routers.logs import container_keyboard
+from app.bot.routers.status import render_card as status_card
+from app.db.models import Alert, AlertKind, as_utc, utcnow
+from app.runtime.docker import DockerClient, DockerError
+from app.texts import Texts
+
+log = logging.getLogger(__name__)
+
+PER_PAGE = 10
+LOG_CHARS = 3000
+
+
+class AlertsAct(StrEnum):
+    LIST = "l"
+    CARD = "c"
+    LOGS = "g"
+    STATUS = "s"
+
+
+class AlertsCb(CallbackData, prefix="al"):
+    a: AlertsAct
+    p: int = 0  # page
+    id: int = 0  # alert id
+    c: str = ""  # 12-char container id prefix for LOGS / STATUS
+
+
+def title(texts: Texts, alert: Alert) -> str:
+    return texts.alerts.kinds[alert.kind]
+
+
+def service(texts: Texts, alert: Alert) -> str:
+    return html.escape(alert.service or texts.alerts.host)
+
+
+def detail(texts: Texts, alert: Alert) -> str:
+    values: defaultdict[str, Any] = defaultdict(str, alert.payload)
+    return html.escape(texts.alerts.details[alert.kind].format_map(values))
+
+
+def _with_log(text: str, alert: Alert) -> str:
+    tail = str(alert.payload.get("log", ""))
+    return f"{text}\n<pre>{html.escape(tail[-LOG_CHARS:])}</pre>" if tail else text
+
+
+def render(texts: Texts, alert: Alert) -> str:
+    """The alert message (AL-3); re-rendered with the counter on repeats (AL-5)."""
+    t = texts.alerts
+    text = t.message.format(
+        title=title(texts, alert),
+        service=service(texts, alert),
+        detail=detail(texts, alert),
+        opened=as_utc(alert.opened_at).strftime("%d.%m %H:%M"),
+        repeats=t.repeats.format(count=alert.count) if alert.count > 1 else "",
+    )
+    return _with_log(text, alert)
+
+
+def keyboard(texts: Texts, alert: Alert, page: int | None = None) -> InlineKeyboardMarkup:
+    """page=None: under the alert message («📋 Список алертов»); a number: on a card («⬅️ К списку»)."""
+    t = texts.alerts
+    kb = InlineKeyboardBuilder()
+    cid = str(alert.payload.get("container_id", ""))[:12]
+    if cid:
+        kb.button(text=t.logs_button, callback_data=AlertsCb(a=AlertsAct.LOGS, c=cid))
+        kb.button(text=t.status_button, callback_data=AlertsCb(a=AlertsAct.STATUS, c=cid))
+    back = t.list_button if page is None else t.to_list
+    kb.button(text=back, callback_data=AlertsCb(a=AlertsAct.LIST, p=page or 0))
+    kb.adjust(2, 1)
+    return kb.as_markup()
+
+
+def render_list(
+    texts: Texts, alerts: list[Alert], total: int, active: int, page: int
+) -> tuple[str, InlineKeyboardMarkup]:
+    t = texts.alerts
+    pages = max((total + PER_PAGE - 1) // PER_PAGE, 1)
+    lines = [t.list_title.format(active=active, total=total)]
+    if not alerts:
+        lines += ["", t.list_empty]
+    kb = InlineKeyboardBuilder()
+    for a in alerts:
+        line = t.list_line.format(
+            mark=t.active_mark if a.resolved_at is None else t.resolved_mark,
+            title=title(texts, a),
+            service=a.service or t.host,
+            time=as_utc(a.last_at).strftime("%d.%m %H:%M"),
+            repeats=t.repeats.format(count=a.count) if a.count > 1 else "",
+        )
+        kb.button(text=line[:64], callback_data=AlertsCb(a=AlertsAct.CARD, p=page, id=a.id))
+    nav = []
+    if page > 0:
+        kb.button(text=t.prev, callback_data=AlertsCb(a=AlertsAct.LIST, p=page - 1))
+        nav.append(1)
+    if page + 1 < pages:
+        kb.button(text=t.next, callback_data=AlertsCb(a=AlertsAct.LIST, p=page + 1))
+        nav.append(1)
+    kb.button(text=texts.menu.back, callback_data=Nav(action=NavAction.BACK))
+    kb.adjust(*([1] * len(alerts)), *([len(nav)] if nav else []), 1)
+    if pages > 1:
+        lines.append(t.page.format(page=page + 1, pages=pages))
+    return "\n".join(lines), kb.as_markup()
+
+
+def render_alert_card(texts: Texts, a: Alert, page: int) -> tuple[str, InlineKeyboardMarkup]:
+    t = texts.alerts
+    text = t.card.format(
+        title=title(texts, a),
+        service=service(texts, a),
+        detail=detail(texts, a),
+        opened=as_utc(a.opened_at).strftime("%d.%m %H:%M"),
+        resolved=as_utc(a.resolved_at).strftime("%d.%m %H:%M UTC") if a.resolved_at else t.ongoing,
+        count=a.count,
+    )
+    return _with_log(text, a), keyboard(texts, a, page)
+
+
+def build_router(texts: Texts, alerts: AlertService | None, docker: DockerClient | None = None) -> Router:
+    router = Router(name="alerts")
+    t = texts.alerts
+
+    async def list_screen(page: int) -> tuple[str, InlineKeyboardMarkup]:
+        if alerts is None:
+            return render_list(texts, [], 0, 0, 0)
+        rows, total = await alerts.page(page, PER_PAGE)
+        if not rows and page > 0:  # the history shrank since the button was drawn
+            page = 0
+            rows, total = await alerts.page(0, PER_PAGE)
+        return render_list(texts, rows, total, await alerts.active_count(), page)
+
+    @router.message(F.text == texts.menu.alerts)
+    async def open_alerts(message: Message, state: FSMContext) -> None:
+        await state.clear()
+        text, kb = await list_screen(0)
+        await message.answer(text, reply_markup=kb)
+
+    @router.callback_query(AlertsCb.filter())
+    async def on_action(query: CallbackQuery, callback_data: AlertsCb) -> None:
+        if not isinstance(query.message, Message):
+            await query.answer()
+            return
+        msg = query.message
+        # Pressed under an alert: never overwrite it — answer with a new message
+        rows = msg.reply_markup.inline_keyboard if msg.reply_markup else []
+        under_alert = any(b.text == t.list_button for row in rows for b in row)
+        match callback_data.a:
+            case AlertsAct.LOGS | AlertsAct.STATUS:
+                try:
+                    if docker is None:
+                        raise DockerError("docker is not configured (DOCKER_URL)")
+                    c = await docker.get(callback_data.c)
+                    if callback_data.a is AlertsAct.LOGS:
+                        await msg.answer(
+                            texts.logs.container.format(name=c.name), reply_markup=container_keyboard(texts, c.short_id)
+                        )
+                    else:
+                        details = await docker.inspect(c.id)
+                        stats = await docker.stats(c.id) if details.state == "running" else None
+                        text, kb = status_card(texts, details, stats, utcnow())
+                        await msg.answer(text, reply_markup=kb)
+                except DockerError as e:
+                    log.warning("alerts: %s", e)
+                    await query.answer(texts.status.unavailable, show_alert=True)
+                    return
+                await query.answer()
+                return
+            case AlertsAct.CARD:
+                alert = await alerts.get(callback_data.id) if alerts else None
+                if alert is None or alert.kind is AlertKind.AGENT_ATTACK:  # 🛡 lives only in the admins' chats
+                    await query.answer(t.gone, show_alert=True)
+                    return
+                text, kb = render_alert_card(texts, alert, callback_data.p)
+            case AlertsAct.LIST:
+                text, kb = await list_screen(max(callback_data.p, 0))
+        if under_alert:
+            await msg.answer(text, parse_mode=ParseMode.HTML, reply_markup=kb)
+        else:
+            try:
+                await msg.edit_text(text, parse_mode=ParseMode.HTML, reply_markup=kb)
+            except TelegramBadRequest as e:
+                if "message is not modified" not in str(e):
+                    raise
+        await query.answer()
+
+    return router

app/bot/routers/common.py

@@ -1,12 +1,12 @@
 """/start (incl. invite deep links), /help, /menu, main menu buttons and in-section navigation."""
 
-from aiogram import F, Router
+from aiogram import Router
 from aiogram.filters import Command, CommandObject, CommandStart
 from aiogram.fsm.context import FSMContext
 from aiogram.types import CallbackQuery, Message
 
 from app.access.service import AccessService, RedeemStatus
-from app.bot.menu import Nav, NavAction, Section, main_menu, section_by_button, section_nav
+from app.bot.menu import Nav, NavAction, main_menu
 from app.bot.middlewares.access import tg_user
 from app.db.models import Role, User
 from app.texts import Texts
@@ -14,10 +14,6 @@ from app.texts import Texts
 
 def build_router(texts: Texts, access: AccessService) -> Router:
     router = Router(name="common")
-    sections = section_by_button(texts)
-    section_text = {
-        Section.ALERTS: texts.section.alerts,
-    }
 
     @router.message(CommandStart())
     async def start(message: Message, command: CommandObject, state: FSMContext, member: User | None) -> None:
@@ -46,12 +42,6 @@ def build_router(texts: Texts, access: AccessService) -> Router:
     async def help_(message: Message) -> None:
         await message.answer(texts.help.text)
 
-    # Sections with their own routers (members, commit) are matched there first
-    @router.message(F.text.in_({k for k, v in sections.items() if v in section_text}))
-    async def open_section(message: Message, state: FSMContext) -> None:
-        await state.clear()
-        await message.answer(section_text[sections[message.text or ""]], reply_markup=section_nav(texts))
-
     @router.callback_query(Nav.filter())
     async def navigate(query: CallbackQuery, callback_data: Nav, state: FSMContext) -> None:
         # At a section's root "Back" and "Cancel" both close it; nested screens will override "Back" per section.

app/bot/routers/logs.py

@@ -6,19 +6,19 @@ import logging
 from datetime import timedelta
 from enum import StrEnum
 
-from aiogram import Bot, F, Router
+from aiogram import F, Router
 from aiogram.enums import ParseMode
-from aiogram.exceptions import TelegramAPIError
 from aiogram.filters.callback_data import CallbackData
 from aiogram.fsm.context import FSMContext
 from aiogram.fsm.state import State, StatesGroup
 from aiogram.types import BufferedInputFile, CallbackQuery, InlineKeyboardMarkup, Message
 from aiogram.utils.keyboard import InlineKeyboardBuilder
 
-from app.access.service import AccessService
 from app.agent.service import Agent, Outcome
+from app.alerts.detectors import Condition
+from app.alerts.service import AlertService
 from app.bot.menu import Nav, NavAction, free_input
-from app.db.models import LogMode, Role, User, utcnow
+from app.db.models import AlertKind, LogMode, User, utcnow
 from app.logs.grep import GrepError, GrepErrorCode, GrepResult, parse_grep, run, to_command
 from app.logs.journal import LogJournal
 from app.runtime.docker import Container, DockerClient, DockerError
@@ -146,7 +146,7 @@ def build_router(
     docker: DockerClient | None,
     journal: LogJournal,
     agent: Agent | None = None,
-    access: AccessService | None = None,
+    alerts: AlertService | None = None,
 ) -> Router:
     router = Router(name="logs")
     t = texts.logs
@@ -156,23 +156,16 @@ def build_router(
             raise DockerError("docker is not configured (DOCKER_URL)")
         return docker
 
-    async def alert_admins(bot: Bot, member: User, prompt: str, stage: str, reason: str) -> None:
-        """🛡 to the admins only (SPEC AL-2, L-12). The asker sees just a polite refusal."""
-        if access is None:
+    async def alert_admins(member: User, prompt: str, stage: str, reason: str) -> None:
+        """🛡 to the admins only (SPEC AL-2, L-12); repeats by the same member bump one alert's counter."""
+        if alerts is None:
             return
-        text = t.attack_alert.format(
-            who=html.escape(who(member)),
-            stage=stage,
-            reason=html.escape(reason),
-            prompt=html.escape(prompt[:1000]),
+        condition = Condition(
+            AlertKind.AGENT_ATTACK,
+            f"user {member.tg_id}",
+            {"who": who(member), "stage": stage, "reason": reason, "log": prompt[:1000]},
         )
-        for admin in await access.list_members():
-            if admin.role is not Role.ADMIN:
-                continue
-            try:
-                await bot.send_message(admin.tg_id, text, parse_mode=ParseMode.HTML)
-            except TelegramAPIError as e:  # one blocked admin must not hide the alert from the others
-                log.warning("attack alert to %s failed: %s", admin.tg_id, e)
+        await alerts.event(condition, admins_only=True)
 
     @router.message(F.text == texts.menu.logs)
     async def open_logs(message: Message, state: FSMContext) -> None:
@@ -275,7 +268,7 @@ def build_router(
         await deliver(message, texts, head, query.count, result, container_keyboard(texts, c.short_id), c.name)
 
     @router.message(LogsFlow.agent, F.text, free_input(texts))
-    async def got_question(message: Message, state: FSMContext, member: User, bot: Bot) -> None:
+    async def got_question(message: Message, state: FSMContext, member: User) -> None:
         if agent is None:  # pragma: no cover - the button doesn't open this state without an agent
             await state.clear()
             return
@@ -296,7 +289,7 @@ def build_router(
         match answer.outcome:
             case Outcome.ATTACK:
                 await message.answer(t.agent_refused, reply_markup=kb)
-                await alert_admins(bot, member, prompt, answer.stage, answer.reason)
+                await alert_admins(member, prompt, answer.stage, answer.reason)
                 return
             case Outcome.QUOTA:
                 await message.answer(t.agent_quota.format(quota=agent.quota), reply_markup=kb)

app/db/models.py

@@ -45,6 +45,11 @@ def utcnow() -> datetime:
     return datetime.now(UTC)
 
 
+def as_utc(dt: datetime) -> datetime:
+    """SQLite hands timestamps back naive (they are stored in UTC); Postgres — aware. Arithmetic needs aware."""
+    return dt if dt.tzinfo is not None else dt.replace(tzinfo=UTC)
+
+
 def enum(cls: type[StrEnum], name: str) -> Enum:
     return Enum(
         cls,

app/main.py

@@ -10,8 +10,13 @@ from aiogram.client.session.aiohttp import AiohttpSession
 from app import __version__
 from app.access.service import AccessService
 from app.agent.service import Agent
+from app.alerts.broadcast import TelegramNotifier
+from app.alerts.detectors import load_alerts_config
+from app.alerts.monitor import AlertMonitor
+from app.alerts.service import AlertService
 from app.bot.fsm import make_storage
 from app.bot.middlewares.access import AccessMiddleware
+from app.bot.routers import alerts as alerts_router
 from app.bot.routers import commit, common, logs, members, status
 from app.config import Settings
 from app.db.maintenance import cleanup_loop
@@ -41,6 +46,7 @@ def build_dispatcher(
     docker: DockerClient | None = None,
     checker: Checker | None = None,
     agent: Agent | None = None,
+    alerts: AlertService | None = None,
 ) -> Dispatcher:
     dp = Dispatcher(storage=make_storage(sessions))
     gate = AccessMiddleware(access, texts)
@@ -52,7 +58,8 @@ def build_dispatcher(
         commit.build_router(texts, repo, DraftService(sessions), Publisher(repo, sessions), commit_url_template)
     )
     dp.include_router(status.build_router(texts, docker, checker))
-    dp.include_router(logs.build_router(texts, docker, LogJournal(sessions), agent, access))
+    dp.include_router(logs.build_router(texts, docker, LogJournal(sessions), agent, alerts))
+    dp.include_router(alerts_router.build_router(texts, alerts, docker))
     dp.include_router(common.build_router(texts, access))
     return dp
 
@@ -60,6 +67,7 @@ def build_dispatcher(
 async def run(settings: Settings) -> None:
     texts = load_texts(settings.runtime.config_dir)
     checks_cfg = load_checks(settings.runtime.config_dir)
+    alerts_cfg = load_alerts_config(settings.runtime.config_dir)
     engine = make_engine(settings.database.url)
     await upgrade(engine)
     sessions = make_sessionmaker(engine)
@@ -72,8 +80,25 @@ async def run(settings: Settings) -> None:
     # The mock provider answers `{}` to the guard — every question would be an «attack»: no real LLM, no agent
     llm = make_provider(settings.llm) if settings.llm.provider != "mock" else None
     agent = Agent.from_config(llm, sessions, settings.runtime.config_dir) if llm else None
+    proxy = settings.telegram.proxy_url
+    session = AiohttpSession(proxy=proxy.get_secret_value()) if proxy else None
+    bot = Bot(settings.telegram.bot_token.get_secret_value(), session=session)
+    alerts = AlertService(
+        sessions,
+        TelegramNotifier(bot, access, sessions, texts),
+        alerts_cfg,
+        log_source=(lambda cid, n: docker.logs(cid, tail=n)) if docker else None,
+    )
     dp = build_dispatcher(
-        texts, sessions, access, UploadsRepo(settings.git), settings.git.commit_url_template, docker, checker, agent
+        texts,
+        sessions,
+        access,
+        UploadsRepo(settings.git),
+        settings.git.commit_url_template,
+        docker,
+        checker,
+        agent,
+        alerts,
     )
     rt = settings.runtime
     background = [
@@ -81,10 +106,10 @@ async def run(settings: Settings) -> None:
         asyncio.create_task(cleanup_loop(sessions)),
     ]
     if docker:
-        background.append(asyncio.create_task(snapshot_loop(docker, store, settings.docker.snapshot_interval)))
-    proxy = settings.telegram.proxy_url
-    session = AiohttpSession(proxy=proxy.get_secret_value()) if proxy else None
-    bot = Bot(settings.telegram.bot_token.get_secret_value(), session=session)
+        monitor = AlertMonitor(alerts, store, alerts_cfg)
+        background.append(
+            asyncio.create_task(snapshot_loop(docker, store, settings.docker.snapshot_interval, monitor.on_round))
+        )
     log.info(
         "tgit %s starting, %d bootstrap admin(s), telegram %s, docker %s, agent %s",
         __version__,

app/runtime/snapshots.py

@@ -6,13 +6,15 @@ Old rows are removed by `app.db.maintenance`.
 
 import asyncio
 import logging
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
 from datetime import datetime
 
 from sqlalchemy import select
 
 from app.db.models import ContainerSnapshot, utcnow
 from app.db.session import SessionMaker
-from app.runtime.docker import DockerClient, DockerError
+from app.runtime.docker import Details, DockerClient, DockerError, Stats
 
 log = logging.getLogger(__name__)
 
@@ -35,6 +37,15 @@ class SnapshotStore:
                 .limit(1)
             )
 
+    async def since(self, container: str, since: datetime) -> list[ContainerSnapshot]:
+        async with self._sessions() as session:
+            rows = await session.scalars(
+                select(ContainerSnapshot)
+                .where(ContainerSnapshot.container == container, ContainerSnapshot.at >= since)
+                .order_by(ContainerSnapshot.at)
+            )
+            return list(rows)
+
     async def latest(self, container: str) -> ContainerSnapshot | None:
         async with self._sessions() as session:
             return await session.scalar(
@@ -45,36 +56,63 @@ class SnapshotStore:
             )
 
 
-async def _snapshot(docker: DockerClient, container_id: str, at: datetime) -> ContainerSnapshot | None:
+@dataclass(frozen=True)
+class Observation:
+    """One container in one round: what the alert detectors look at (#15)."""
+
+    details: Details
+    stats: Stats | None  # None when not running or stats failed
+
+
+async def _observe(docker: DockerClient, container_id: str) -> Observation | None:
     try:
-        details, stats = await asyncio.gather(docker.inspect(container_id), docker.stats(container_id))
+        details = await docker.inspect(container_id)
     except DockerError as e:
         log.warning("snapshot %s: %s", container_id[:12], e)
         return None
+    stats = None
+    if details.state == "running":
+        try:
+            stats = await docker.stats(container_id)
+        except DockerError as e:
+            log.warning("stats %s: %s", container_id[:12], e)
+    return Observation(details, stats)
+
+
+def _row(o: Observation, at: datetime) -> ContainerSnapshot | None:
+    if o.stats is None:
+        return None
     return ContainerSnapshot(
-        container=details.name,
+        container=o.details.name,
         at=at,
-        restart_count=details.restart_count,
-        cpu_pct=stats.cpu_pct,
-        mem_pct=round(stats.mem_pct, 1),
-        health=details.health,
+        restart_count=o.details.restart_count,
+        cpu_pct=o.stats.cpu_pct,
+        mem_pct=round(o.stats.mem_pct, 1),
+        health=o.details.health,
     )
 
 
-async def collect_once(docker: DockerClient, store: SnapshotStore, now: datetime | None = None) -> int:
+async def collect_once(docker: DockerClient, store: SnapshotStore, now: datetime | None = None) -> list[Observation]:
+    """Inspect every managed container, stats for the running ones; a snapshot row per running container."""
     at = now or utcnow()
-    running = [c for c in await docker.containers() if c.state == "running"]
-    rows = await asyncio.gather(*(_snapshot(docker, c.id, at) for c in running))
-    snapshots = [r for r in rows if r is not None]
-    await store.add(snapshots)
-    return len(snapshots)
+    found = await asyncio.gather(*(_observe(docker, c.id) for c in await docker.containers()))
+    observations = [o for o in found if o is not None]
+    await store.add([row for o in observations if (row := _row(o, at)) is not None])
+    return observations
 
 
-async def snapshot_loop(docker: DockerClient, store: SnapshotStore, interval: float) -> None:
+# Called after every round: the observations, or None when Docker itself was unreachable
+RoundHook = Callable[[list[Observation] | None], Awaitable[None]]
+
+
+async def snapshot_loop(
+    docker: DockerClient, store: SnapshotStore, interval: float, on_round: RoundHook | None = None
+) -> None:
     failing = False
     while True:
+        observations: list[Observation] | None = None
         try:
-            await collect_once(docker, store)
+            observations = await collect_once(docker, store)
             if failing:
                 log.info("snapshots: docker reachable again")
             failing = False
@@ -84,4 +122,9 @@ async def snapshot_loop(docker: DockerClient, store: SnapshotStore, interval: fl
             failing = True
         except Exception:  # a broken round must not kill the bot
             log.exception("snapshots failed")
+        if on_round is not None:
+            try:
+                await on_round(observations)
+            except Exception:
+                log.exception("snapshot hook failed")
         await asyncio.sleep(interval)

app/texts.py

@@ -3,7 +3,9 @@
 from pathlib import Path
 
 import yaml
-from pydantic import BaseModel, ConfigDict
+from pydantic import BaseModel, ConfigDict, field_validator
+
+from app.db.models import AlertKind
 
 
 class _Strict(BaseModel):
@@ -32,9 +34,6 @@ class HelpTexts(_Strict):
 
 
 class SectionTexts(_Strict):
-    commit: str
-    alerts: str
-    members: str
     cancelled: str
 
 
@@ -170,7 +169,6 @@ class LogsTexts(_Strict):
     agent_unavailable: str
     agent_not_understood: str
     head_agent: str
-    attack_alert: str
     to_list: str
     to_container: str
     ask_grep: str
@@ -190,6 +188,38 @@ class LogsTexts(_Strict):
     err_regex: str
 
 
+class AlertsTexts(_Strict):
+    kinds: dict[AlertKind, str]
+    details: dict[AlertKind, str]
+    message: str
+    repeats: str
+    resolved: str
+    logs_button: str
+    status_button: str
+    list_button: str
+    list_title: str
+    list_empty: str
+    list_line: str
+    active_mark: str
+    resolved_mark: str
+    page: str
+    prev: str
+    next: str
+    card: str
+    ongoing: str
+    to_list: str
+    host: str
+    gone: str
+
+    @field_validator("kinds", "details")
+    @classmethod
+    def _every_kind(cls, value: dict[AlertKind, str]) -> dict[AlertKind, str]:
+        missing = set(AlertKind) - set(value)
+        if missing:
+            raise ValueError(f"no text for {sorted(missing)}")
+        return value
+
+
 class Texts(_Strict):
     menu: MenuTexts
     start: StartTexts
@@ -199,6 +229,7 @@ class Texts(_Strict):
     commit: CommitTexts
     status: StatusTexts
     logs: LogsTexts
+    alerts: AlertsTexts
 
 
 def load_texts(config_dir: Path) -> Texts: