feat(alerts): алерты всей команде — детекторы, дедупликация, список (#15)
CI / ruff · mypy · pytest (pull_request) Successful in 4m49s

- детекторы по состоянию после каждого раунда снапшотов: краш, OOM,
  цикл рестартов, unhealthy, перегрузка ≥ 1 мин, мало места, Docker
  недоступен; пороги в config/alerts.yaml
- AlertService: ключ kind+service, повтор в 10 мин — счётчик в исходном
  сообщении, «✅ восстановлено» ответом; разовые события (🛡) со счётчиком
- рассылка всем активным (🛡 — только админам), темп ~25/с, RetryAfter,
  заблокировавший бота пропускается; alert_deliveries для правок и ответов
- «🚨 Алерты»: активные + 7 дней по 10, карточка; кнопки «📜 Логи»,
  «🩺 Статус» под алертом открывают новое сообщение, а не затирают алерт
- 🛡 атаки через агента идут через общий сервис алертов
- убраны заглушки «Раздел в разработке» — у всех разделов свои роутеры

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-17 15:36:09 +03:00
co-authored by Claude Opus 5.5
parent 65acf3abf1
commit 6c6f766abc
24 changed files with 1355 additions and 107 deletions
+59 -16
View File
@@ -6,13 +6,15 @@ Old rows are removed by `app.db.maintenance`.
import asyncio
import logging
from collections.abc import Awaitable, Callable
from dataclasses import dataclass
from datetime import datetime
from sqlalchemy import select
from app.db.models import ContainerSnapshot, utcnow
from app.db.session import SessionMaker
from app.runtime.docker import DockerClient, DockerError
from app.runtime.docker import Details, DockerClient, DockerError, Stats
log = logging.getLogger(__name__)
@@ -35,6 +37,15 @@ class SnapshotStore:
.limit(1)
)
async def since(self, container: str, since: datetime) -> list[ContainerSnapshot]:
async with self._sessions() as session:
rows = await session.scalars(
select(ContainerSnapshot)
.where(ContainerSnapshot.container == container, ContainerSnapshot.at >= since)
.order_by(ContainerSnapshot.at)
)
return list(rows)
async def latest(self, container: str) -> ContainerSnapshot | None:
async with self._sessions() as session:
return await session.scalar(
@@ -45,36 +56,63 @@ class SnapshotStore:
)
async def _snapshot(docker: DockerClient, container_id: str, at: datetime) -> ContainerSnapshot | None:
@dataclass(frozen=True)
class Observation:
"""One container in one round: what the alert detectors look at (#15)."""
details: Details
stats: Stats | None # None when not running or stats failed
async def _observe(docker: DockerClient, container_id: str) -> Observation | None:
try:
details, stats = await asyncio.gather(docker.inspect(container_id), docker.stats(container_id))
details = await docker.inspect(container_id)
except DockerError as e:
log.warning("snapshot %s: %s", container_id[:12], e)
return None
stats = None
if details.state == "running":
try:
stats = await docker.stats(container_id)
except DockerError as e:
log.warning("stats %s: %s", container_id[:12], e)
return Observation(details, stats)
def _row(o: Observation, at: datetime) -> ContainerSnapshot | None:
if o.stats is None:
return None
return ContainerSnapshot(
container=details.name,
container=o.details.name,
at=at,
restart_count=details.restart_count,
cpu_pct=stats.cpu_pct,
mem_pct=round(stats.mem_pct, 1),
health=details.health,
restart_count=o.details.restart_count,
cpu_pct=o.stats.cpu_pct,
mem_pct=round(o.stats.mem_pct, 1),
health=o.details.health,
)
async def collect_once(docker: DockerClient, store: SnapshotStore, now: datetime | None = None) -> int:
async def collect_once(docker: DockerClient, store: SnapshotStore, now: datetime | None = None) -> list[Observation]:
"""Inspect every managed container, stats for the running ones; a snapshot row per running container."""
at = now or utcnow()
running = [c for c in await docker.containers() if c.state == "running"]
rows = await asyncio.gather(*(_snapshot(docker, c.id, at) for c in running))
snapshots = [r for r in rows if r is not None]
await store.add(snapshots)
return len(snapshots)
found = await asyncio.gather(*(_observe(docker, c.id) for c in await docker.containers()))
observations = [o for o in found if o is not None]
await store.add([row for o in observations if (row := _row(o, at)) is not None])
return observations
async def snapshot_loop(docker: DockerClient, store: SnapshotStore, interval: float) -> None:
# Called after every round: the observations, or None when Docker itself was unreachable
RoundHook = Callable[[list[Observation] | None], Awaitable[None]]
async def snapshot_loop(
docker: DockerClient, store: SnapshotStore, interval: float, on_round: RoundHook | None = None
) -> None:
failing = False
while True:
observations: list[Observation] | None = None
try:
await collect_once(docker, store)
observations = await collect_once(docker, store)
if failing:
log.info("snapshots: docker reachable again")
failing = False
@@ -84,4 +122,9 @@ async def snapshot_loop(docker: DockerClient, store: SnapshotStore, interval: fl
failing = True
except Exception: # a broken round must not kill the bot
log.exception("snapshots failed")
if on_round is not None:
try:
await on_round(observations)
except Exception:
log.exception("snapshot hook failed")
await asyncio.sleep(interval)