feat(alerts): алерты всей команде — детекторы, дедупликация, список (#15)
CI / ruff · mypy · pytest (pull_request) Successful in 4m49s
CI / ruff · mypy · pytest (pull_request) Successful in 4m49s
- детекторы по состоянию после каждого раунда снапшотов: краш, OOM, цикл рестартов, unhealthy, перегрузка ≥ 1 мин, мало места, Docker недоступен; пороги в config/alerts.yaml - AlertService: ключ kind+service, повтор в 10 мин — счётчик в исходном сообщении, «✅ восстановлено» ответом; разовые события (🛡) со счётчиком - рассылка всем активным (🛡 — только админам), темп ~25/с, RetryAfter, заблокировавший бота пропускается; alert_deliveries для правок и ответов - «🚨 Алерты»: активные + 7 дней по 10, карточка; кнопки «📜 Логи», «🩺 Статус» под алертом открывают новое сообщение, а не затирают алерт - 🛡 атаки через агента идут через общий сервис алертов - убраны заглушки «Раздел в разработке» — у всех разделов свои роутеры Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+59
-16
@@ -6,13 +6,15 @@ Old rows are removed by `app.db.maintenance`.
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.db.models import ContainerSnapshot, utcnow
|
||||
from app.db.session import SessionMaker
|
||||
from app.runtime.docker import DockerClient, DockerError
|
||||
from app.runtime.docker import Details, DockerClient, DockerError, Stats
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -35,6 +37,15 @@ class SnapshotStore:
|
||||
.limit(1)
|
||||
)
|
||||
|
||||
async def since(self, container: str, since: datetime) -> list[ContainerSnapshot]:
|
||||
async with self._sessions() as session:
|
||||
rows = await session.scalars(
|
||||
select(ContainerSnapshot)
|
||||
.where(ContainerSnapshot.container == container, ContainerSnapshot.at >= since)
|
||||
.order_by(ContainerSnapshot.at)
|
||||
)
|
||||
return list(rows)
|
||||
|
||||
async def latest(self, container: str) -> ContainerSnapshot | None:
|
||||
async with self._sessions() as session:
|
||||
return await session.scalar(
|
||||
@@ -45,36 +56,63 @@ class SnapshotStore:
|
||||
)
|
||||
|
||||
|
||||
async def _snapshot(docker: DockerClient, container_id: str, at: datetime) -> ContainerSnapshot | None:
|
||||
@dataclass(frozen=True)
|
||||
class Observation:
|
||||
"""One container in one round: what the alert detectors look at (#15)."""
|
||||
|
||||
details: Details
|
||||
stats: Stats | None # None when not running or stats failed
|
||||
|
||||
|
||||
async def _observe(docker: DockerClient, container_id: str) -> Observation | None:
|
||||
try:
|
||||
details, stats = await asyncio.gather(docker.inspect(container_id), docker.stats(container_id))
|
||||
details = await docker.inspect(container_id)
|
||||
except DockerError as e:
|
||||
log.warning("snapshot %s: %s", container_id[:12], e)
|
||||
return None
|
||||
stats = None
|
||||
if details.state == "running":
|
||||
try:
|
||||
stats = await docker.stats(container_id)
|
||||
except DockerError as e:
|
||||
log.warning("stats %s: %s", container_id[:12], e)
|
||||
return Observation(details, stats)
|
||||
|
||||
|
||||
def _row(o: Observation, at: datetime) -> ContainerSnapshot | None:
|
||||
if o.stats is None:
|
||||
return None
|
||||
return ContainerSnapshot(
|
||||
container=details.name,
|
||||
container=o.details.name,
|
||||
at=at,
|
||||
restart_count=details.restart_count,
|
||||
cpu_pct=stats.cpu_pct,
|
||||
mem_pct=round(stats.mem_pct, 1),
|
||||
health=details.health,
|
||||
restart_count=o.details.restart_count,
|
||||
cpu_pct=o.stats.cpu_pct,
|
||||
mem_pct=round(o.stats.mem_pct, 1),
|
||||
health=o.details.health,
|
||||
)
|
||||
|
||||
|
||||
async def collect_once(docker: DockerClient, store: SnapshotStore, now: datetime | None = None) -> int:
|
||||
async def collect_once(docker: DockerClient, store: SnapshotStore, now: datetime | None = None) -> list[Observation]:
|
||||
"""Inspect every managed container, stats for the running ones; a snapshot row per running container."""
|
||||
at = now or utcnow()
|
||||
running = [c for c in await docker.containers() if c.state == "running"]
|
||||
rows = await asyncio.gather(*(_snapshot(docker, c.id, at) for c in running))
|
||||
snapshots = [r for r in rows if r is not None]
|
||||
await store.add(snapshots)
|
||||
return len(snapshots)
|
||||
found = await asyncio.gather(*(_observe(docker, c.id) for c in await docker.containers()))
|
||||
observations = [o for o in found if o is not None]
|
||||
await store.add([row for o in observations if (row := _row(o, at)) is not None])
|
||||
return observations
|
||||
|
||||
|
||||
async def snapshot_loop(docker: DockerClient, store: SnapshotStore, interval: float) -> None:
|
||||
# Called after every round: the observations, or None when Docker itself was unreachable
|
||||
RoundHook = Callable[[list[Observation] | None], Awaitable[None]]
|
||||
|
||||
|
||||
async def snapshot_loop(
|
||||
docker: DockerClient, store: SnapshotStore, interval: float, on_round: RoundHook | None = None
|
||||
) -> None:
|
||||
failing = False
|
||||
while True:
|
||||
observations: list[Observation] | None = None
|
||||
try:
|
||||
await collect_once(docker, store)
|
||||
observations = await collect_once(docker, store)
|
||||
if failing:
|
||||
log.info("snapshots: docker reachable again")
|
||||
failing = False
|
||||
@@ -84,4 +122,9 @@ async def snapshot_loop(docker: DockerClient, store: SnapshotStore, interval: fl
|
||||
failing = True
|
||||
except Exception: # a broken round must not kill the bot
|
||||
log.exception("snapshots failed")
|
||||
if on_round is not None:
|
||||
try:
|
||||
await on_round(observations)
|
||||
except Exception:
|
||||
log.exception("snapshot hook failed")
|
||||
await asyncio.sleep(interval)
|
||||
|
||||
Reference in New Issue
Block a user