tgit/app/alerts/service.py

"""Open / repeat / resolve alerts (SPEC AL-4, AL-5). Messages go through a `Notifier` (the Telegram broadcast)."""

import logging
from collections.abc import Awaitable, Callable, Iterable
from datetime import datetime, timedelta
from typing import Protocol

from sqlalchemy import or_, select
from sqlalchemy.orm import selectinload

from app.alerts.detectors import AlertsConfig, Condition
from app.db.models import Alert, AlertKind, as_utc, utcnow
from app.db.session import SessionMaker

log = logging.getLogger(__name__)

HISTORY = timedelta(days=7)  # AL-6
EDIT_EVERY = timedelta(minutes=1)
LOG_CHARS = 3000

LogSource = Callable[[str, int], Awaitable[str]]  # (container id, lines) -> tail


class Notifier(Protocol):
    async def opened(self, alert: Alert, admins_only: bool) -> None: ...

    async def repeated(self, alert: Alert) -> None: ...

    async def resolved(self, alert: Alert) -> None: ...


class AlertService:
    def __init__(
        self, sessions: SessionMaker, notifier: Notifier, cfg: AlertsConfig, log_source: LogSource | None = None
    ) -> None:
        self._sessions = sessions
        self._notifier = notifier
        self._cfg = cfg
        self._log_source = log_source

    async def _tail(self, c: Condition) -> str:
        if self._log_source is None or c.container_id is None or not self._cfg.log_tail:
            return ""
        try:
            text = await self._log_source(c.container_id, self._cfg.log_tail)
        except Exception as e:  # an alert without a log tail is still an alert
            log.warning("alert log tail for %s: %s", c.service, e)
            return ""
        return text[-LOG_CHARS:]

    async def sync(
        self, conditions: Iterable[Condition], scope: frozenset[AlertKind], now: datetime | None = None
    ) -> None:
        """Make open alerts of `scope` kinds match `conditions` exactly: open the new, resolve the gone."""
        now = now or utcnow()
        wanted = {c.key: c for c in conditions if c.kind in scope}
        async with self._sessions.begin() as session:
            rows = await session.scalars(
                select(Alert)
                .options(selectinload(Alert.deliveries))
                .where(Alert.kind.in_(scope), or_(Alert.resolved_at.is_(None), Alert.last_at >= now - self._window))
            )
            by_key: dict[tuple[AlertKind, str | None], Alert] = {}
            for a in rows:  # the open one wins over a recently resolved one with the same key
                if (a.kind, a.service) not in by_key or a.resolved_at is None:
                    by_key[(a.kind, a.service)] = a
            opened, repeated, resolved = [], [], []
            for key, c in wanted.items():
                known = by_key.get(key)
                if known is not None and known.resolved_at is None:
                    known.last_at = now
                    if c.bump:  # one more incident while still open (a crash that Docker already restarted)
                        known.count += 1
                        known.payload = {**known.payload, **c.payload}
                        if self._may_edit(known, now):
                            repeated.append(known)
                    continue
                if (
                    known is not None
                    and known.resolved_at is not None
                    and now - as_utc(known.resolved_at) <= self._window
                ):
                    known.resolved_at, known.last_at, known.count = None, now, known.count + 1  # back within the window
                    known.payload = {**known.payload, **c.payload}
                    if self._may_edit(known, now):
                        repeated.append(known)
                    continue
                fresh = Alert(
                    kind=c.kind, service=c.service, payload=await self._payload(c), opened_at=now, last_at=now
                )
                session.add(fresh)
                opened.append(fresh)
            for key, a in by_key.items():
                # hysteresis: gone for a while, not just for one round — otherwise a flapping service
                # would get «✅ восстановлено» after every blip
                if a.resolved_at is None and key not in wanted and now - as_utc(a.last_at) >= self._settle:
                    a.resolved_at = now
                    resolved.append(a)
            await session.flush()
        for a in opened:
            await self._notifier.opened(a, admins_only=False)
        for a in repeated:
            await self._notifier.repeated(a)
        for a in resolved:
            await self._notifier.resolved(a)

    async def event(self, c: Condition, admins_only: bool = False, now: datetime | None = None) -> Alert:
        """A one-shot alert (🛡 attack, CD failure): opened already resolved; repeats bump the counter (AL-5)."""
        now = now or utcnow()
        async with self._sessions.begin() as session:
            recent = await session.scalar(
                select(Alert)
                .options(selectinload(Alert.deliveries))
                .where(Alert.kind == c.kind, Alert.service == c.service, Alert.last_at >= now - self._window)
                .order_by(Alert.last_at.desc())
                .limit(1)
            )
            if recent is not None:
                recent.count += 1
                recent.last_at = now
                recent.payload = {**recent.payload, **c.payload}
                alert, new, edit = recent, False, self._may_edit(recent, now)
            else:
                alert = Alert(
                    kind=c.kind, service=c.service, payload=dict(c.payload), opened_at=now, last_at=now, resolved_at=now
                )
                session.add(alert)
                await session.flush()
                new, edit = True, False
        if new:
            await self._notifier.opened(alert, admins_only=admins_only)
        elif edit:
            await self._notifier.repeated(alert)
        return alert

    async def _payload(self, c: Condition) -> dict[str, object]:
        payload: dict[str, object] = dict(c.payload)
        if c.container_id:
            payload["container_id"] = c.container_id
        tail = await self._tail(c)
        if tail:
            payload["log"] = tail
        return payload

    @property
    def _window(self) -> timedelta:
        return timedelta(seconds=self._cfg.repeat_window)

    @property
    def _settle(self) -> timedelta:
        return timedelta(seconds=self._cfg.resolve_after)

    def _may_edit(self, alert: Alert, now: datetime) -> bool:
        """Counter edits at most once a minute per alert; the count itself is always exact in the DB."""
        last = alert.payload.get("edited_at")
        if isinstance(last, str) and now - as_utc(datetime.fromisoformat(last)) < EDIT_EVERY:
            return False
        alert.payload = {**alert.payload, "edited_at": now.isoformat()}
        return True

    async def page(self, page: int, per_page: int = 10, now: datetime | None = None) -> tuple[list[Alert], int]:
        """Active first, then the last 7 days, newest first (AL-6). Admin-only kinds are left out."""
        now = now or utcnow()
        async with self._sessions() as session:
            rows = list(
                await session.scalars(
                    select(Alert)
                    .where(Alert.kind != AlertKind.AGENT_ATTACK)
                    .where(or_(Alert.resolved_at.is_(None), Alert.last_at >= now - HISTORY))
                )
            )
        rows.sort(key=lambda a: (a.resolved_at is not None, -as_utc(a.last_at).timestamp()))
        return rows[page * per_page : (page + 1) * per_page], len(rows)

    async def get(self, alert_id: int) -> Alert | None:
        async with self._sessions() as session:
            return await session.get(Alert, alert_id)

    async def active_count(self) -> int:
        async with self._sessions() as session:
            rows = await session.scalars(
                select(Alert.id).where(Alert.resolved_at.is_(None), Alert.kind != AlertKind.AGENT_ATTACK)
            )
            return len(list(rows))