"""Glue: after each snapshot round run the detectors and sync open alerts (hooked into `snapshot_loop`).""" import os from collections.abc import Callable from datetime import datetime, timedelta from pathlib import Path from app.alerts.detectors import ( MONITORED, AlertsConfig, Condition, container_conditions, disk_condition, docker_condition, ) from app.alerts.service import AlertService from app.db.models import AlertKind, utcnow from app.runtime.snapshots import Observation, SnapshotStore WITHOUT_DOCKER = frozenset({AlertKind.DEPENDENCY_DOWN, AlertKind.DISK_LOW}) def free_disk_pct(path: Path) -> float | None: try: st = os.statvfs(path) except OSError: return None return 100.0 * st.f_bavail / st.f_blocks if st.f_blocks else None class AlertMonitor: def __init__( self, alerts: AlertService, store: SnapshotStore, cfg: AlertsConfig, disk: Callable[[Path], float | None] = free_disk_pct, clock: Callable[[], datetime] = utcnow, ) -> None: self._alerts = alerts self._store = store self._cfg = cfg self._disk = disk self._clock = clock self._docker_down_since: datetime | None = None @property def _history(self) -> timedelta: return timedelta(seconds=max(self._cfg.restart_loop_window, self._cfg.overload_for) + 60) async def on_round(self, observations: list[Observation] | None) -> None: now = self._clock() conditions: list[Condition] = disk_condition(self._disk(self._cfg.disk_path), self._cfg) if observations is None: # Docker is unreachable: container alerts can't be judged — leave them as they are self._docker_down_since = self._docker_down_since or now conditions += docker_condition(self._docker_down_since, self._cfg, now) await self._alerts.sync(conditions, WITHOUT_DOCKER, now) return self._docker_down_since = None for o in observations: history = await self._store.since(o.details.name, now - self._history) conditions += container_conditions(o, history, self._cfg, now) await self._alerts.sync(conditions, MONITORED, now)