"""Every 15 s: restart count, CPU and RAM of each running managed container → `container_snapshots`. The history answers «are restarts growing over the last 5 minutes» (S-3) and feeds alerts (#15). Old rows are removed by `app.db.maintenance`. """ import asyncio import logging from datetime import datetime from sqlalchemy import select from app.db.models import ContainerSnapshot, utcnow from app.db.session import SessionMaker from app.runtime.docker import DockerClient, DockerError log = logging.getLogger(__name__) class SnapshotStore: def __init__(self, sessions: SessionMaker) -> None: self._sessions = sessions async def add(self, rows: list[ContainerSnapshot]) -> None: if rows: async with self._sessions.begin() as session: session.add_all(rows) async def oldest_since(self, container: str, since: datetime) -> ContainerSnapshot | None: async with self._sessions() as session: return await session.scalar( select(ContainerSnapshot) .where(ContainerSnapshot.container == container, ContainerSnapshot.at >= since) .order_by(ContainerSnapshot.at) .limit(1) ) async def latest(self, container: str) -> ContainerSnapshot | None: async with self._sessions() as session: return await session.scalar( select(ContainerSnapshot) .where(ContainerSnapshot.container == container) .order_by(ContainerSnapshot.at.desc()) .limit(1) ) async def _snapshot(docker: DockerClient, container_id: str, at: datetime) -> ContainerSnapshot | None: try: details, stats = await asyncio.gather(docker.inspect(container_id), docker.stats(container_id)) except DockerError as e: log.warning("snapshot %s: %s", container_id[:12], e) return None return ContainerSnapshot( container=details.name, at=at, restart_count=details.restart_count, cpu_pct=stats.cpu_pct, mem_pct=round(stats.mem_pct, 1), health=details.health, ) async def collect_once(docker: DockerClient, store: SnapshotStore, now: datetime | None = None) -> int: at = now or utcnow() running = [c for c in await docker.containers() if c.state == "running"] rows = await asyncio.gather(*(_snapshot(docker, c.id, at) for c in running)) snapshots = [r for r in rows if r is not None] await store.add(snapshots) return len(snapshots) async def snapshot_loop(docker: DockerClient, store: SnapshotStore, interval: float) -> None: failing = False while True: try: await collect_once(docker, store) if failing: log.info("snapshots: docker reachable again") failing = False except DockerError as e: if not failing: # one line per outage, not one every 15 s log.warning("snapshots: %s", e) failing = True except Exception: # a broken round must not kill the bot log.exception("snapshots failed") await asyncio.sleep(interval)