feat(status): раздел «Статус» — контейнеры, карточка, проверки (#14)
CI / ruff · mypy · pytest (pull_request) Successful in 1m11s
CI / ruff · mypy · pytest (pull_request) Successful in 1m11s
- docker-socket-proxy в Compose (только GET /containers/*), сокет только у него - app/runtime/docker.py: read-only клиент; видны только контейнеры с меткой tgit.managed=true, чужие id отклоняются до запроса - app/runtime/checks.py: running, health, рестарты за окно, CPU/RAM, ошибки в логе, HTTP-проба; у каждой проверки свой таймаут, отчёт всегда полный - app/runtime/snapshots.py: снапшоты раз в 15 с в container_snapshots - бот: список с 🟢🟡🔴⚪, карточка, «🩺 Запустить проверки», «🔄 Обновить» - деплой ставит метки tgit.sha / tgit.deployed_at на контейнер бота - имя джоба CD без неотрисованного выражения Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,87 @@
|
||||
"""Every 15 s: restart count, CPU and RAM of each running managed container → `container_snapshots`.
|
||||
|
||||
The history answers «are restarts growing over the last 5 minutes» (S-3) and feeds alerts (#15).
|
||||
Old rows are removed by `app.db.maintenance`.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.db.models import ContainerSnapshot, utcnow
|
||||
from app.db.session import SessionMaker
|
||||
from app.runtime.docker import DockerClient, DockerError
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SnapshotStore:
|
||||
def __init__(self, sessions: SessionMaker) -> None:
|
||||
self._sessions = sessions
|
||||
|
||||
async def add(self, rows: list[ContainerSnapshot]) -> None:
|
||||
if rows:
|
||||
async with self._sessions.begin() as session:
|
||||
session.add_all(rows)
|
||||
|
||||
async def oldest_since(self, container: str, since: datetime) -> ContainerSnapshot | None:
|
||||
async with self._sessions() as session:
|
||||
return await session.scalar(
|
||||
select(ContainerSnapshot)
|
||||
.where(ContainerSnapshot.container == container, ContainerSnapshot.at >= since)
|
||||
.order_by(ContainerSnapshot.at)
|
||||
.limit(1)
|
||||
)
|
||||
|
||||
async def latest(self, container: str) -> ContainerSnapshot | None:
|
||||
async with self._sessions() as session:
|
||||
return await session.scalar(
|
||||
select(ContainerSnapshot)
|
||||
.where(ContainerSnapshot.container == container)
|
||||
.order_by(ContainerSnapshot.at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
|
||||
|
||||
async def _snapshot(docker: DockerClient, container_id: str, at: datetime) -> ContainerSnapshot | None:
|
||||
try:
|
||||
details, stats = await asyncio.gather(docker.inspect(container_id), docker.stats(container_id))
|
||||
except DockerError as e:
|
||||
log.warning("snapshot %s: %s", container_id[:12], e)
|
||||
return None
|
||||
return ContainerSnapshot(
|
||||
container=details.name,
|
||||
at=at,
|
||||
restart_count=details.restart_count,
|
||||
cpu_pct=stats.cpu_pct,
|
||||
mem_pct=round(stats.mem_pct, 1),
|
||||
health=details.health,
|
||||
)
|
||||
|
||||
|
||||
async def collect_once(docker: DockerClient, store: SnapshotStore, now: datetime | None = None) -> int:
|
||||
at = now or utcnow()
|
||||
running = [c for c in await docker.containers() if c.state == "running"]
|
||||
rows = await asyncio.gather(*(_snapshot(docker, c.id, at) for c in running))
|
||||
snapshots = [r for r in rows if r is not None]
|
||||
await store.add(snapshots)
|
||||
return len(snapshots)
|
||||
|
||||
|
||||
async def snapshot_loop(docker: DockerClient, store: SnapshotStore, interval: float) -> None:
|
||||
failing = False
|
||||
while True:
|
||||
try:
|
||||
await collect_once(docker, store)
|
||||
if failing:
|
||||
log.info("snapshots: docker reachable again")
|
||||
failing = False
|
||||
except DockerError as e:
|
||||
if not failing: # one line per outage, not one every 15 s
|
||||
log.warning("snapshots: %s", e)
|
||||
failing = True
|
||||
except Exception: # a broken round must not kill the bot
|
||||
log.exception("snapshots failed")
|
||||
await asyncio.sleep(interval)
|
||||
Reference in New Issue
Block a user