Files
ElebobaandClaude Opus 5.5 79ea5ce2a5
CI / ruff · mypy · pytest (pull_request) Successful in 24m47s
fix(alerts): краш при restart-политике, гистерезис закрытия, редкие правки
Найдено смоуком на живых контейнерах (examples/, #19):
- контейнер с restart: on-failure поднимается за секунду — между раундами
  его не видно в exited; краш теперь распознаётся и по росту RestartCount
  (сравнение со снапшотами до текущего раунда), повторы — счётчик
- алерт закрывается, только когда условия нет resolve_after (60 с): иначе
  флапающий сервис получал бы «✅ восстановлено» после каждого раунда
- правки сообщений со счётчиком — не чаще раза в минуту на алерт

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-17 15:52:54 +03:00

213 lines
8.4 KiB
Python

import re
from collections.abc import AsyncIterator
from datetime import timedelta
from pathlib import Path
import pytest
from aiogram import Bot, Dispatcher
from aiogram.methods import AnswerCallbackQuery, EditMessageText, SendMessage, SendPhoto
from app.access.service import AccessService
from app.alerts import broadcast
from app.alerts.broadcast import TelegramNotifier
from app.alerts.detectors import load_alerts_config
from app.alerts.monitor import AlertMonitor, free_disk_pct
from app.alerts.service import AlertService
from app.bot.routers.alerts import PER_PAGE, AlertsAct, AlertsCb
from app.bot.routers.members import MembersAction, MembersCb
from app.db.models import utcnow
from app.db.session import SessionMaker
from app.main import build_dispatcher
from app.repo.git import UploadsRepo
from app.runtime.docker import DockerClient
from app.runtime.snapshots import SnapshotStore, collect_once
from app.texts import Texts
from tests.conftest import ADMIN, ROOT
from tests.fakes import TgClient
from tests.runtime.fake_docker import FakeDocker, now_minus
MEMBER, BLOCKED = 5005, 6006
CFG = load_alerts_config(ROOT / "config").model_copy(update={"resolve_after": 0}) # hysteresis: tests/alerts
@pytest.fixture(autouse=True)
def _no_pacing(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(broadcast, "PACE", 0)
@pytest.fixture
def fake() -> FakeDocker:
fake = FakeDocker()
fake.add("web", logs=[(now_minus(1), f"line {i}") for i in range(40)])
return fake
@pytest.fixture
async def docker(fake: FakeDocker) -> AsyncIterator[DockerClient]:
async with fake.client() as client:
yield client
class Env:
def __init__(self, tg: TgClient, alerts: AlertService, monitor: AlertMonitor, store: SnapshotStore) -> None:
self.tg, self.alerts, self.monitor, self.store = tg, alerts, monitor, store
self.disk_free: float | None = 50.0
@pytest.fixture
async def env(
texts: Texts, sessions: SessionMaker, access: AccessService, repo: UploadsRepo, docker: DockerClient
) -> Env:
store = SnapshotStore(sessions)
holder: dict[str, object] = {}
def dispatcher(bot: Bot) -> Dispatcher:
alerts = AlertService(
sessions, TelegramNotifier(bot, access, sessions, texts), CFG, lambda cid, n: docker.logs(cid, tail=n)
)
holder["alerts"] = alerts
return build_dispatcher(texts, sessions, access, repo, docker=docker, alerts=alerts)
tg = TgClient(dispatcher)
alerts = holder["alerts"]
assert isinstance(alerts, AlertService)
e = Env(tg, alerts, None, store) # type: ignore[arg-type]
e.monitor = AlertMonitor(alerts, store, CFG, disk=lambda _: e.disk_free)
for user in (MEMBER, BLOCKED):
await tg.press(ADMIN, MembersCb(action=MembersAction.INVITE).pack())
token = re.search(r"start=([A-Za-z0-9_-]+)", tg.session.sent(SendPhoto)[-1].caption or "")
assert token
await tg.send(user, f"/start {token.group(1)}")
tg.session.blocked.add(BLOCKED)
tg.session.requests.clear()
return e
async def round_(env: Env, docker: DockerClient) -> None:
await env.monitor.on_round(await collect_once(docker, env.store))
async def test_crash_broadcast_to_everyone_and_recovery_reply(env: Env, fake: FakeDocker, docker: DockerClient) -> None:
web = next(iter(fake.containers.values()))
web.state, web.health, web.exit_code = "exited", None, 1
await round_(env, docker)
await round_(env, docker) # still down: no second message
sent = env.tg.session.sent(SendMessage)
assert sorted(m.chat_id for m in sent) == [ADMIN, MEMBER, BLOCKED] # tried everyone, the blocked one quietly
msg = sent[0]
assert msg.text.startswith("💥 Краш · <b>web</b>\nКонтейнер вышел с кодом 1.")
assert "line 39" in msg.text and "line 9\n" not in msg.text # last 30 lines
buttons = [b.text for row in msg.reply_markup.inline_keyboard for b in row]
assert buttons == ["📜 Логи", "🩺 Статус", "📋 Список алертов"]
env.tg.session.requests.clear()
web.state, web.health, web.exit_code = "running", "healthy", 0
await round_(env, docker)
recovered = env.tg.session.sent(SendMessage)
assert sorted(m.chat_id for m in recovered) == [ADMIN, MEMBER] # only where the alert was delivered
assert all(m.text.startswith("✅ Восстановлено · 💥 Краш · web") for m in recovered)
assert all(m.reply_parameters is not None for m in recovered)
async def test_flap_edits_counter_instead_of_new_message(env: Env, fake: FakeDocker, docker: DockerClient) -> None:
web = next(iter(fake.containers.values()))
web.health = "unhealthy"
await round_(env, docker)
web.health = "healthy"
await round_(env, docker)
env.tg.session.requests.clear()
web.health = "unhealthy"
await round_(env, docker)
assert not env.tg.session.sent(SendMessage)
edits = env.tg.session.sent(EditMessageText)
assert len(edits) == 2 and all("повторов: 2" in e.text for e in edits)
async def test_docker_down_and_disk_low(env: Env, fake: FakeDocker, docker: DockerClient) -> None:
env.disk_free = 3.0
await round_(env, docker)
disk, *_ = env.tg.session.sent(SendMessage)
assert disk.text.startswith("💾 Мало места на хосте · <b>хост</b>\nСвободно 3.0% диска.")
env.tg.session.requests.clear()
clock = [utcnow()]
env.monitor._clock = lambda: clock[0]
await env.monitor.on_round(None)
clock[0] += timedelta(minutes=6)
await env.monitor.on_round(None)
down = env.tg.session.sent(SendMessage)
assert len(down) == 3 and down[0].text.startswith("🔌 Зависимость недоступна · <b>docker</b>")
async def test_list_and_card(env: Env, fake: FakeDocker, docker: DockerClient, texts: Texts) -> None:
web = next(iter(fake.containers.values()))
web.state, web.health, web.exit_code = "exited", None, 2
await round_(env, docker)
env.tg.session.requests.clear()
await env.tg.send(MEMBER, texts.menu.alerts)
[listing] = env.tg.session.sent(SendMessage)
assert listing.text.startswith("🚨 Алерты · активных: 1 · всего за 7 дней: 1")
[first, _back] = [row[0] for row in listing.reply_markup.inline_keyboard]
assert first.text.startswith("🔴 💥 Краш · web")
await env.tg.press(MEMBER, first.callback_data, message_text=listing.text)
[card] = env.tg.session.sent(EditMessageText)
assert "Конец: продолжается" in card.text and "Повторов: 1" in card.text
async def test_buttons_under_alert_open_new_messages(
env: Env, fake: FakeDocker, docker: DockerClient, texts: Texts
) -> None:
web = next(iter(fake.containers.values()))
web.health = "unhealthy"
await round_(env, docker)
alert_msg = env.tg.session.sent(SendMessage)[0]
env.tg.session.requests.clear()
cid = web.id[:12]
for act in (AlertsAct.LOGS, AlertsAct.STATUS, AlertsAct.LIST):
await env.tg.press_markup(MEMBER, AlertsCb(a=act, c=cid).pack(), alert_msg.reply_markup)
assert not env.tg.session.sent(EditMessageText) # the alert itself stays intact
logs_msg, status_msg, list_msg = env.tg.session.sent(SendMessage)
assert logs_msg.text.startswith("📜 Логи · web")
assert status_msg.text.startswith("🔴 web")
assert list_msg.text.startswith("🚨 Алерты")
async def test_pagination(env: Env, texts: Texts) -> None:
from app.alerts.detectors import MONITORED, Condition
from app.db.models import AlertKind
now = utcnow()
for i in range(PER_PAGE + 3):
await env.alerts.sync([Condition(AlertKind.UNHEALTHY, f"s{i:02}")], frozenset({AlertKind.UNHEALTHY}), now)
env.tg.session.requests.clear()
await env.tg.press(MEMBER, AlertsCb(a=AlertsAct.LIST, p=1).pack())
[page2] = env.tg.session.sent(EditMessageText)
assert "Стр. 2 из 2" in page2.text
assert len(page2.reply_markup.inline_keyboard) == 3 + 1 + 1 # 3 alerts, ◀️, back
assert MONITORED
async def test_missing_or_admin_only_alert_card(env: Env, texts: Texts) -> None:
await env.tg.press(MEMBER, AlertsCb(a=AlertsAct.CARD, id=999).pack())
[ack] = env.tg.session.sent(AnswerCallbackQuery)
assert ack.text == texts.alerts.gone
def test_free_disk_pct(tmp_path: Path) -> None:
pct = free_disk_pct(tmp_path)
assert pct is not None and 0 <= pct <= 100
assert free_disk_pct(tmp_path / "nope") is None