CI / ruff · mypy · pytest (pull_request) Successful in 24m47s
Найдено смоуком на живых контейнерах (examples/, #19): - контейнер с restart: on-failure поднимается за секунду — между раундами его не видно в exited; краш теперь распознаётся и по росту RestartCount (сравнение со снапшотами до текущего раунда), повторы — счётчик - алерт закрывается, только когда условия нет resolve_after (60 с): иначе флапающий сервис получал бы «✅ восстановлено» после каждого раунда - правки сообщений со счётчиком — не чаще раза в минуту на алерт Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
213 lines
8.4 KiB
Python
213 lines
8.4 KiB
Python
import re
|
|
from collections.abc import AsyncIterator
|
|
from datetime import timedelta
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from aiogram import Bot, Dispatcher
|
|
from aiogram.methods import AnswerCallbackQuery, EditMessageText, SendMessage, SendPhoto
|
|
|
|
from app.access.service import AccessService
|
|
from app.alerts import broadcast
|
|
from app.alerts.broadcast import TelegramNotifier
|
|
from app.alerts.detectors import load_alerts_config
|
|
from app.alerts.monitor import AlertMonitor, free_disk_pct
|
|
from app.alerts.service import AlertService
|
|
from app.bot.routers.alerts import PER_PAGE, AlertsAct, AlertsCb
|
|
from app.bot.routers.members import MembersAction, MembersCb
|
|
from app.db.models import utcnow
|
|
from app.db.session import SessionMaker
|
|
from app.main import build_dispatcher
|
|
from app.repo.git import UploadsRepo
|
|
from app.runtime.docker import DockerClient
|
|
from app.runtime.snapshots import SnapshotStore, collect_once
|
|
from app.texts import Texts
|
|
from tests.conftest import ADMIN, ROOT
|
|
from tests.fakes import TgClient
|
|
from tests.runtime.fake_docker import FakeDocker, now_minus
|
|
|
|
MEMBER, BLOCKED = 5005, 6006
|
|
CFG = load_alerts_config(ROOT / "config").model_copy(update={"resolve_after": 0}) # hysteresis: tests/alerts
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _no_pacing(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(broadcast, "PACE", 0)
|
|
|
|
|
|
@pytest.fixture
|
|
def fake() -> FakeDocker:
|
|
fake = FakeDocker()
|
|
fake.add("web", logs=[(now_minus(1), f"line {i}") for i in range(40)])
|
|
return fake
|
|
|
|
|
|
@pytest.fixture
|
|
async def docker(fake: FakeDocker) -> AsyncIterator[DockerClient]:
|
|
async with fake.client() as client:
|
|
yield client
|
|
|
|
|
|
class Env:
|
|
def __init__(self, tg: TgClient, alerts: AlertService, monitor: AlertMonitor, store: SnapshotStore) -> None:
|
|
self.tg, self.alerts, self.monitor, self.store = tg, alerts, monitor, store
|
|
self.disk_free: float | None = 50.0
|
|
|
|
|
|
@pytest.fixture
|
|
async def env(
|
|
texts: Texts, sessions: SessionMaker, access: AccessService, repo: UploadsRepo, docker: DockerClient
|
|
) -> Env:
|
|
store = SnapshotStore(sessions)
|
|
holder: dict[str, object] = {}
|
|
|
|
def dispatcher(bot: Bot) -> Dispatcher:
|
|
alerts = AlertService(
|
|
sessions, TelegramNotifier(bot, access, sessions, texts), CFG, lambda cid, n: docker.logs(cid, tail=n)
|
|
)
|
|
holder["alerts"] = alerts
|
|
return build_dispatcher(texts, sessions, access, repo, docker=docker, alerts=alerts)
|
|
|
|
tg = TgClient(dispatcher)
|
|
alerts = holder["alerts"]
|
|
assert isinstance(alerts, AlertService)
|
|
e = Env(tg, alerts, None, store) # type: ignore[arg-type]
|
|
e.monitor = AlertMonitor(alerts, store, CFG, disk=lambda _: e.disk_free)
|
|
for user in (MEMBER, BLOCKED):
|
|
await tg.press(ADMIN, MembersCb(action=MembersAction.INVITE).pack())
|
|
token = re.search(r"start=([A-Za-z0-9_-]+)", tg.session.sent(SendPhoto)[-1].caption or "")
|
|
assert token
|
|
await tg.send(user, f"/start {token.group(1)}")
|
|
tg.session.blocked.add(BLOCKED)
|
|
tg.session.requests.clear()
|
|
return e
|
|
|
|
|
|
async def round_(env: Env, docker: DockerClient) -> None:
|
|
await env.monitor.on_round(await collect_once(docker, env.store))
|
|
|
|
|
|
async def test_crash_broadcast_to_everyone_and_recovery_reply(env: Env, fake: FakeDocker, docker: DockerClient) -> None:
|
|
web = next(iter(fake.containers.values()))
|
|
web.state, web.health, web.exit_code = "exited", None, 1
|
|
|
|
await round_(env, docker)
|
|
await round_(env, docker) # still down: no second message
|
|
|
|
sent = env.tg.session.sent(SendMessage)
|
|
assert sorted(m.chat_id for m in sent) == [ADMIN, MEMBER, BLOCKED] # tried everyone, the blocked one quietly
|
|
msg = sent[0]
|
|
assert msg.text.startswith("💥 Краш · <b>web</b>\nКонтейнер вышел с кодом 1.")
|
|
assert "line 39" in msg.text and "line 9\n" not in msg.text # last 30 lines
|
|
buttons = [b.text for row in msg.reply_markup.inline_keyboard for b in row]
|
|
assert buttons == ["📜 Логи", "🩺 Статус", "📋 Список алертов"]
|
|
|
|
env.tg.session.requests.clear()
|
|
web.state, web.health, web.exit_code = "running", "healthy", 0
|
|
await round_(env, docker)
|
|
|
|
recovered = env.tg.session.sent(SendMessage)
|
|
assert sorted(m.chat_id for m in recovered) == [ADMIN, MEMBER] # only where the alert was delivered
|
|
assert all(m.text.startswith("✅ Восстановлено · 💥 Краш · web") for m in recovered)
|
|
assert all(m.reply_parameters is not None for m in recovered)
|
|
|
|
|
|
async def test_flap_edits_counter_instead_of_new_message(env: Env, fake: FakeDocker, docker: DockerClient) -> None:
|
|
web = next(iter(fake.containers.values()))
|
|
web.health = "unhealthy"
|
|
await round_(env, docker)
|
|
web.health = "healthy"
|
|
await round_(env, docker)
|
|
env.tg.session.requests.clear()
|
|
|
|
web.health = "unhealthy"
|
|
await round_(env, docker)
|
|
|
|
assert not env.tg.session.sent(SendMessage)
|
|
edits = env.tg.session.sent(EditMessageText)
|
|
assert len(edits) == 2 and all("повторов: 2" in e.text for e in edits)
|
|
|
|
|
|
async def test_docker_down_and_disk_low(env: Env, fake: FakeDocker, docker: DockerClient) -> None:
|
|
env.disk_free = 3.0
|
|
await round_(env, docker)
|
|
disk, *_ = env.tg.session.sent(SendMessage)
|
|
assert disk.text.startswith("💾 Мало места на хосте · <b>хост</b>\nСвободно 3.0% диска.")
|
|
|
|
env.tg.session.requests.clear()
|
|
clock = [utcnow()]
|
|
env.monitor._clock = lambda: clock[0]
|
|
await env.monitor.on_round(None)
|
|
clock[0] += timedelta(minutes=6)
|
|
await env.monitor.on_round(None)
|
|
|
|
down = env.tg.session.sent(SendMessage)
|
|
assert len(down) == 3 and down[0].text.startswith("🔌 Зависимость недоступна · <b>docker</b>")
|
|
|
|
|
|
async def test_list_and_card(env: Env, fake: FakeDocker, docker: DockerClient, texts: Texts) -> None:
|
|
web = next(iter(fake.containers.values()))
|
|
web.state, web.health, web.exit_code = "exited", None, 2
|
|
await round_(env, docker)
|
|
env.tg.session.requests.clear()
|
|
|
|
await env.tg.send(MEMBER, texts.menu.alerts)
|
|
[listing] = env.tg.session.sent(SendMessage)
|
|
assert listing.text.startswith("🚨 Алерты · активных: 1 · всего за 7 дней: 1")
|
|
[first, _back] = [row[0] for row in listing.reply_markup.inline_keyboard]
|
|
assert first.text.startswith("🔴 💥 Краш · web")
|
|
|
|
await env.tg.press(MEMBER, first.callback_data, message_text=listing.text)
|
|
[card] = env.tg.session.sent(EditMessageText)
|
|
assert "Конец: продолжается" in card.text and "Повторов: 1" in card.text
|
|
|
|
|
|
async def test_buttons_under_alert_open_new_messages(
|
|
env: Env, fake: FakeDocker, docker: DockerClient, texts: Texts
|
|
) -> None:
|
|
web = next(iter(fake.containers.values()))
|
|
web.health = "unhealthy"
|
|
await round_(env, docker)
|
|
alert_msg = env.tg.session.sent(SendMessage)[0]
|
|
env.tg.session.requests.clear()
|
|
cid = web.id[:12]
|
|
|
|
for act in (AlertsAct.LOGS, AlertsAct.STATUS, AlertsAct.LIST):
|
|
await env.tg.press_markup(MEMBER, AlertsCb(a=act, c=cid).pack(), alert_msg.reply_markup)
|
|
|
|
assert not env.tg.session.sent(EditMessageText) # the alert itself stays intact
|
|
logs_msg, status_msg, list_msg = env.tg.session.sent(SendMessage)
|
|
assert logs_msg.text.startswith("📜 Логи · web")
|
|
assert status_msg.text.startswith("🔴 web")
|
|
assert list_msg.text.startswith("🚨 Алерты")
|
|
|
|
|
|
async def test_pagination(env: Env, texts: Texts) -> None:
|
|
from app.alerts.detectors import MONITORED, Condition
|
|
from app.db.models import AlertKind
|
|
|
|
now = utcnow()
|
|
for i in range(PER_PAGE + 3):
|
|
await env.alerts.sync([Condition(AlertKind.UNHEALTHY, f"s{i:02}")], frozenset({AlertKind.UNHEALTHY}), now)
|
|
env.tg.session.requests.clear()
|
|
|
|
await env.tg.press(MEMBER, AlertsCb(a=AlertsAct.LIST, p=1).pack())
|
|
|
|
[page2] = env.tg.session.sent(EditMessageText)
|
|
assert "Стр. 2 из 2" in page2.text
|
|
assert len(page2.reply_markup.inline_keyboard) == 3 + 1 + 1 # 3 alerts, ◀️, back
|
|
assert MONITORED
|
|
|
|
|
|
async def test_missing_or_admin_only_alert_card(env: Env, texts: Texts) -> None:
|
|
await env.tg.press(MEMBER, AlertsCb(a=AlertsAct.CARD, id=999).pack())
|
|
|
|
[ack] = env.tg.session.sent(AnswerCallbackQuery)
|
|
assert ack.text == texts.alerts.gone
|
|
|
|
|
|
def test_free_disk_pct(tmp_path: Path) -> None:
|
|
pct = free_disk_pct(tmp_path)
|
|
assert pct is not None and 0 <= pct <= 100
|
|
assert free_disk_pct(tmp_path / "nope") is None
|