feat(alerts): алерты всей команде — детекторы, дедупликация, список (#15)
CI / ruff · mypy · pytest (pull_request) Successful in 4m49s
CI / ruff · mypy · pytest (pull_request) Successful in 4m49s
- детекторы по состоянию после каждого раунда снапшотов: краш, OOM, цикл рестартов, unhealthy, перегрузка ≥ 1 мин, мало места, Docker недоступен; пороги в config/alerts.yaml - AlertService: ключ kind+service, повтор в 10 мин — счётчик в исходном сообщении, «✅ восстановлено» ответом; разовые события (🛡) со счётчиком - рассылка всем активным (🛡 — только админам), темп ~25/с, RetryAfter, заблокировавший бота пропускается; alert_deliveries для правок и ответов - «🚨 Алерты»: активные + 7 дней по 10, карточка; кнопки «📜 Логи», «🩺 Статус» под алертом открывают новое сообщение, а не затирают алерт - 🛡 атаки через агента идут через общий сервис алертов - убраны заглушки «Раздел в разработке» — у всех разделов свои роутеры Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,205 @@
|
||||
"""«🚨 Алерты»: the alert message itself, the list (active first, then 7 days, 10 per page) and cards (SPEC AL-3, AL-6).
|
||||
|
||||
Buttons under an alert open «📜 Логи» / «🩺 Статус» as a new message: editing would wipe the alert out of the chat.
|
||||
"""
|
||||
|
||||
import html
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from enum import StrEnum
|
||||
from typing import Any
|
||||
|
||||
from aiogram import F, Router
|
||||
from aiogram.enums import ParseMode
|
||||
from aiogram.exceptions import TelegramBadRequest
|
||||
from aiogram.filters.callback_data import CallbackData
|
||||
from aiogram.fsm.context import FSMContext
|
||||
from aiogram.types import CallbackQuery, InlineKeyboardMarkup, Message
|
||||
from aiogram.utils.keyboard import InlineKeyboardBuilder
|
||||
|
||||
from app.alerts.service import AlertService
|
||||
from app.bot.menu import Nav, NavAction
|
||||
from app.bot.routers.logs import container_keyboard
|
||||
from app.bot.routers.status import render_card as status_card
|
||||
from app.db.models import Alert, AlertKind, as_utc, utcnow
|
||||
from app.runtime.docker import DockerClient, DockerError
|
||||
from app.texts import Texts
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
PER_PAGE = 10
|
||||
LOG_CHARS = 3000
|
||||
|
||||
|
||||
class AlertsAct(StrEnum):
|
||||
LIST = "l"
|
||||
CARD = "c"
|
||||
LOGS = "g"
|
||||
STATUS = "s"
|
||||
|
||||
|
||||
class AlertsCb(CallbackData, prefix="al"):
|
||||
a: AlertsAct
|
||||
p: int = 0 # page
|
||||
id: int = 0 # alert id
|
||||
c: str = "" # 12-char container id prefix for LOGS / STATUS
|
||||
|
||||
|
||||
def title(texts: Texts, alert: Alert) -> str:
|
||||
return texts.alerts.kinds[alert.kind]
|
||||
|
||||
|
||||
def service(texts: Texts, alert: Alert) -> str:
|
||||
return html.escape(alert.service or texts.alerts.host)
|
||||
|
||||
|
||||
def detail(texts: Texts, alert: Alert) -> str:
|
||||
values: defaultdict[str, Any] = defaultdict(str, alert.payload)
|
||||
return html.escape(texts.alerts.details[alert.kind].format_map(values))
|
||||
|
||||
|
||||
def _with_log(text: str, alert: Alert) -> str:
|
||||
tail = str(alert.payload.get("log", ""))
|
||||
return f"{text}\n<pre>{html.escape(tail[-LOG_CHARS:])}</pre>" if tail else text
|
||||
|
||||
|
||||
def render(texts: Texts, alert: Alert) -> str:
|
||||
"""The alert message (AL-3); re-rendered with the counter on repeats (AL-5)."""
|
||||
t = texts.alerts
|
||||
text = t.message.format(
|
||||
title=title(texts, alert),
|
||||
service=service(texts, alert),
|
||||
detail=detail(texts, alert),
|
||||
opened=as_utc(alert.opened_at).strftime("%d.%m %H:%M"),
|
||||
repeats=t.repeats.format(count=alert.count) if alert.count > 1 else "",
|
||||
)
|
||||
return _with_log(text, alert)
|
||||
|
||||
|
||||
def keyboard(texts: Texts, alert: Alert, page: int | None = None) -> InlineKeyboardMarkup:
|
||||
"""page=None: under the alert message («📋 Список алертов»); a number: on a card («⬅️ К списку»)."""
|
||||
t = texts.alerts
|
||||
kb = InlineKeyboardBuilder()
|
||||
cid = str(alert.payload.get("container_id", ""))[:12]
|
||||
if cid:
|
||||
kb.button(text=t.logs_button, callback_data=AlertsCb(a=AlertsAct.LOGS, c=cid))
|
||||
kb.button(text=t.status_button, callback_data=AlertsCb(a=AlertsAct.STATUS, c=cid))
|
||||
back = t.list_button if page is None else t.to_list
|
||||
kb.button(text=back, callback_data=AlertsCb(a=AlertsAct.LIST, p=page or 0))
|
||||
kb.adjust(2, 1)
|
||||
return kb.as_markup()
|
||||
|
||||
|
||||
def render_list(
|
||||
texts: Texts, alerts: list[Alert], total: int, active: int, page: int
|
||||
) -> tuple[str, InlineKeyboardMarkup]:
|
||||
t = texts.alerts
|
||||
pages = max((total + PER_PAGE - 1) // PER_PAGE, 1)
|
||||
lines = [t.list_title.format(active=active, total=total)]
|
||||
if not alerts:
|
||||
lines += ["", t.list_empty]
|
||||
kb = InlineKeyboardBuilder()
|
||||
for a in alerts:
|
||||
line = t.list_line.format(
|
||||
mark=t.active_mark if a.resolved_at is None else t.resolved_mark,
|
||||
title=title(texts, a),
|
||||
service=a.service or t.host,
|
||||
time=as_utc(a.last_at).strftime("%d.%m %H:%M"),
|
||||
repeats=t.repeats.format(count=a.count) if a.count > 1 else "",
|
||||
)
|
||||
kb.button(text=line[:64], callback_data=AlertsCb(a=AlertsAct.CARD, p=page, id=a.id))
|
||||
nav = []
|
||||
if page > 0:
|
||||
kb.button(text=t.prev, callback_data=AlertsCb(a=AlertsAct.LIST, p=page - 1))
|
||||
nav.append(1)
|
||||
if page + 1 < pages:
|
||||
kb.button(text=t.next, callback_data=AlertsCb(a=AlertsAct.LIST, p=page + 1))
|
||||
nav.append(1)
|
||||
kb.button(text=texts.menu.back, callback_data=Nav(action=NavAction.BACK))
|
||||
kb.adjust(*([1] * len(alerts)), *([len(nav)] if nav else []), 1)
|
||||
if pages > 1:
|
||||
lines.append(t.page.format(page=page + 1, pages=pages))
|
||||
return "\n".join(lines), kb.as_markup()
|
||||
|
||||
|
||||
def render_alert_card(texts: Texts, a: Alert, page: int) -> tuple[str, InlineKeyboardMarkup]:
|
||||
t = texts.alerts
|
||||
text = t.card.format(
|
||||
title=title(texts, a),
|
||||
service=service(texts, a),
|
||||
detail=detail(texts, a),
|
||||
opened=as_utc(a.opened_at).strftime("%d.%m %H:%M"),
|
||||
resolved=as_utc(a.resolved_at).strftime("%d.%m %H:%M UTC") if a.resolved_at else t.ongoing,
|
||||
count=a.count,
|
||||
)
|
||||
return _with_log(text, a), keyboard(texts, a, page)
|
||||
|
||||
|
||||
def build_router(texts: Texts, alerts: AlertService | None, docker: DockerClient | None = None) -> Router:
|
||||
router = Router(name="alerts")
|
||||
t = texts.alerts
|
||||
|
||||
async def list_screen(page: int) -> tuple[str, InlineKeyboardMarkup]:
|
||||
if alerts is None:
|
||||
return render_list(texts, [], 0, 0, 0)
|
||||
rows, total = await alerts.page(page, PER_PAGE)
|
||||
if not rows and page > 0: # the history shrank since the button was drawn
|
||||
page = 0
|
||||
rows, total = await alerts.page(0, PER_PAGE)
|
||||
return render_list(texts, rows, total, await alerts.active_count(), page)
|
||||
|
||||
@router.message(F.text == texts.menu.alerts)
|
||||
async def open_alerts(message: Message, state: FSMContext) -> None:
|
||||
await state.clear()
|
||||
text, kb = await list_screen(0)
|
||||
await message.answer(text, reply_markup=kb)
|
||||
|
||||
@router.callback_query(AlertsCb.filter())
|
||||
async def on_action(query: CallbackQuery, callback_data: AlertsCb) -> None:
|
||||
if not isinstance(query.message, Message):
|
||||
await query.answer()
|
||||
return
|
||||
msg = query.message
|
||||
# Pressed under an alert: never overwrite it — answer with a new message
|
||||
rows = msg.reply_markup.inline_keyboard if msg.reply_markup else []
|
||||
under_alert = any(b.text == t.list_button for row in rows for b in row)
|
||||
match callback_data.a:
|
||||
case AlertsAct.LOGS | AlertsAct.STATUS:
|
||||
try:
|
||||
if docker is None:
|
||||
raise DockerError("docker is not configured (DOCKER_URL)")
|
||||
c = await docker.get(callback_data.c)
|
||||
if callback_data.a is AlertsAct.LOGS:
|
||||
await msg.answer(
|
||||
texts.logs.container.format(name=c.name), reply_markup=container_keyboard(texts, c.short_id)
|
||||
)
|
||||
else:
|
||||
details = await docker.inspect(c.id)
|
||||
stats = await docker.stats(c.id) if details.state == "running" else None
|
||||
text, kb = status_card(texts, details, stats, utcnow())
|
||||
await msg.answer(text, reply_markup=kb)
|
||||
except DockerError as e:
|
||||
log.warning("alerts: %s", e)
|
||||
await query.answer(texts.status.unavailable, show_alert=True)
|
||||
return
|
||||
await query.answer()
|
||||
return
|
||||
case AlertsAct.CARD:
|
||||
alert = await alerts.get(callback_data.id) if alerts else None
|
||||
if alert is None or alert.kind is AlertKind.AGENT_ATTACK: # 🛡 lives only in the admins' chats
|
||||
await query.answer(t.gone, show_alert=True)
|
||||
return
|
||||
text, kb = render_alert_card(texts, alert, callback_data.p)
|
||||
case AlertsAct.LIST:
|
||||
text, kb = await list_screen(max(callback_data.p, 0))
|
||||
if under_alert:
|
||||
await msg.answer(text, parse_mode=ParseMode.HTML, reply_markup=kb)
|
||||
else:
|
||||
try:
|
||||
await msg.edit_text(text, parse_mode=ParseMode.HTML, reply_markup=kb)
|
||||
except TelegramBadRequest as e:
|
||||
if "message is not modified" not in str(e):
|
||||
raise
|
||||
await query.answer()
|
||||
|
||||
return router
|
||||
@@ -1,12 +1,12 @@
|
||||
"""/start (incl. invite deep links), /help, /menu, main menu buttons and in-section navigation."""
|
||||
|
||||
from aiogram import F, Router
|
||||
from aiogram import Router
|
||||
from aiogram.filters import Command, CommandObject, CommandStart
|
||||
from aiogram.fsm.context import FSMContext
|
||||
from aiogram.types import CallbackQuery, Message
|
||||
|
||||
from app.access.service import AccessService, RedeemStatus
|
||||
from app.bot.menu import Nav, NavAction, Section, main_menu, section_by_button, section_nav
|
||||
from app.bot.menu import Nav, NavAction, main_menu
|
||||
from app.bot.middlewares.access import tg_user
|
||||
from app.db.models import Role, User
|
||||
from app.texts import Texts
|
||||
@@ -14,10 +14,6 @@ from app.texts import Texts
|
||||
|
||||
def build_router(texts: Texts, access: AccessService) -> Router:
|
||||
router = Router(name="common")
|
||||
sections = section_by_button(texts)
|
||||
section_text = {
|
||||
Section.ALERTS: texts.section.alerts,
|
||||
}
|
||||
|
||||
@router.message(CommandStart())
|
||||
async def start(message: Message, command: CommandObject, state: FSMContext, member: User | None) -> None:
|
||||
@@ -46,12 +42,6 @@ def build_router(texts: Texts, access: AccessService) -> Router:
|
||||
async def help_(message: Message) -> None:
|
||||
await message.answer(texts.help.text)
|
||||
|
||||
# Sections with their own routers (members, commit) are matched there first
|
||||
@router.message(F.text.in_({k for k, v in sections.items() if v in section_text}))
|
||||
async def open_section(message: Message, state: FSMContext) -> None:
|
||||
await state.clear()
|
||||
await message.answer(section_text[sections[message.text or ""]], reply_markup=section_nav(texts))
|
||||
|
||||
@router.callback_query(Nav.filter())
|
||||
async def navigate(query: CallbackQuery, callback_data: Nav, state: FSMContext) -> None:
|
||||
# At a section's root "Back" and "Cancel" both close it; nested screens will override "Back" per section.
|
||||
|
||||
+15
-22
@@ -6,19 +6,19 @@ import logging
|
||||
from datetime import timedelta
|
||||
from enum import StrEnum
|
||||
|
||||
from aiogram import Bot, F, Router
|
||||
from aiogram import F, Router
|
||||
from aiogram.enums import ParseMode
|
||||
from aiogram.exceptions import TelegramAPIError
|
||||
from aiogram.filters.callback_data import CallbackData
|
||||
from aiogram.fsm.context import FSMContext
|
||||
from aiogram.fsm.state import State, StatesGroup
|
||||
from aiogram.types import BufferedInputFile, CallbackQuery, InlineKeyboardMarkup, Message
|
||||
from aiogram.utils.keyboard import InlineKeyboardBuilder
|
||||
|
||||
from app.access.service import AccessService
|
||||
from app.agent.service import Agent, Outcome
|
||||
from app.alerts.detectors import Condition
|
||||
from app.alerts.service import AlertService
|
||||
from app.bot.menu import Nav, NavAction, free_input
|
||||
from app.db.models import LogMode, Role, User, utcnow
|
||||
from app.db.models import AlertKind, LogMode, User, utcnow
|
||||
from app.logs.grep import GrepError, GrepErrorCode, GrepResult, parse_grep, run, to_command
|
||||
from app.logs.journal import LogJournal
|
||||
from app.runtime.docker import Container, DockerClient, DockerError
|
||||
@@ -146,7 +146,7 @@ def build_router(
|
||||
docker: DockerClient | None,
|
||||
journal: LogJournal,
|
||||
agent: Agent | None = None,
|
||||
access: AccessService | None = None,
|
||||
alerts: AlertService | None = None,
|
||||
) -> Router:
|
||||
router = Router(name="logs")
|
||||
t = texts.logs
|
||||
@@ -156,23 +156,16 @@ def build_router(
|
||||
raise DockerError("docker is not configured (DOCKER_URL)")
|
||||
return docker
|
||||
|
||||
async def alert_admins(bot: Bot, member: User, prompt: str, stage: str, reason: str) -> None:
|
||||
"""🛡 to the admins only (SPEC AL-2, L-12). The asker sees just a polite refusal."""
|
||||
if access is None:
|
||||
async def alert_admins(member: User, prompt: str, stage: str, reason: str) -> None:
|
||||
"""🛡 to the admins only (SPEC AL-2, L-12); repeats by the same member bump one alert's counter."""
|
||||
if alerts is None:
|
||||
return
|
||||
text = t.attack_alert.format(
|
||||
who=html.escape(who(member)),
|
||||
stage=stage,
|
||||
reason=html.escape(reason),
|
||||
prompt=html.escape(prompt[:1000]),
|
||||
condition = Condition(
|
||||
AlertKind.AGENT_ATTACK,
|
||||
f"user {member.tg_id}",
|
||||
{"who": who(member), "stage": stage, "reason": reason, "log": prompt[:1000]},
|
||||
)
|
||||
for admin in await access.list_members():
|
||||
if admin.role is not Role.ADMIN:
|
||||
continue
|
||||
try:
|
||||
await bot.send_message(admin.tg_id, text, parse_mode=ParseMode.HTML)
|
||||
except TelegramAPIError as e: # one blocked admin must not hide the alert from the others
|
||||
log.warning("attack alert to %s failed: %s", admin.tg_id, e)
|
||||
await alerts.event(condition, admins_only=True)
|
||||
|
||||
@router.message(F.text == texts.menu.logs)
|
||||
async def open_logs(message: Message, state: FSMContext) -> None:
|
||||
@@ -275,7 +268,7 @@ def build_router(
|
||||
await deliver(message, texts, head, query.count, result, container_keyboard(texts, c.short_id), c.name)
|
||||
|
||||
@router.message(LogsFlow.agent, F.text, free_input(texts))
|
||||
async def got_question(message: Message, state: FSMContext, member: User, bot: Bot) -> None:
|
||||
async def got_question(message: Message, state: FSMContext, member: User) -> None:
|
||||
if agent is None: # pragma: no cover - the button doesn't open this state without an agent
|
||||
await state.clear()
|
||||
return
|
||||
@@ -296,7 +289,7 @@ def build_router(
|
||||
match answer.outcome:
|
||||
case Outcome.ATTACK:
|
||||
await message.answer(t.agent_refused, reply_markup=kb)
|
||||
await alert_admins(bot, member, prompt, answer.stage, answer.reason)
|
||||
await alert_admins(member, prompt, answer.stage, answer.reason)
|
||||
return
|
||||
case Outcome.QUOTA:
|
||||
await message.answer(t.agent_quota.format(quota=agent.quota), reply_markup=kb)
|
||||
|
||||
Reference in New Issue
Block a user