feat(logs): раздел «Логи» — хвост и свой grep (#16)
CI / ruff · mypy · pytest (pull_request) Successful in 9m40s
CI / ruff · mypy · pytest (pull_request) Successful in 9m40s
- app/logs/grep.py: подмножество grep (шаблон, -i -v -E -c -C N), разбор shlex без исполнения, BRE → regex, таймаут на строку и бюджет на поиск, лимиты 200 символов / 500 строк вывода - app/logs/journal.py: каждый поиск в log_queries (L-6) - бот: выбор контейнера, хвост 50/200 строк или 5 мин/1 ч/24 ч, свой grep по 24 ч, длинный результат — файлом .log (L-5) - логи Docker: не больше 50 000 строк и 5 МБ, сохраняются новейшие - fix: кнопки меню и /menu, /help больше не проглатываются шагами ввода (имя файла, содержимое, сообщение коммита, grep) — фильтр free_input Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -2,9 +2,11 @@
|
||||
|
||||
from enum import StrEnum
|
||||
|
||||
from aiogram import F
|
||||
from aiogram.filters.callback_data import CallbackData
|
||||
from aiogram.types import InlineKeyboardMarkup, ReplyKeyboardMarkup
|
||||
from aiogram.utils.keyboard import InlineKeyboardBuilder, ReplyKeyboardBuilder
|
||||
from magic_filter import MagicFilter
|
||||
|
||||
from app.texts import Texts
|
||||
|
||||
@@ -38,6 +40,12 @@ def section_by_button(texts: Texts) -> dict[str, Section]:
|
||||
}
|
||||
|
||||
|
||||
def free_input(texts: Texts) -> MagicFilter:
|
||||
"""Input for a dialog step — anything but a main-menu button or a bot command: those must work from any step,
|
||||
otherwise «🩺 Статус» pressed while the bot waits for a file name becomes the file name."""
|
||||
return ~F.text.in_(set(section_by_button(texts))) & ~F.text.regexp(r"^/(start|menu|help)(@\w+)?(\s|$)")
|
||||
|
||||
|
||||
def main_menu(texts: Texts, *, is_admin: bool) -> ReplyKeyboardMarkup:
|
||||
m = texts.menu
|
||||
kb = ReplyKeyboardBuilder()
|
||||
|
||||
@@ -16,7 +16,7 @@ from aiogram.fsm.state import State, StatesGroup
|
||||
from aiogram.types import BufferedInputFile, CallbackQuery, InlineKeyboardMarkup, Message
|
||||
from aiogram.utils.keyboard import InlineKeyboardBuilder
|
||||
|
||||
from app.bot.menu import Nav, NavAction
|
||||
from app.bot.menu import Nav, NavAction, free_input
|
||||
from app.db.models import ChangeOp, User
|
||||
from app.repo.drafts import DraftFull, DraftService, DraftView
|
||||
from app.repo.git import Entry, GitError, UploadsRepo
|
||||
@@ -520,7 +520,7 @@ def build_router(
|
||||
|
||||
# ---- text input --------------------------------------------------------------------------------------------
|
||||
|
||||
@router.message(Flow.file_name, F.text)
|
||||
@router.message(Flow.file_name, F.text, free_input(texts))
|
||||
async def got_file_name(message: Message, state: FSMContext, member: User) -> None:
|
||||
data = await state.get_data()
|
||||
directory = data.get("commit_path", "")
|
||||
@@ -541,7 +541,7 @@ def build_router(
|
||||
await state.update_data(commit_target=path, commit_op=ChangeOp.ADD.value)
|
||||
await message.answer(t.ask_content.format(path=path), reply_markup=to_dir_keyboard(texts))
|
||||
|
||||
@router.message(Flow.dir_name, F.text)
|
||||
@router.message(Flow.dir_name, F.text, free_input(texts))
|
||||
async def got_dir_name(message: Message, state: FSMContext, member: User) -> None:
|
||||
data = await state.get_data()
|
||||
directory = data.get("commit_path", "")
|
||||
@@ -562,7 +562,7 @@ def build_router(
|
||||
text, kb = await dir_screen(state, member.tg_id, path, 0)
|
||||
await message.answer(text, reply_markup=kb)
|
||||
|
||||
@router.message(Flow.message, F.text)
|
||||
@router.message(Flow.message, F.text, free_input(texts))
|
||||
async def got_message(message: Message, state: FSMContext, member: User) -> None:
|
||||
try:
|
||||
text = check_message(message.text or "")
|
||||
@@ -579,7 +579,7 @@ def build_router(
|
||||
screen, kb = await draft_screen(state, member.tg_id)
|
||||
await message.answer(screen, reply_markup=kb)
|
||||
|
||||
@router.message(Flow.content, F.text | F.document)
|
||||
@router.message(Flow.content, F.text | F.document, free_input(texts))
|
||||
async def got_content(message: Message, state: FSMContext, member: User, bot: Bot) -> None:
|
||||
data = await state.get_data()
|
||||
target: str = data["commit_target"]
|
||||
|
||||
@@ -16,7 +16,6 @@ def build_router(texts: Texts, access: AccessService) -> Router:
|
||||
router = Router(name="common")
|
||||
sections = section_by_button(texts)
|
||||
section_text = {
|
||||
Section.LOGS: texts.section.logs,
|
||||
Section.ALERTS: texts.section.alerts,
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
"""«📜 Логи»: pick a container, then its tail or your own grep (SPEC L-1…L-3, L-5, L-6). The agent is #18."""
|
||||
|
||||
import asyncio
|
||||
import html
|
||||
import logging
|
||||
from datetime import timedelta
|
||||
from enum import StrEnum
|
||||
|
||||
from aiogram import F, Router
|
||||
from aiogram.enums import ParseMode
|
||||
from aiogram.filters.callback_data import CallbackData
|
||||
from aiogram.fsm.context import FSMContext
|
||||
from aiogram.fsm.state import State, StatesGroup
|
||||
from aiogram.types import BufferedInputFile, CallbackQuery, InlineKeyboardMarkup, Message
|
||||
from aiogram.utils.keyboard import InlineKeyboardBuilder
|
||||
|
||||
from app.bot.menu import Nav, NavAction, free_input
|
||||
from app.db.models import LogMode, User, utcnow
|
||||
from app.logs.grep import GrepError, GrepErrorCode, GrepResult, parse_grep, run, to_command
|
||||
from app.logs.journal import LogJournal
|
||||
from app.runtime.docker import Container, DockerClient, DockerError
|
||||
from app.texts import LogsTexts, Texts
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
TAILS = (50, 200) # lines (L-2)
|
||||
PERIODS = (5, 60, 24 * 60) # minutes (L-2)
|
||||
GREP_WINDOW = timedelta(hours=24)
|
||||
MAX_MESSAGE = 3500 # leave room under Telegram's 4096 for the header and HTML markup
|
||||
MAX_FILE = 5 * 1024 * 1024 # L-5
|
||||
|
||||
|
||||
class LogsAct(StrEnum):
|
||||
LIST = "l"
|
||||
OPEN = "o"
|
||||
TAIL = "t"
|
||||
SINCE = "s"
|
||||
GREP = "g"
|
||||
AGENT = "a"
|
||||
|
||||
|
||||
class LogsCb(CallbackData, prefix="lg"):
|
||||
a: LogsAct
|
||||
id: str = "" # 12-char container id prefix
|
||||
n: int = 0 # lines for TAIL, minutes for SINCE
|
||||
|
||||
|
||||
class LogsFlow(StatesGroup):
|
||||
grep = State()
|
||||
|
||||
|
||||
def period(t: LogsTexts, minutes: int) -> str:
|
||||
return t.hours.format(n=minutes // 60) if minutes >= 60 else t.minutes.format(n=minutes)
|
||||
|
||||
|
||||
def list_keyboard(texts: Texts, containers: list[Container]) -> InlineKeyboardMarkup:
|
||||
kb = InlineKeyboardBuilder()
|
||||
for c in containers:
|
||||
kb.button(text=f"{c.light.value} {c.name}", callback_data=LogsCb(a=LogsAct.OPEN, id=c.short_id))
|
||||
kb.button(text=texts.menu.back, callback_data=Nav(action=NavAction.BACK))
|
||||
kb.adjust(1)
|
||||
return kb.as_markup()
|
||||
|
||||
|
||||
def container_keyboard(texts: Texts, short_id: str) -> InlineKeyboardMarkup:
|
||||
t = texts.logs
|
||||
kb = InlineKeyboardBuilder()
|
||||
for n in TAILS:
|
||||
kb.button(text=t.tail.format(n=n), callback_data=LogsCb(a=LogsAct.TAIL, id=short_id, n=n))
|
||||
for minutes in PERIODS:
|
||||
kb.button(text=period(t, minutes), callback_data=LogsCb(a=LogsAct.SINCE, id=short_id, n=minutes))
|
||||
kb.button(text=t.grep_button, callback_data=LogsCb(a=LogsAct.GREP, id=short_id))
|
||||
kb.button(text=t.agent_button, callback_data=LogsCb(a=LogsAct.AGENT, id=short_id))
|
||||
kb.button(text=t.to_list, callback_data=LogsCb(a=LogsAct.LIST))
|
||||
kb.adjust(2, 3, 2, 1)
|
||||
return kb.as_markup()
|
||||
|
||||
|
||||
def notes(t: LogsTexts, result: GrepResult) -> list[str]:
|
||||
out = []
|
||||
if result.truncated:
|
||||
out.append(t.truncated.format(n=len(result.lines)))
|
||||
if result.partial:
|
||||
out.append(t.partial)
|
||||
if result.skipped:
|
||||
out.append(t.skipped.format(n=result.skipped))
|
||||
return out
|
||||
|
||||
|
||||
def grep_error(t: LogsTexts, e: GrepError) -> str:
|
||||
template = {
|
||||
GrepErrorCode.EMPTY: t.err_empty,
|
||||
GrepErrorCode.QUOTES: t.err_quotes,
|
||||
GrepErrorCode.FLAG: t.err_flag,
|
||||
GrepErrorCode.CONTEXT: t.err_context,
|
||||
GrepErrorCode.EXTRA: t.err_extra,
|
||||
GrepErrorCode.TOO_LONG: t.err_too_long,
|
||||
GrepErrorCode.REGEX: t.err_regex,
|
||||
}[e.code]
|
||||
return template.format(arg=html.escape(e.arg[:100]))
|
||||
|
||||
|
||||
async def send_result(
|
||||
msg: Message, texts: Texts, head: str, body: str, footer: list[str], kb: InlineKeyboardMarkup, filename: str
|
||||
) -> None:
|
||||
"""Short results inline as <pre>, long ones as a .log document (L-5); the container keyboard rides along."""
|
||||
t = texts.logs
|
||||
tail = "".join(f"\n{html.escape(n)}" for n in footer)
|
||||
if not body:
|
||||
await msg.answer(f"{head}\n{t.nothing}{tail}", parse_mode=ParseMode.HTML, reply_markup=kb)
|
||||
return
|
||||
escaped = html.escape(body)
|
||||
if len(escaped) <= MAX_MESSAGE:
|
||||
await msg.answer(f"{head}\n<pre>{escaped}</pre>{tail}", parse_mode=ParseMode.HTML, reply_markup=kb)
|
||||
return
|
||||
data = body.encode()[-MAX_FILE:]
|
||||
await msg.answer_document(
|
||||
BufferedInputFile(data, filename),
|
||||
caption=f"{head}\n{t.sent_as_file}{tail}",
|
||||
parse_mode=ParseMode.HTML,
|
||||
reply_markup=kb,
|
||||
)
|
||||
|
||||
|
||||
def build_router(texts: Texts, docker: DockerClient | None, journal: LogJournal) -> Router:
|
||||
router = Router(name="logs")
|
||||
t = texts.logs
|
||||
|
||||
def need_docker() -> DockerClient:
|
||||
if docker is None:
|
||||
raise DockerError("docker is not configured (DOCKER_URL)")
|
||||
return docker
|
||||
|
||||
@router.message(F.text == texts.menu.logs)
|
||||
async def open_logs(message: Message, state: FSMContext) -> None:
|
||||
await state.clear()
|
||||
try:
|
||||
containers = await need_docker().containers()
|
||||
except DockerError as e:
|
||||
log.warning("logs: %s", e)
|
||||
await message.answer(t.unavailable)
|
||||
return
|
||||
await message.answer(t.title if containers else t.empty, reply_markup=list_keyboard(texts, containers))
|
||||
|
||||
@router.callback_query(LogsCb.filter())
|
||||
async def on_action(query: CallbackQuery, callback_data: LogsCb, state: FSMContext, member: User) -> None:
|
||||
if not isinstance(query.message, Message):
|
||||
await query.answer()
|
||||
return
|
||||
msg = query.message
|
||||
try:
|
||||
d = need_docker()
|
||||
if callback_data.a is LogsAct.LIST:
|
||||
await state.clear()
|
||||
containers = await d.containers()
|
||||
await msg.edit_text(t.title if containers else t.empty, reply_markup=list_keyboard(texts, containers))
|
||||
await query.answer()
|
||||
return
|
||||
c = await d.get(callback_data.id)
|
||||
kb = container_keyboard(texts, c.short_id)
|
||||
match callback_data.a:
|
||||
case LogsAct.OPEN:
|
||||
await state.clear()
|
||||
await msg.edit_text(t.container.format(name=c.name), reply_markup=kb)
|
||||
case LogsAct.TAIL | LogsAct.SINCE:
|
||||
await query.answer()
|
||||
if callback_data.a is LogsAct.TAIL:
|
||||
n = min(max(callback_data.n, 1), max(TAILS))
|
||||
text = await d.logs(c.id, tail=n)
|
||||
scope, q = t.tail.format(n=n), {"tail": n}
|
||||
else:
|
||||
minutes = min(max(callback_data.n, 1), max(PERIODS))
|
||||
text = await d.logs(c.id, since=utcnow() - timedelta(minutes=minutes))
|
||||
scope, q = t.since.format(period=period(t, minutes)), {"since_minutes": minutes}
|
||||
lines = text.splitlines()
|
||||
await journal.record(member.tg_id, c.name, LogMode.TAIL, q, len(lines))
|
||||
head = t.head_tail.format(name=html.escape(c.name), scope=scope, lines=len(lines))
|
||||
await send_result(msg, texts, head, "\n".join(lines), [], kb, f"{c.name}.log")
|
||||
return
|
||||
case LogsAct.GREP:
|
||||
await state.set_state(LogsFlow.grep)
|
||||
await state.update_data(logs_id=c.short_id)
|
||||
back = InlineKeyboardBuilder()
|
||||
back.button(text=t.to_container, callback_data=LogsCb(a=LogsAct.OPEN, id=c.short_id))
|
||||
await msg.answer(
|
||||
t.ask_grep.format(name=html.escape(c.name)),
|
||||
parse_mode=ParseMode.HTML,
|
||||
reply_markup=back.as_markup(),
|
||||
)
|
||||
case LogsAct.AGENT:
|
||||
await query.answer(t.agent_soon, show_alert=True)
|
||||
return
|
||||
except DockerError as e:
|
||||
log.warning("logs: %s", e)
|
||||
await query.answer(t.unavailable, show_alert=True)
|
||||
return
|
||||
await query.answer()
|
||||
|
||||
@router.message(LogsFlow.grep, F.text, free_input(texts))
|
||||
async def got_grep(message: Message, state: FSMContext, member: User) -> None:
|
||||
try:
|
||||
query = parse_grep(message.text or "")
|
||||
except GrepError as e:
|
||||
await message.answer(grep_error(t, e), parse_mode=ParseMode.HTML)
|
||||
return # stay in the state: fix and resend
|
||||
data = await state.get_data()
|
||||
try:
|
||||
d = need_docker()
|
||||
c = await d.get(data.get("logs_id", ""))
|
||||
text = await d.logs(c.id, since=utcnow() - GREP_WINDOW)
|
||||
except DockerError as e:
|
||||
log.warning("logs: %s", e)
|
||||
await message.answer(t.unavailable)
|
||||
return
|
||||
await state.clear()
|
||||
result = await asyncio.to_thread(run, query, text)
|
||||
await journal.record(member.tg_id, c.name, LogMode.GREP, query.to_dict(), result.matched)
|
||||
head = t.head_grep.format(
|
||||
command=html.escape(to_command(query)), name=html.escape(c.name), matched=result.matched
|
||||
)
|
||||
body = "" if query.count else "\n".join(result.lines)
|
||||
footer = notes(t, result)
|
||||
if query.count: # -c: the number in the header is the answer
|
||||
await message.answer(
|
||||
head + "".join(f"\n{html.escape(n)}" for n in footer),
|
||||
parse_mode=ParseMode.HTML,
|
||||
reply_markup=container_keyboard(texts, c.short_id),
|
||||
)
|
||||
return
|
||||
await send_result(
|
||||
message, texts, head, body, footer, container_keyboard(texts, c.short_id), f"{c.name}-grep.log"
|
||||
)
|
||||
|
||||
return router
|
||||
@@ -93,7 +93,9 @@ class DockerSettings(BaseSettings):
|
||||
label: str = "tgit.managed=true"
|
||||
timeout: float = Field(default=10.0, gt=0)
|
||||
snapshot_interval: float = Field(default=15.0, gt=0)
|
||||
# One logs request reads at most this much (SPEC L-5: 5 MB); the newest lines win
|
||||
max_log_bytes: int = Field(default=5 * 1024 * 1024, gt=0)
|
||||
max_log_lines: int = Field(default=50_000, gt=0)
|
||||
|
||||
|
||||
class RuntimeSettings(BaseSettings):
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""Log search: a grep subset executed in Python, never in a shell (ARCHITECTURE §2.7)."""
|
||||
@@ -0,0 +1,201 @@
|
||||
"""`grep` for logs: parse a grep-like expression, run it with the `regex` module, show it back as a command.
|
||||
|
||||
Nothing here executes anything: `shlex.split` only tokenizes, the pattern goes to `regex` with a per-line timeout
|
||||
and an overall time budget, so `(a+)+$` on a long line costs at most the budget, not the bot.
|
||||
Supported: PATTERN, -i, -v, -E, -c, -C N (also -C2, -iv, `--`, a leading `grep`). Anything else is an error.
|
||||
"""
|
||||
|
||||
import shlex
|
||||
import time
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from enum import StrEnum
|
||||
from typing import Any
|
||||
|
||||
import regex
|
||||
|
||||
MAX_PATTERN = 200
|
||||
MAX_CONTEXT = 10
|
||||
MAX_LINES = 500 # output lines, context and `--` separators included
|
||||
LINE_TIMEOUT = 0.05 # seconds per line
|
||||
BUDGET = 3.0 # seconds per search; on overrun the result is partial and says so
|
||||
|
||||
_BRE_SPECIAL = set("+?|(){}")
|
||||
|
||||
|
||||
class GrepErrorCode(StrEnum):
|
||||
EMPTY = "empty"
|
||||
QUOTES = "quotes"
|
||||
FLAG = "flag"
|
||||
CONTEXT = "context"
|
||||
EXTRA = "extra"
|
||||
TOO_LONG = "too_long"
|
||||
REGEX = "regex"
|
||||
|
||||
|
||||
class GrepError(ValueError):
|
||||
def __init__(self, code: GrepErrorCode, arg: str = "") -> None:
|
||||
super().__init__(f"{code}: {arg}" if arg else code.value)
|
||||
self.code = code
|
||||
self.arg = arg
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GrepQuery:
|
||||
pattern: str
|
||||
ignore_case: bool = False
|
||||
invert: bool = False
|
||||
extended: bool = False
|
||||
count: bool = False
|
||||
context: int = 0
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
def compile(self) -> "regex.Pattern[str]":
|
||||
source = self.pattern if self.extended else bre_to_ere(self.pattern)
|
||||
try:
|
||||
return regex.compile(source, regex.IGNORECASE if self.ignore_case else 0)
|
||||
except regex.error as e:
|
||||
raise GrepError(GrepErrorCode.REGEX, str(e)) from e
|
||||
|
||||
|
||||
def bre_to_ere(pattern: str) -> str:
|
||||
"""Basic grep regex → Python: without -E `+ ? | ( ) { }` are literals and `\\+` etc. are the operators."""
|
||||
out: list[str] = []
|
||||
i, in_class = 0, False
|
||||
while i < len(pattern):
|
||||
ch = pattern[i]
|
||||
if in_class:
|
||||
out.append(ch)
|
||||
if ch == "]" and not (out[-2:-1] == ["["] or out[-3:-1] == ["[", "^"]):
|
||||
in_class = False
|
||||
elif ch == "[":
|
||||
in_class = True
|
||||
out.append(ch)
|
||||
elif ch == "\\" and i + 1 < len(pattern):
|
||||
nxt = pattern[i + 1]
|
||||
out.append(nxt if nxt in _BRE_SPECIAL else ch + nxt)
|
||||
i += 1
|
||||
elif ch in _BRE_SPECIAL:
|
||||
out.append("\\" + ch)
|
||||
else:
|
||||
out.append(ch)
|
||||
i += 1
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def _context(value: str) -> int:
|
||||
if not value.isdigit() or int(value) > MAX_CONTEXT:
|
||||
raise GrepError(GrepErrorCode.CONTEXT, value)
|
||||
return int(value)
|
||||
|
||||
|
||||
def parse_grep(expr: str) -> GrepQuery:
|
||||
try:
|
||||
tokens = shlex.split(expr)
|
||||
except ValueError as e:
|
||||
raise GrepError(GrepErrorCode.QUOTES) from e
|
||||
if tokens[:1] == ["grep"]:
|
||||
tokens = tokens[1:]
|
||||
opts: dict[str, Any] = {}
|
||||
patterns: list[str] = []
|
||||
i = 0
|
||||
while i < len(tokens):
|
||||
tok = tokens[i]
|
||||
if patterns or tok == "--" or not tok.startswith("-") or tok == "-":
|
||||
if tok == "--" and not patterns:
|
||||
patterns.extend(tokens[i + 1 :])
|
||||
break
|
||||
patterns.append(tok)
|
||||
elif tok.startswith("-C"):
|
||||
if len(tok) > 2:
|
||||
opts["context"] = _context(tok[2:])
|
||||
elif i + 1 < len(tokens):
|
||||
opts["context"] = _context(tokens[i + 1])
|
||||
i += 1
|
||||
else:
|
||||
raise GrepError(GrepErrorCode.CONTEXT, "")
|
||||
else:
|
||||
for j, flag in enumerate(tok[1:], start=1):
|
||||
match flag:
|
||||
case "i":
|
||||
opts["ignore_case"] = True
|
||||
case "v":
|
||||
opts["invert"] = True
|
||||
case "E":
|
||||
opts["extended"] = True
|
||||
case "c":
|
||||
opts["count"] = True
|
||||
case "C" if tok[j + 1 :].isdigit():
|
||||
opts["context"] = _context(tok[j + 1 :])
|
||||
break
|
||||
case _:
|
||||
raise GrepError(GrepErrorCode.FLAG, f"-{flag}")
|
||||
i += 1
|
||||
if not patterns or not patterns[0]:
|
||||
raise GrepError(GrepErrorCode.EMPTY)
|
||||
if len(patterns) > 1:
|
||||
raise GrepError(GrepErrorCode.EXTRA, patterns[1])
|
||||
if len(patterns[0]) > MAX_PATTERN:
|
||||
raise GrepError(GrepErrorCode.TOO_LONG, str(len(patterns[0])))
|
||||
query = GrepQuery(pattern=patterns[0], **opts)
|
||||
query.compile() # syntax errors now, not halfway through the logs
|
||||
return query
|
||||
|
||||
|
||||
def to_command(query: GrepQuery) -> str:
|
||||
"""The query as a shell-looking line, for the user to see what was searched (SPEC L-15)."""
|
||||
flags = "".join(
|
||||
f for f, on in (("i", query.ignore_case), ("v", query.invert), ("E", query.extended), ("c", query.count)) if on
|
||||
)
|
||||
parts = ["grep"]
|
||||
if flags:
|
||||
parts.append(f"-{flags}")
|
||||
if query.context:
|
||||
parts += ["-C", str(query.context)]
|
||||
if query.pattern.startswith("-"):
|
||||
parts.append("--")
|
||||
parts.append(shlex.quote(query.pattern))
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
@dataclass
|
||||
class GrepResult:
|
||||
lines: list[str] = field(default_factory=list)
|
||||
matched: int = 0
|
||||
truncated: bool = False # more output than MAX_LINES
|
||||
partial: bool = False # time budget ran out: not every line was searched
|
||||
skipped: int = 0 # lines where the pattern hit the per-line timeout
|
||||
|
||||
|
||||
def run(query: GrepQuery, text: str, max_lines: int = MAX_LINES, budget: float = BUDGET) -> GrepResult:
|
||||
pattern = query.compile()
|
||||
lines = text.splitlines()
|
||||
result = GrepResult()
|
||||
deadline = time.monotonic() + budget
|
||||
last_emitted = -1 # index of the last line already in the output (for context merging)
|
||||
for n, line in enumerate(lines):
|
||||
if time.monotonic() > deadline:
|
||||
result.partial = True
|
||||
break
|
||||
try:
|
||||
hit = pattern.search(line, timeout=LINE_TIMEOUT) is not None
|
||||
except TimeoutError:
|
||||
result.skipped += 1
|
||||
continue
|
||||
if hit == query.invert:
|
||||
continue
|
||||
result.matched += 1
|
||||
if query.count or result.truncated:
|
||||
continue
|
||||
start = max(n - query.context, last_emitted + 1)
|
||||
if query.context and result.lines and start > last_emitted + 1:
|
||||
result.lines.append("--")
|
||||
end = min(n + query.context, len(lines) - 1)
|
||||
for k in range(start, end + 1):
|
||||
if len(result.lines) >= max_lines:
|
||||
result.truncated = True
|
||||
break
|
||||
result.lines.append(lines[k])
|
||||
last_emitted = k
|
||||
return result
|
||||
@@ -0,0 +1,15 @@
|
||||
"""Every log search is recorded: who, which container, what, how many lines came back (SPEC L-6)."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.db.models import LogMode, LogQuery
|
||||
from app.db.session import SessionMaker
|
||||
|
||||
|
||||
class LogJournal:
|
||||
def __init__(self, sessions: SessionMaker) -> None:
|
||||
self._sessions = sessions
|
||||
|
||||
async def record(self, user_id: int, container: str, mode: LogMode, query: dict[str, Any], lines: int) -> None:
|
||||
async with self._sessions.begin() as session:
|
||||
session.add(LogQuery(user_id=user_id, container=container, mode=mode, query=query, lines=lines))
|
||||
+3
-1
@@ -11,12 +11,13 @@ from app import __version__
|
||||
from app.access.service import AccessService
|
||||
from app.bot.fsm import make_storage
|
||||
from app.bot.middlewares.access import AccessMiddleware
|
||||
from app.bot.routers import commit, common, members, status
|
||||
from app.bot.routers import commit, common, logs, members, status
|
||||
from app.config import Settings
|
||||
from app.db.maintenance import cleanup_loop
|
||||
from app.db.migrate import upgrade
|
||||
from app.db.session import SessionMaker, make_engine, make_sessionmaker
|
||||
from app.health import heartbeat
|
||||
from app.logs.journal import LogJournal
|
||||
from app.logs_setup import setup_logging
|
||||
from app.repo.drafts import DraftService
|
||||
from app.repo.git import UploadsRepo
|
||||
@@ -48,6 +49,7 @@ def build_dispatcher(
|
||||
commit.build_router(texts, repo, DraftService(sessions), Publisher(repo, sessions), commit_url_template)
|
||||
)
|
||||
dp.include_router(status.build_router(texts, docker, checker))
|
||||
dp.include_router(logs.build_router(texts, docker, LogJournal(sessions)))
|
||||
dp.include_router(common.build_router(texts, access))
|
||||
return dp
|
||||
|
||||
|
||||
+21
-18
@@ -171,7 +171,7 @@ class DockerClient:
|
||||
def __init__(self, settings: DockerSettings, transport: httpx.AsyncBaseTransport | None = None) -> None:
|
||||
self._s = settings
|
||||
self._http = httpx.AsyncClient(base_url=settings.url, timeout=settings.timeout, transport=transport)
|
||||
self._known: set[str] = set() # ids from the last listing: the only ones card/logs/stats may touch
|
||||
self._known: dict[str, Container] = {} # the last listing: the only ids card/logs/stats may touch
|
||||
|
||||
async def __aenter__(self) -> Self:
|
||||
return self
|
||||
@@ -204,22 +204,22 @@ class DockerClient:
|
||||
labels=labels,
|
||||
)
|
||||
)
|
||||
self._known = {c.id for c in result}
|
||||
self._known = {c.id: c for c in result}
|
||||
return sorted(result, key=lambda c: c.name)
|
||||
|
||||
async def resolve(self, short_id: str) -> str:
|
||||
"""Full id of a managed container by its 12-char prefix (callback data carries only that)."""
|
||||
if not self._known:
|
||||
await self.containers()
|
||||
for full in self._known:
|
||||
if full.startswith(short_id):
|
||||
return full
|
||||
await self.containers() # maybe recreated since the last listing
|
||||
for full in self._known:
|
||||
if full.startswith(short_id):
|
||||
return full
|
||||
async def get(self, short_id: str) -> Container:
|
||||
"""A managed container by its 12-char id prefix (callback data carries only that)."""
|
||||
for attempt in range(2):
|
||||
if attempt or not self._known:
|
||||
await self.containers() # first use, or recreated since the last listing
|
||||
for full, container in self._known.items():
|
||||
if short_id and full.startswith(short_id):
|
||||
return container
|
||||
raise DockerError(f"container {short_id} is not managed by tgit")
|
||||
|
||||
async def resolve(self, short_id: str) -> str:
|
||||
return (await self.get(short_id)).id
|
||||
|
||||
async def inspect(self, container_id: str) -> Details:
|
||||
self._guard(container_id)
|
||||
d = await self._get_json(f"/containers/{container_id}/json")
|
||||
@@ -246,15 +246,18 @@ class DockerClient:
|
||||
return parse_stats(await self._get_json(f"/containers/{container_id}/stats", params={"stream": "false"}))
|
||||
|
||||
async def logs(self, container_id: str, since: datetime | None = None, tail: int | None = None) -> str:
|
||||
"""stdout+stderr in order; at most max_log_lines lines and max_log_bytes — the newest ones are kept."""
|
||||
self._guard(container_id)
|
||||
params = {"stdout": "1", "stderr": "1"}
|
||||
lines = min(tail, self._s.max_log_lines) if tail is not None else self._s.max_log_lines
|
||||
params = {"stdout": "1", "stderr": "1", "tail": str(lines)}
|
||||
if since is not None:
|
||||
params["since"] = str(int(since.timestamp()))
|
||||
if tail is not None:
|
||||
params["tail"] = str(tail)
|
||||
response = await self._request(f"/containers/{container_id}/logs", params)
|
||||
raw = response.content[: self._s.max_log_bytes]
|
||||
return demux_logs(raw)
|
||||
text = demux_logs(response.content)
|
||||
if len(text) > self._s.max_log_bytes:
|
||||
cut = text[-self._s.max_log_bytes :]
|
||||
text = cut[cut.find("\n") + 1 :] # drop the torn first line
|
||||
return text
|
||||
|
||||
async def ping(self) -> bool:
|
||||
try:
|
||||
|
||||
+32
-1
@@ -33,7 +33,6 @@ class HelpTexts(_Strict):
|
||||
|
||||
class SectionTexts(_Strict):
|
||||
commit: str
|
||||
logs: str
|
||||
alerts: str
|
||||
members: str
|
||||
cancelled: str
|
||||
@@ -152,6 +151,37 @@ class StatusTexts(_Strict):
|
||||
seconds: str
|
||||
|
||||
|
||||
class LogsTexts(_Strict):
|
||||
title: str
|
||||
empty: str
|
||||
unavailable: str
|
||||
container: str
|
||||
tail: str
|
||||
since: str
|
||||
minutes: str
|
||||
hours: str
|
||||
grep_button: str
|
||||
agent_button: str
|
||||
agent_soon: str
|
||||
to_list: str
|
||||
to_container: str
|
||||
ask_grep: str
|
||||
head_tail: str
|
||||
head_grep: str
|
||||
nothing: str
|
||||
truncated: str
|
||||
partial: str
|
||||
skipped: str
|
||||
sent_as_file: str
|
||||
err_empty: str
|
||||
err_quotes: str
|
||||
err_flag: str
|
||||
err_context: str
|
||||
err_extra: str
|
||||
err_too_long: str
|
||||
err_regex: str
|
||||
|
||||
|
||||
class Texts(_Strict):
|
||||
menu: MenuTexts
|
||||
start: StartTexts
|
||||
@@ -160,6 +190,7 @@ class Texts(_Strict):
|
||||
members: MembersTexts
|
||||
commit: CommitTexts
|
||||
status: StatusTexts
|
||||
logs: LogsTexts
|
||||
|
||||
|
||||
def load_texts(config_dir: Path) -> Texts:
|
||||
|
||||
Reference in New Issue
Block a user