diff --git a/app/config.py b/app/config.py index 64ed71b..492efd8 100644 --- a/app/config.py +++ b/app/config.py @@ -1,7 +1,7 @@ """Typed configuration from the environment / `.env` — one settings class per subsystem (ARCHITECTURE §2.11).""" from pathlib import Path -from typing import Annotated +from typing import Annotated, Literal from pydantic import Field, SecretStr, field_validator from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict @@ -29,6 +29,19 @@ class AccessSettings(BaseSettings): return value +class LLMSettings(BaseSettings): + model_config = SettingsConfigDict(**_ENV, env_prefix="LLM_") + + # mock: deterministic offline answers (tests, demo without network); openai_compat: any OpenAI-style API (ADR-008) + provider: Literal["mock", "openai_compat"] = "mock" + base_url: str = "" + model: str = "" + api_key: SecretStr = SecretStr("") + timeout: float = Field(default=30.0, gt=0) + max_attempts: int = Field(default=3, ge=1) + temperature: float = Field(default=0.0, ge=0, le=2) + + class RuntimeSettings(BaseSettings): model_config = _ENV @@ -46,4 +59,5 @@ class Settings(BaseSettings): telegram: TelegramSettings = Field(default_factory=TelegramSettings) access: AccessSettings = Field(default_factory=AccessSettings) + llm: LLMSettings = Field(default_factory=LLMSettings) runtime: RuntimeSettings = Field(default_factory=RuntimeSettings) diff --git a/app/llm/__init__.py b/app/llm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/llm/cli.py b/app/llm/cli.py new file mode 100644 index 0000000..012c637 --- /dev/null +++ b/app/llm/cli.py @@ -0,0 +1,28 @@ +"""Debug CLI: `python -m app.llm.cli 'вопрос'` — one completion with the configured provider.""" + +import asyncio +import sys + +from app.config import LLMSettings +from app.llm.factory import make_provider +from app.llm.provider import ChatMessage + + +async def ask(question: str) -> None: + settings = LLMSettings() + provider = make_provider(settings) + try: + result = await provider.complete([ChatMessage("user", question)]) + finally: + await provider.aclose() + print(f"[{settings.provider} {settings.model or '-'}] tokens={result.tokens}\n{result.text}") + + +def main() -> None: + if len(sys.argv) < 2: + sys.exit("usage: python -m app.llm.cli 'question'") + asyncio.run(ask(" ".join(sys.argv[1:]))) + + +if __name__ == "__main__": + main() diff --git a/app/llm/factory.py b/app/llm/factory.py new file mode 100644 index 0000000..815557f --- /dev/null +++ b/app/llm/factory.py @@ -0,0 +1,10 @@ +from app.config import LLMSettings +from app.llm.mock import MockProvider +from app.llm.openai_compat import OpenAICompatProvider +from app.llm.provider import LLMProvider + + +def make_provider(settings: LLMSettings) -> LLMProvider: + if settings.provider == "openai_compat": + return OpenAICompatProvider(settings) + return MockProvider() diff --git a/app/llm/mock.py b/app/llm/mock.py new file mode 100644 index 0000000..a6c8e79 --- /dev/null +++ b/app/llm/mock.py @@ -0,0 +1,36 @@ +"""Deterministic offline provider for tests and demos without network or tokens. + +Rules are checked in order: the first whose `contains` is found in the last user message wins. +Without a match it echoes the message (or returns `{}` when JSON was requested). +""" + +import json +from dataclasses import dataclass +from typing import Any + +from app.llm.provider import ChatMessage, JsonSchema, LLMResult + + +@dataclass(frozen=True) +class MockRule: + contains: str + reply: str | dict[str, Any] + + +class MockProvider: + def __init__(self, rules: list[MockRule] | None = None) -> None: + self.rules = list(rules or []) + self.calls: list[tuple[list[ChatMessage], JsonSchema | None]] = [] + + async def aclose(self) -> None: + pass + + async def complete(self, messages: list[ChatMessage], schema: JsonSchema | None = None) -> LLMResult: + self.calls.append((messages, schema)) + last = next((m.content for m in reversed(messages) if m.role == "user"), "") + reply: str | dict[str, Any] | None = next((r.reply for r in self.rules if r.contains in last), None) + if reply is None: + reply = {} if schema is not None else f"mock: {last}" + text = json.dumps(reply, ensure_ascii=False) if isinstance(reply, dict) else reply + data = reply if isinstance(reply, dict) and schema is not None else None + return LLMResult(text=text, data=data, prompt_tokens=len(last.split()), completion_tokens=len(text.split())) diff --git a/app/llm/openai_compat.py b/app/llm/openai_compat.py new file mode 100644 index 0000000..0e9f9de --- /dev/null +++ b/app/llm/openai_compat.py @@ -0,0 +1,84 @@ +"""Any OpenAI-style `/chat/completions` endpoint: OpenAI, YandexGPT compatibility API, a local model server.""" + +import asyncio +import logging +from collections.abc import Awaitable, Callable +from typing import Any + +import httpx + +from app.config import LLMSettings +from app.llm.parsing import parse_json_object +from app.llm.provider import ChatMessage, JsonSchema, LLMError, LLMResult + +log = logging.getLogger(__name__) + +Sleep = Callable[[float], Awaitable[None]] + +RETRY_STATUSES = {408, 409, 429, 500, 502, 503, 504} + + +class OpenAICompatProvider: + def __init__( + self, + settings: LLMSettings, + transport: httpx.AsyncBaseTransport | None = None, + sleep: Sleep = asyncio.sleep, + ) -> None: + if not settings.base_url or not settings.model: + raise LLMError("LLM_BASE_URL and LLM_MODEL are required for the openai_compat provider") + self._settings = settings + self._sleep = sleep + self._http = httpx.AsyncClient( + base_url=settings.base_url.rstrip("/"), + headers={"Authorization": f"Bearer {settings.api_key.get_secret_value()}"}, + timeout=settings.timeout, + transport=transport, + ) + + async def aclose(self) -> None: + await self._http.aclose() + + async def complete(self, messages: list[ChatMessage], schema: JsonSchema | None = None) -> LLMResult: + body: dict[str, Any] = { + "model": self._settings.model, + "messages": [{"role": m.role, "content": m.content} for m in messages], + "temperature": self._settings.temperature, + } + if schema is not None: + body["response_format"] = { + "type": "json_schema", + "json_schema": {"name": schema.name, "schema": schema.schema, "strict": True}, + } + data = await self._post(body) + try: + text = data["choices"][0]["message"]["content"] or "" + except (KeyError, IndexError, TypeError) as e: + raise LLMError(f"unexpected completion shape: {list(data)}") from e + usage = data.get("usage") or {} + return LLMResult( + text=text, + data=parse_json_object(text) if schema is not None else None, + prompt_tokens=int(usage.get("prompt_tokens", 0)), + completion_tokens=int(usage.get("completion_tokens", 0)), + raw=data, + ) + + async def _post(self, body: dict[str, Any]) -> dict[str, Any]: + attempts = self._settings.max_attempts + for attempt in range(1, attempts + 1): + try: + response = await self._http.post("/chat/completions", json=body) + except httpx.TransportError as e: + if attempt == attempts: + raise LLMError(f"LLM unreachable: {type(e).__name__}") from e + else: + if response.status_code < 400: + result: dict[str, Any] = response.json() + return result + if response.status_code not in RETRY_STATUSES or attempt == attempts: + # never echo the request: it may contain user text; the response body is the provider's error + raise LLMError(f"LLM HTTP {response.status_code}: {response.text[:200]}") + log.warning("llm request failed, retry %d/%d", attempt, attempts - 1) + await self._sleep(min(1.0 * 2 ** (attempt - 1), 8.0)) + raise AssertionError("unreachable") diff --git a/app/llm/parsing.py b/app/llm/parsing.py new file mode 100644 index 0000000..3caa1ed --- /dev/null +++ b/app/llm/parsing.py @@ -0,0 +1,18 @@ +import json +import re +from typing import Any + +_FENCE = re.compile(r"^```(?:json)?\s*(.*?)\s*```$", re.DOTALL) + + +def parse_json_object(text: str) -> dict[str, Any] | None: + """Best effort: models sometimes wrap JSON in ``` fences even when asked not to.""" + candidate = text.strip() + fenced = _FENCE.match(candidate) + if fenced: + candidate = fenced.group(1) + try: + value = json.loads(candidate) + except ValueError: + return None + return value if isinstance(value, dict) else None diff --git a/app/llm/provider.py b/app/llm/provider.py new file mode 100644 index 0000000..ee85911 --- /dev/null +++ b/app/llm/provider.py @@ -0,0 +1,45 @@ +"""LLM provider contract (ADR-008): the rest of the bot depends on this, never on a vendor SDK.""" + +from dataclasses import dataclass, field +from typing import Any, Literal, Protocol + +Role = Literal["system", "user", "assistant"] + + +@dataclass(frozen=True) +class ChatMessage: + role: Role + content: str + + +@dataclass(frozen=True) +class JsonSchema: + """Ask the model for a JSON object of this shape (structured output).""" + + name: str + schema: dict[str, Any] + + +@dataclass(frozen=True) +class LLMResult: + text: str + # Parsed object when a JSON schema was requested and the reply was valid JSON; None otherwise. + # The caller still validates it — a model can return well-formed JSON of the wrong shape. + data: dict[str, Any] | None = None + prompt_tokens: int = 0 + completion_tokens: int = 0 + raw: dict[str, Any] = field(default_factory=dict, repr=False) + + @property + def tokens(self) -> int: + return self.prompt_tokens + self.completion_tokens + + +class LLMError(Exception): + pass + + +class LLMProvider(Protocol): + async def complete(self, messages: list[ChatMessage], schema: JsonSchema | None = None) -> LLMResult: ... + + async def aclose(self) -> None: ... diff --git a/tests/llm/__init__.py b/tests/llm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/llm/test_cli.py b/tests/llm/test_cli.py new file mode 100644 index 0000000..9cd4dbe --- /dev/null +++ b/tests/llm/test_cli.py @@ -0,0 +1,21 @@ +import sys + +import pytest + +from app.llm import cli + + +def test_cli_asks_configured_provider(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setenv("LLM_PROVIDER", "mock") + monkeypatch.setattr(sys, "argv", ["cli", "привет", "мир"]) + + cli.main() + + out = capsys.readouterr().out + assert "[mock -]" in out and "mock: привет мир" in out + + +def test_cli_usage(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sys, "argv", ["cli"]) + with pytest.raises(SystemExit): + cli.main() diff --git a/tests/llm/test_providers.py b/tests/llm/test_providers.py new file mode 100644 index 0000000..db38204 --- /dev/null +++ b/tests/llm/test_providers.py @@ -0,0 +1,139 @@ +import json +from typing import Any + +import httpx +import pytest +import respx + +from app.config import LLMSettings +from app.llm.factory import make_provider +from app.llm.mock import MockProvider, MockRule +from app.llm.openai_compat import OpenAICompatProvider +from app.llm.parsing import parse_json_object +from app.llm.provider import ChatMessage, JsonSchema, LLMError + +BASE = "https://llm.example.test/v1" +SCHEMA = JsonSchema("verdict", {"type": "object", "properties": {"verdict": {"type": "string"}}}) + + +def settings(**kw: Any) -> LLMSettings: + base = {"provider": "openai_compat", "base_url": BASE, "model": "m-1", "api_key": "k", "max_attempts": 3} + return LLMSettings(**(base | kw)) + + +def completion(content: str, prompt: int = 5, completion_tokens: int = 3) -> dict[str, Any]: + return { + "choices": [{"message": {"role": "assistant", "content": content}}], + "usage": {"prompt_tokens": prompt, "completion_tokens": completion_tokens}, + } + + +class Sleeps: + def __init__(self) -> None: + self.delays: list[float] = [] + + async def __call__(self, delay: float) -> None: + self.delays.append(delay) + + +@respx.mock +async def test_plain_completion_sends_messages_and_counts_tokens() -> None: + route = respx.post(f"{BASE}/chat/completions").respond(json=completion("привет")) + provider = OpenAICompatProvider(settings()) + + result = await provider.complete([ChatMessage("system", "s"), ChatMessage("user", "u")]) + await provider.aclose() + + sent = json.loads(route.calls.last.request.content) + assert route.calls.last.request.headers["Authorization"] == "Bearer k" + assert sent["model"] == "m-1" and sent["temperature"] == 0.0 + assert sent["messages"] == [{"role": "system", "content": "s"}, {"role": "user", "content": "u"}] + assert "response_format" not in sent + assert (result.text, result.data, result.tokens) == ("привет", None, 8) + + +@respx.mock +async def test_schema_requests_structured_output_and_parses_it() -> None: + route = respx.post(f"{BASE}/chat/completions").respond(json=completion('```json\n{"verdict": "safe"}\n```')) + provider = OpenAICompatProvider(settings()) + + result = await provider.complete([ChatMessage("user", "x")], SCHEMA) + + fmt = json.loads(route.calls.last.request.content)["response_format"] + assert fmt == {"type": "json_schema", "json_schema": {"name": "verdict", "schema": SCHEMA.schema, "strict": True}} + assert result.data == {"verdict": "safe"} + + +@respx.mock +async def test_invalid_json_gives_no_data() -> None: + respx.post(f"{BASE}/chat/completions").respond(json=completion("not json")) + + result = await OpenAICompatProvider(settings()).complete([ChatMessage("user", "x")], SCHEMA) + + assert result.data is None and result.text == "not json" + + +@respx.mock +async def test_retries_rate_limit_and_network_errors() -> None: + sleeps = Sleeps() + respx.post(f"{BASE}/chat/completions").mock( + side_effect=[httpx.Response(429), httpx.ConnectError("x"), httpx.Response(200, json=completion("ok"))] + ) + + result = await OpenAICompatProvider(settings(), sleep=sleeps).complete([ChatMessage("user", "x")]) + + assert result.text == "ok" and sleeps.delays == [1.0, 2.0] + + +@respx.mock +async def test_client_error_is_not_retried() -> None: + route = respx.post(f"{BASE}/chat/completions").respond(401, text="bad key") + + with pytest.raises(LLMError, match="401"): + await OpenAICompatProvider(settings()).complete([ChatMessage("user", "x")]) + assert route.call_count == 1 + + +@respx.mock +async def test_unreachable_after_all_attempts() -> None: + respx.post(f"{BASE}/chat/completions").mock(side_effect=httpx.ConnectError("down")) + + with pytest.raises(LLMError, match="unreachable"): + await OpenAICompatProvider(settings(), sleep=Sleeps()).complete([ChatMessage("user", "x")]) + + +@respx.mock +async def test_unexpected_shape_is_an_error() -> None: + respx.post(f"{BASE}/chat/completions").respond(json={"choices": []}) + + with pytest.raises(LLMError, match="shape"): + await OpenAICompatProvider(settings()).complete([ChatMessage("user", "x")]) + + +def test_openai_compat_requires_url_and_model() -> None: + with pytest.raises(LLMError): + OpenAICompatProvider(settings(base_url="")) + + +async def test_mock_rules_and_fallbacks() -> None: + mock = MockProvider([MockRule("взлом", {"verdict": "attack"}), MockRule("привет", "здравствуй")]) + + assert (await mock.complete([ChatMessage("user", "попытка взлома")], SCHEMA)).data == {"verdict": "attack"} + assert (await mock.complete([ChatMessage("user", "привет")])).text == "здравствуй" + assert (await mock.complete([ChatMessage("user", "что-то")])).text == "mock: что-то" + assert (await mock.complete([ChatMessage("user", "что-то")], SCHEMA)).data == {} + assert len(mock.calls) == 4 + await mock.aclose() + + +def test_factory_picks_provider() -> None: + assert isinstance(make_provider(LLMSettings(provider="mock")), MockProvider) + assert isinstance(make_provider(settings()), OpenAICompatProvider) + + +@pytest.mark.parametrize( + ("text", "expected"), + [('{"a": 1}', {"a": 1}), ('```json\n{"a": 1}\n```', {"a": 1}), ("[1]", None), ("nope", None)], +) +def test_parse_json_object(text: str, expected: dict[str, Any] | None) -> None: + assert parse_json_object(text) == expected