feat(llm): провайдер за протоколом — OpenAI-совместимый API и mock, фабрика по конфигу
- LLMProvider: complete(messages, schema) -> LLMResult (текст, разобранный JSON, токены) - openai_compat: /chat/completions, structured output через response_format json_schema, JSON из ```-обёртки тоже разбирается; ретраи на 429/5xx и сетевые ошибки, 4xx — сразу ошибка - mock: детерминированные правила для тестов и демо без сети - LLMSettings (LLM_PROVIDER/BASE_URL/MODEL/API_KEY…), CLI python -m app.llm.cli Refs #17 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,21 @@
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
from app.llm import cli
|
||||
|
||||
|
||||
def test_cli_asks_configured_provider(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
monkeypatch.setenv("LLM_PROVIDER", "mock")
|
||||
monkeypatch.setattr(sys, "argv", ["cli", "привет", "мир"])
|
||||
|
||||
cli.main()
|
||||
|
||||
out = capsys.readouterr().out
|
||||
assert "[mock -]" in out and "mock: привет мир" in out
|
||||
|
||||
|
||||
def test_cli_usage(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(sys, "argv", ["cli"])
|
||||
with pytest.raises(SystemExit):
|
||||
cli.main()
|
||||
@@ -0,0 +1,139 @@
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from app.config import LLMSettings
|
||||
from app.llm.factory import make_provider
|
||||
from app.llm.mock import MockProvider, MockRule
|
||||
from app.llm.openai_compat import OpenAICompatProvider
|
||||
from app.llm.parsing import parse_json_object
|
||||
from app.llm.provider import ChatMessage, JsonSchema, LLMError
|
||||
|
||||
BASE = "https://llm.example.test/v1"
|
||||
SCHEMA = JsonSchema("verdict", {"type": "object", "properties": {"verdict": {"type": "string"}}})
|
||||
|
||||
|
||||
def settings(**kw: Any) -> LLMSettings:
|
||||
base = {"provider": "openai_compat", "base_url": BASE, "model": "m-1", "api_key": "k", "max_attempts": 3}
|
||||
return LLMSettings(**(base | kw))
|
||||
|
||||
|
||||
def completion(content: str, prompt: int = 5, completion_tokens: int = 3) -> dict[str, Any]:
|
||||
return {
|
||||
"choices": [{"message": {"role": "assistant", "content": content}}],
|
||||
"usage": {"prompt_tokens": prompt, "completion_tokens": completion_tokens},
|
||||
}
|
||||
|
||||
|
||||
class Sleeps:
|
||||
def __init__(self) -> None:
|
||||
self.delays: list[float] = []
|
||||
|
||||
async def __call__(self, delay: float) -> None:
|
||||
self.delays.append(delay)
|
||||
|
||||
|
||||
@respx.mock
|
||||
async def test_plain_completion_sends_messages_and_counts_tokens() -> None:
|
||||
route = respx.post(f"{BASE}/chat/completions").respond(json=completion("привет"))
|
||||
provider = OpenAICompatProvider(settings())
|
||||
|
||||
result = await provider.complete([ChatMessage("system", "s"), ChatMessage("user", "u")])
|
||||
await provider.aclose()
|
||||
|
||||
sent = json.loads(route.calls.last.request.content)
|
||||
assert route.calls.last.request.headers["Authorization"] == "Bearer k"
|
||||
assert sent["model"] == "m-1" and sent["temperature"] == 0.0
|
||||
assert sent["messages"] == [{"role": "system", "content": "s"}, {"role": "user", "content": "u"}]
|
||||
assert "response_format" not in sent
|
||||
assert (result.text, result.data, result.tokens) == ("привет", None, 8)
|
||||
|
||||
|
||||
@respx.mock
|
||||
async def test_schema_requests_structured_output_and_parses_it() -> None:
|
||||
route = respx.post(f"{BASE}/chat/completions").respond(json=completion('```json\n{"verdict": "safe"}\n```'))
|
||||
provider = OpenAICompatProvider(settings())
|
||||
|
||||
result = await provider.complete([ChatMessage("user", "x")], SCHEMA)
|
||||
|
||||
fmt = json.loads(route.calls.last.request.content)["response_format"]
|
||||
assert fmt == {"type": "json_schema", "json_schema": {"name": "verdict", "schema": SCHEMA.schema, "strict": True}}
|
||||
assert result.data == {"verdict": "safe"}
|
||||
|
||||
|
||||
@respx.mock
|
||||
async def test_invalid_json_gives_no_data() -> None:
|
||||
respx.post(f"{BASE}/chat/completions").respond(json=completion("not json"))
|
||||
|
||||
result = await OpenAICompatProvider(settings()).complete([ChatMessage("user", "x")], SCHEMA)
|
||||
|
||||
assert result.data is None and result.text == "not json"
|
||||
|
||||
|
||||
@respx.mock
|
||||
async def test_retries_rate_limit_and_network_errors() -> None:
|
||||
sleeps = Sleeps()
|
||||
respx.post(f"{BASE}/chat/completions").mock(
|
||||
side_effect=[httpx.Response(429), httpx.ConnectError("x"), httpx.Response(200, json=completion("ok"))]
|
||||
)
|
||||
|
||||
result = await OpenAICompatProvider(settings(), sleep=sleeps).complete([ChatMessage("user", "x")])
|
||||
|
||||
assert result.text == "ok" and sleeps.delays == [1.0, 2.0]
|
||||
|
||||
|
||||
@respx.mock
|
||||
async def test_client_error_is_not_retried() -> None:
|
||||
route = respx.post(f"{BASE}/chat/completions").respond(401, text="bad key")
|
||||
|
||||
with pytest.raises(LLMError, match="401"):
|
||||
await OpenAICompatProvider(settings()).complete([ChatMessage("user", "x")])
|
||||
assert route.call_count == 1
|
||||
|
||||
|
||||
@respx.mock
|
||||
async def test_unreachable_after_all_attempts() -> None:
|
||||
respx.post(f"{BASE}/chat/completions").mock(side_effect=httpx.ConnectError("down"))
|
||||
|
||||
with pytest.raises(LLMError, match="unreachable"):
|
||||
await OpenAICompatProvider(settings(), sleep=Sleeps()).complete([ChatMessage("user", "x")])
|
||||
|
||||
|
||||
@respx.mock
|
||||
async def test_unexpected_shape_is_an_error() -> None:
|
||||
respx.post(f"{BASE}/chat/completions").respond(json={"choices": []})
|
||||
|
||||
with pytest.raises(LLMError, match="shape"):
|
||||
await OpenAICompatProvider(settings()).complete([ChatMessage("user", "x")])
|
||||
|
||||
|
||||
def test_openai_compat_requires_url_and_model() -> None:
|
||||
with pytest.raises(LLMError):
|
||||
OpenAICompatProvider(settings(base_url=""))
|
||||
|
||||
|
||||
async def test_mock_rules_and_fallbacks() -> None:
|
||||
mock = MockProvider([MockRule("взлом", {"verdict": "attack"}), MockRule("привет", "здравствуй")])
|
||||
|
||||
assert (await mock.complete([ChatMessage("user", "попытка взлома")], SCHEMA)).data == {"verdict": "attack"}
|
||||
assert (await mock.complete([ChatMessage("user", "привет")])).text == "здравствуй"
|
||||
assert (await mock.complete([ChatMessage("user", "что-то")])).text == "mock: что-то"
|
||||
assert (await mock.complete([ChatMessage("user", "что-то")], SCHEMA)).data == {}
|
||||
assert len(mock.calls) == 4
|
||||
await mock.aclose()
|
||||
|
||||
|
||||
def test_factory_picks_provider() -> None:
|
||||
assert isinstance(make_provider(LLMSettings(provider="mock")), MockProvider)
|
||||
assert isinstance(make_provider(settings()), OpenAICompatProvider)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "expected"),
|
||||
[('{"a": 1}', {"a": 1}), ('```json\n{"a": 1}\n```', {"a": 1}), ("[1]", None), ("nope", None)],
|
||||
)
|
||||
def test_parse_json_object(text: str, expected: dict[str, Any] | None) -> None:
|
||||
assert parse_json_object(text) == expected
|
||||
Reference in New Issue
Block a user