ElebobaandClaude Opus 5.5
51cc259002
CI / ruff · pytest (pull_request) Successful in 21s
Topic from engine or keyword classifier, section via integrations.drug_info,
sentence retrieval by keyword overlap (dosing sentences dropped), LLM only
rephrases the excerpt through gateway.phrase("med_answer") with forbidden-output
and added-content guards; dose questions and missing answers -> None + fallback text.
Refs #15
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
230 lines
11 KiB
Python
230 lines
11 KiB
Python
"""med_qa: answers about the drug only from the instruction excerpt (SPEC §7 п.5), demo days 2/5/10."""
|
||
|
||
import pytest
|
||
|
||
from app.core.types import ScrubbedText
|
||
from app.integrations.base import DrugSection
|
||
from app.llm import med_qa as med_qa_module
|
||
from app.llm.gateway import LLMGateway, is_forbidden
|
||
from app.llm.med_qa import MedAnswer, adds_content, detect_topic, fallback_text, med_qa, select_excerpt
|
||
from app.llm.mock import MockTasks
|
||
|
||
S = ScrubbedText
|
||
SRC = "ГРЛС: тестовая выдержка (синтетика)"
|
||
|
||
# Synthetic, simplified test excerpts — NOT real ГРЛС data (lane B owns data/meds/grls).
|
||
FAKE_SERTRALINE = {
|
||
"side_effects": (
|
||
"Наиболее частые нежелательные реакции в начале терапии: тошнота, диарея, сухость во рту. "
|
||
"Тошнота обычно возникает в первые дни приёма и проходит в течение 1–2 недель. "
|
||
"Для уменьшения тошноты препарат принимают после еды. "
|
||
"Возможны нарушения сна и головная боль."
|
||
),
|
||
"onset": (
|
||
"Терапевтический эффект развивается постепенно. "
|
||
"Улучшение обычно отмечается через 2–4 недели регулярного приёма. "
|
||
"Суточная доза может увеличиваться врачом до 200 мг."
|
||
),
|
||
"alcohol": (
|
||
"Совместный приём с алкоголем не рекомендуется. "
|
||
"Алкоголь может ухудшать сон и настроение и усиливать побочные эффекты."
|
||
),
|
||
"discontinuation": (
|
||
"Резкое прекращение приёма может вызвать синдром отмены: головокружение, тревогу, нарушения сна. "
|
||
"Отмену проводят постепенно под контролем врача."
|
||
),
|
||
}
|
||
|
||
|
||
class FakeDrugInfo:
|
||
def __init__(self, sections: dict[str, str] | None = None) -> None:
|
||
self._sections = FAKE_SERTRALINE if sections is None else sections
|
||
self.calls: list[str] = []
|
||
|
||
async def sections(self, inn: str) -> list[DrugSection]:
|
||
self.calls.append(inn)
|
||
if inn != "сертралин":
|
||
return []
|
||
return [DrugSection(inn=inn, section=k, text=v, source=SRC) for k, v in self._sections.items()]
|
||
|
||
|
||
class BrokenDrugInfo:
|
||
async def sections(self, inn: str) -> list[DrugSection]:
|
||
raise RuntimeError("grls down")
|
||
|
||
|
||
class FixedPhraseTasks(MockTasks):
|
||
"""Simulates an LLM that returns a fixed rephrasing."""
|
||
|
||
name = "fake-llm"
|
||
|
||
def __init__(self, reply: str) -> None:
|
||
self.reply = reply
|
||
|
||
async def phrase(self, intent: str, payload: ScrubbedText, base_text: str) -> str | None:
|
||
return self.reply
|
||
|
||
|
||
@pytest.fixture
|
||
def info() -> FakeDrugInfo:
|
||
return FakeDrugInfo()
|
||
|
||
|
||
@pytest.fixture
|
||
def mock_gateway(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
monkeypatch.setattr(med_qa_module, "get_gateway", lambda: LLMGateway(MockTasks()))
|
||
|
||
|
||
def use_llm(monkeypatch: pytest.MonkeyPatch, reply: str) -> None:
|
||
monkeypatch.setattr(med_qa_module, "get_gateway", lambda: LLMGateway(FixedPhraseTasks(reply)))
|
||
|
||
|
||
def assert_safe(answer: MedAnswer) -> None:
|
||
assert not is_forbidden(answer.text)
|
||
assert "мг" not in answer.text # never quotes dosing regimens
|
||
assert "врач" in answer.text # doctor note always present
|
||
|
||
|
||
# --- topic detection --------------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
("question", "topic"),
|
||
[
|
||
("подташнивает с утра", "side_effects"),
|
||
("опять тошнит", "side_effects"),
|
||
("тошнота не проходит", "side_effects"),
|
||
("от таблеток только хуже, брошу", "onset"),
|
||
("когда подействует?", "onset"),
|
||
("без пива не могу уснуть", "alcohol"),
|
||
("можно ли выпить вина?", "alcohol"),
|
||
("хочу бросить таблетки", "discontinuation"),
|
||
("как принимать, до еды или после еды?", "administration"),
|
||
("какая сегодня погода", None),
|
||
],
|
||
)
|
||
def test_detect_topic(question: str, topic: str | None) -> None:
|
||
assert detect_topic(question) == topic
|
||
|
||
|
||
async def test_explicit_topic_wins(info: FakeDrugInfo, mock_gateway: None) -> None:
|
||
# The classifier would say "discontinuation"; the engine decided the answer should be about onset.
|
||
answer = await med_qa("сертралин", S("брошу"), topic="onset", drug_info=info)
|
||
assert answer is not None and answer.section == "onset"
|
||
|
||
|
||
async def test_engine_alias_topic(info: FakeDrugInfo, mock_gateway: None) -> None:
|
||
answer = await med_qa("сертралин", S("ну и что"), topic="alcohol_interaction", drug_info=info)
|
||
assert answer is not None and answer.section == "alcohol"
|
||
|
||
|
||
# --- demo days ---------------------------------------------------------------------------------------
|
||
|
||
|
||
async def test_day2_nausea(info: FakeDrugInfo, mock_gateway: None) -> None:
|
||
answer = await med_qa("сертралин", S("подташнивает с утра"), drug_info=info)
|
||
assert answer is not None
|
||
assert answer.section == "side_effects" and answer.source == SRC
|
||
assert "1–2 недель" in answer.text and "после еды" in answer.text
|
||
assert answer.phrased_by_llm is False # mock echoes the excerpt
|
||
assert_safe(answer)
|
||
|
||
|
||
async def test_day5_quit(info: FakeDrugInfo, mock_gateway: None) -> None:
|
||
answer = await med_qa("сертралин", S("от таблеток только хуже, брошу"), topic="quit_intent", drug_info=info)
|
||
assert answer is not None and answer.section == "onset"
|
||
assert "2–4 недели" in answer.text
|
||
assert "200" not in answer.text # dosing sentence is dropped
|
||
assert "не прекращайте" in answer.text
|
||
assert_safe(answer)
|
||
|
||
|
||
async def test_day10_alcohol(info: FakeDrugInfo, mock_gateway: None) -> None:
|
||
answer = await med_qa("сертралин", S("без пива не могу уснуть"), drug_info=info)
|
||
assert answer is not None and answer.section == "alcohol"
|
||
assert "не рекомендуется" in answer.text
|
||
assert_safe(answer)
|
||
|
||
|
||
# --- retrieval -----------------------------------------------------------------------------------------
|
||
|
||
|
||
def test_select_excerpt_picks_relevant_sentences() -> None:
|
||
section = DrugSection(inn="сертралин", section="side_effects", text=FAKE_SERTRALINE["side_effects"], source=SRC)
|
||
excerpt = select_excerpt(section, "подташнивает с утра")
|
||
assert excerpt is not None
|
||
assert "тошнот" in excerpt.lower()
|
||
assert "головная боль" not in excerpt
|
||
|
||
|
||
def test_select_excerpt_none_when_not_relevant() -> None:
|
||
section = DrugSection(inn="сертралин", section="side_effects", text="Возможна сухость во рту.", source=SRC)
|
||
assert select_excerpt(section, "болит колено") is None
|
||
|
||
|
||
async def test_missing_section_returns_none(mock_gateway: None) -> None:
|
||
info = FakeDrugInfo({"side_effects": FAKE_SERTRALINE["side_effects"]})
|
||
assert await med_qa("сертралин", S("можно пиво?"), drug_info=info) is None
|
||
assert "врач" in fallback_text()
|
||
|
||
|
||
async def test_unknown_drug_and_unknown_topic(info: FakeDrugInfo, mock_gateway: None) -> None:
|
||
assert await med_qa("флуоксетин", S("тошнит"), drug_info=info) is None
|
||
assert await med_qa("сертралин", S("какая погода"), drug_info=info) is None
|
||
|
||
|
||
async def test_default_registry_has_no_data(mock_gateway: None) -> None:
|
||
# Before lane B (#14) registers real excerpts the default NoDrugInfo returns nothing.
|
||
assert await med_qa("сертралин", S("тошнит")) is None
|
||
|
||
|
||
async def test_drug_info_failure_returns_none(mock_gateway: None) -> None:
|
||
assert await med_qa("сертралин", S("тошнит"), drug_info=BrokenDrugInfo()) is None
|
||
|
||
|
||
@pytest.mark.parametrize("question", ["можно увеличить дозу?", "можно пить 100 мг?", "можно пить полтаблетки?"])
|
||
async def test_dose_questions_go_to_doctor(info: FakeDrugInfo, mock_gateway: None, question: str) -> None:
|
||
assert await med_qa("сертралин", S(question), topic="administration", drug_info=info) is None
|
||
assert info.calls == []
|
||
|
||
|
||
# --- LLM rephrasing guard --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_forbidden_llm_output_falls_back_to_excerpt(info: FakeDrugInfo, monkeypatch: pytest.MonkeyPatch) -> None:
|
||
use_llm(monkeypatch, "Тошнота пройдёт. Уменьшите дозу сертралина вдвое.")
|
||
answer = await med_qa("сертралин", S("подташнивает с утра"), drug_info=info)
|
||
assert answer is not None and answer.phrased_by_llm is False
|
||
assert "Тошнота обычно возникает" in answer.text
|
||
assert_safe(answer)
|
||
|
||
|
||
async def test_llm_adding_numbers_is_rejected(info: FakeDrugInfo, monkeypatch: pytest.MonkeyPatch) -> None:
|
||
use_llm(monkeypatch, "Тошнота обычно проходит за 3 дня.")
|
||
answer = await med_qa("сертралин", S("подташнивает с утра"), drug_info=info)
|
||
assert answer is not None and answer.phrased_by_llm is False
|
||
assert "3 дня" not in answer.text
|
||
|
||
|
||
async def test_llm_inventing_content_is_rejected(info: FakeDrugInfo, monkeypatch: pytest.MonkeyPatch) -> None:
|
||
use_llm(monkeypatch, "Попейте имбирный чай, погуляйте вечером и съешьте кусочек лимона.")
|
||
answer = await med_qa("сертралин", S("подташнивает с утра"), drug_info=info)
|
||
assert answer is not None and answer.phrased_by_llm is False
|
||
assert "имбир" not in answer.text
|
||
|
||
|
||
async def test_faithful_rephrasing_is_accepted(info: FakeDrugInfo, monkeypatch: pytest.MonkeyPatch) -> None:
|
||
reply = "Тошнота обычно бывает в первые дни приёма и проходит за 1–2 недели. Препарат принимают после еды."
|
||
use_llm(monkeypatch, reply)
|
||
answer = await med_qa("сертралин", S("подташнивает с утра"), drug_info=info)
|
||
assert answer is not None and answer.phrased_by_llm is True
|
||
assert reply in answer.text
|
||
assert_safe(answer)
|
||
|
||
|
||
def test_adds_content() -> None:
|
||
excerpt = "Тошнота обычно проходит в течение 1–2 недель."
|
||
assert adds_content("Тошнота проходит за 5 недель.", excerpt, "тошнит") is True
|
||
assert adds_content("Тошнота обычно проходит в течение 1–2 недель.", excerpt, "тошнит") is False
|
||
assert adds_content("", excerpt, "тошнит") is True
|