fix(llm): RAG citation pages, scrub placeholders, disease-aware document choice
CI / ruff · pytest (pull_request) Successful in 1m2s

- converter marks a paragraph glued across a page break with ⟦стр. N⟧; the chunker
  strips the marker and extends the chunk's page range (wrong-page citations 24 -> 3,
  the rest are scale-form table rows split by a page)
- [NAME]/[PHONE] placeholders from privacy.scrub() no longer count as unknown words
  (a scrubbed question used to fall below the coverage gate)
- a question naming one disease boosts its guideline in retrieval and passage choice
  ("ТМС при депрессии" was answered from the PTSD guideline); "антидепрессант" is not a cue

Refs #62

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-04 04:31:59 +03:00
co-authored by Claude Opus 5.5
parent 6c212d560c
commit fe29ebd629
8 changed files with 206 additions and 91 deletions
+22 -5
View File
@@ -20,6 +20,8 @@ from app.llm.rag.corpus import Audience, Chunk, RagConfig, load_chunks, load_con
from app.llm.rag.stemmer import stem
_WORD = re.compile(r"[а-яёa-z0-9]+(?:-[а-яёa-z0-9]+)*", re.IGNORECASE)
# `privacy.scrub()` placeholders ("[NAME]", "[PHONE]") are not words of the question.
_PLACEHOLDER = re.compile(r"\[[A-Z_]+\]")
# Function words and generic KR boilerplate that carry no topical signal.
STOP_WORDS = frozenset(
@@ -36,7 +38,7 @@ STOP_WORDS = frozenset(
def tokens(text: str) -> list[str]:
"""Stems of content words (hyphenated words are kept whole and also split: "когнитивно-поведенческая")."""
out: list[str] = []
for word in _WORD.findall(text.lower().replace("ё", "е")):
for word in _WORD.findall(_PLACEHOLDER.sub(" ", text).lower().replace("ё", "е")):
parts = [p for p in word.split("-") if p not in STOP_WORDS and (len(p) > 1 or p.isdigit())]
if "-" in word and len(parts) > 1:
out.append("-".join(stem(p) for p in parts)) # the compound itself, in any case form
@@ -97,6 +99,8 @@ class BM25Index:
self._k1, self._b, self._min_score, self._top_k = r.bm25_k1, r.bm25_b, r.min_score, r.top_k
self._candidates, self._window, self._window_weight = r.rerank_candidates, r.window_sentences, r.window_weight
self._section_boost = r.section_boost
self._doc_cues = {doc: re.compile("|".join(cues), re.IGNORECASE) for doc, cues in r.doc_cues.items() if cues}
self._doc_cue_boost = r.doc_cue_boost
self.chunks = chunks
self._sentences = [[(s, set(tokens(s))) for s in sentences(c.text)] for c in chunks]
self._expander = SynonymExpander(cfg.synonyms)
@@ -124,10 +128,20 @@ class BM25Index:
score += self._idf[term] * f * (self._k1 + 1) / (f + norm) * weight
return score
def boost(self, chunk: Chunk, audience: Audience | None) -> float:
"""Section multiplier for the audience (`retrieval.section_boost`), 1.0 by default."""
def boost(self, chunk: Chunk, audience: Audience | None, cued: set[str] | None = None) -> float:
"""Section multiplier for the audience (`retrieval.section_boost`) × disease-cue multiplier."""
boosts = self._section_boost.get(audience or "", {})
return next((b for prefix, b in boosts.items() if chunk.section.startswith(prefix)), 1.0)
section = next((b for prefix, b in boosts.items() if chunk.section.startswith(prefix)), 1.0)
return section * (self._doc_cue_boost if cued and chunk.doc.id in cued else 1.0)
def cued_docs(self, query: str) -> set[str]:
"""Documents whose disease the query names ("ТМС при депрессии" -> {"kr_depression"}).
Only a one-sided mention counts: a question naming both disorders favours neither.
"""
q = _PLACEHOLDER.sub(" ", query).replace("ё", "е")
cued = {doc for doc, cue in self._doc_cues.items() if cue.search(q)}
return cued if len(cued) < len(self._doc_cues) else set()
def overlap(self, text: str, terms: dict[str, float]) -> float:
"""Sum of idf × weight of the query terms present in `text` (sentence relevance for `answer()`)."""
@@ -193,11 +207,14 @@ class BM25Index:
and (s := self._score(i, terms)) >= threshold
]
scored.sort(key=lambda x: (-x[0], x[1]))
cued = self.cued_docs(query)
hits = []
for bm25, i in scored[: max(self._candidates, top_k or 0)]:
bonus, passage = self._passage(i, terms)
chunk = self.chunks[i]
hits.append(Hit(chunk, (bm25 + self._window_weight * bonus) * self.boost(chunk, audience), bm25, passage))
hits.append(
Hit(chunk, (bm25 + self._window_weight * bonus) * self.boost(chunk, audience, cued), bm25, passage)
)
hits.sort(key=lambda h: -h.score) # stable: BM25 order breaks ties
return hits[: top_k or self._top_k]