fix(llm): RAG citation pages, scrub placeholders, disease-aware document choice
CI / ruff · pytest (pull_request) Successful in 1m2s
CI / ruff · pytest (pull_request) Successful in 1m2s
- converter marks a paragraph glued across a page break with ⟦стр. N⟧; the chunker
strips the marker and extends the chunk's page range (wrong-page citations 24 -> 3,
the rest are scale-form table rows split by a page)
- [NAME]/[PHONE] placeholders from privacy.scrub() no longer count as unknown words
(a scrubbed question used to fall below the coverage gate)
- a question naming one disease boosts its guideline in retrieval and passage choice
("ТМС при депрессии" was answered from the PTSD guideline); "антидепрессант" is not a cue
Refs #62
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+22
-5
@@ -20,6 +20,8 @@ from app.llm.rag.corpus import Audience, Chunk, RagConfig, load_chunks, load_con
|
||||
from app.llm.rag.stemmer import stem
|
||||
|
||||
_WORD = re.compile(r"[а-яёa-z0-9]+(?:-[а-яёa-z0-9]+)*", re.IGNORECASE)
|
||||
# `privacy.scrub()` placeholders ("[NAME]", "[PHONE]") are not words of the question.
|
||||
_PLACEHOLDER = re.compile(r"\[[A-Z_]+\]")
|
||||
|
||||
# Function words and generic KR boilerplate that carry no topical signal.
|
||||
STOP_WORDS = frozenset(
|
||||
@@ -36,7 +38,7 @@ STOP_WORDS = frozenset(
|
||||
def tokens(text: str) -> list[str]:
|
||||
"""Stems of content words (hyphenated words are kept whole and also split: "когнитивно-поведенческая")."""
|
||||
out: list[str] = []
|
||||
for word in _WORD.findall(text.lower().replace("ё", "е")):
|
||||
for word in _WORD.findall(_PLACEHOLDER.sub(" ", text).lower().replace("ё", "е")):
|
||||
parts = [p for p in word.split("-") if p not in STOP_WORDS and (len(p) > 1 or p.isdigit())]
|
||||
if "-" in word and len(parts) > 1:
|
||||
out.append("-".join(stem(p) for p in parts)) # the compound itself, in any case form
|
||||
@@ -97,6 +99,8 @@ class BM25Index:
|
||||
self._k1, self._b, self._min_score, self._top_k = r.bm25_k1, r.bm25_b, r.min_score, r.top_k
|
||||
self._candidates, self._window, self._window_weight = r.rerank_candidates, r.window_sentences, r.window_weight
|
||||
self._section_boost = r.section_boost
|
||||
self._doc_cues = {doc: re.compile("|".join(cues), re.IGNORECASE) for doc, cues in r.doc_cues.items() if cues}
|
||||
self._doc_cue_boost = r.doc_cue_boost
|
||||
self.chunks = chunks
|
||||
self._sentences = [[(s, set(tokens(s))) for s in sentences(c.text)] for c in chunks]
|
||||
self._expander = SynonymExpander(cfg.synonyms)
|
||||
@@ -124,10 +128,20 @@ class BM25Index:
|
||||
score += self._idf[term] * f * (self._k1 + 1) / (f + norm) * weight
|
||||
return score
|
||||
|
||||
def boost(self, chunk: Chunk, audience: Audience | None) -> float:
|
||||
"""Section multiplier for the audience (`retrieval.section_boost`), 1.0 by default."""
|
||||
def boost(self, chunk: Chunk, audience: Audience | None, cued: set[str] | None = None) -> float:
|
||||
"""Section multiplier for the audience (`retrieval.section_boost`) × disease-cue multiplier."""
|
||||
boosts = self._section_boost.get(audience or "", {})
|
||||
return next((b for prefix, b in boosts.items() if chunk.section.startswith(prefix)), 1.0)
|
||||
section = next((b for prefix, b in boosts.items() if chunk.section.startswith(prefix)), 1.0)
|
||||
return section * (self._doc_cue_boost if cued and chunk.doc.id in cued else 1.0)
|
||||
|
||||
def cued_docs(self, query: str) -> set[str]:
|
||||
"""Documents whose disease the query names ("ТМС при депрессии" -> {"kr_depression"}).
|
||||
|
||||
Only a one-sided mention counts: a question naming both disorders favours neither.
|
||||
"""
|
||||
q = _PLACEHOLDER.sub(" ", query).replace("ё", "е")
|
||||
cued = {doc for doc, cue in self._doc_cues.items() if cue.search(q)}
|
||||
return cued if len(cued) < len(self._doc_cues) else set()
|
||||
|
||||
def overlap(self, text: str, terms: dict[str, float]) -> float:
|
||||
"""Sum of idf × weight of the query terms present in `text` (sentence relevance for `answer()`)."""
|
||||
@@ -193,11 +207,14 @@ class BM25Index:
|
||||
and (s := self._score(i, terms)) >= threshold
|
||||
]
|
||||
scored.sort(key=lambda x: (-x[0], x[1]))
|
||||
cued = self.cued_docs(query)
|
||||
hits = []
|
||||
for bm25, i in scored[: max(self._candidates, top_k or 0)]:
|
||||
bonus, passage = self._passage(i, terms)
|
||||
chunk = self.chunks[i]
|
||||
hits.append(Hit(chunk, (bm25 + self._window_weight * bonus) * self.boost(chunk, audience), bm25, passage))
|
||||
hits.append(
|
||||
Hit(chunk, (bm25 + self._window_weight * bonus) * self.boost(chunk, audience, cued), bm25, passage)
|
||||
)
|
||||
hits.sort(key=lambda h: -h.score) # stable: BM25 order breaks ties
|
||||
return hits[: top_k or self._top_k]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user