fix(llm): RAG citation pages, scrub placeholders, disease-aware document choice
CI / ruff · pytest (pull_request) Successful in 1m2s
CI / ruff · pytest (pull_request) Successful in 1m2s
- converter marks a paragraph glued across a page break with ⟦стр. N⟧; the chunker
strips the marker and extends the chunk's page range (wrong-page citations 24 -> 3,
the rest are scale-form table rows split by a page)
- [NAME]/[PHONE] placeholders from privacy.scrub() no longer count as unknown words
(a scrubbed question used to fall below the coverage gate)
- a question naming one disease boosts its guideline in retrieval and passage choice
("ТМС при депрессии" was answered from the PTSD guideline); "антидепрессант" is not a cue
Refs #62
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+4
-2
@@ -198,6 +198,7 @@ def best_passage(
|
||||
audience: Audience,
|
||||
cfg: AnswerConfig,
|
||||
defined: set[str] | None = None,
|
||||
cued: set[str] | None = None,
|
||||
) -> Passage | None:
|
||||
"""Best sentence of the chunk extended by adjacent relevant sentences (contiguous, within limits)."""
|
||||
sents = [s if _allowed(s, audience) else None for s in chunk_sentences(chunk, clinical=audience != "patient")]
|
||||
@@ -235,7 +236,7 @@ def best_passage(
|
||||
size += length(nxt)
|
||||
text = " ".join(s for s in sents[lo : hi + 1] if s)
|
||||
definition = scores[best] > index.overlap(head, terms) # got DEFINITION_BONUS
|
||||
return Passage(chunk, text, index.overlap(text, terms) * index.boost(chunk, audience), definition)
|
||||
return Passage(chunk, text, index.overlap(text, terms) * index.boost(chunk, audience, cued), definition)
|
||||
|
||||
|
||||
def select(index: BM25Index, question: str, audience: Audience, docs: set[str] | None = None) -> list[Passage]:
|
||||
@@ -244,7 +245,8 @@ def select(index: BM25Index, question: str, audience: Audience, docs: set[str] |
|
||||
hits = index.search(question, audience=audience, docs=docs, top_k=cfg.max_chunks)
|
||||
terms = index.query_terms(question)
|
||||
defined = _defined_term(question)
|
||||
passages = [p for h in hits if (p := best_passage(index, h.chunk, terms, audience, cfg, defined))]
|
||||
cued = index.cued_docs(question)
|
||||
passages = [p for h in hits if (p := best_passage(index, h.chunk, terms, audience, cfg, defined, cued))]
|
||||
if not passages:
|
||||
return []
|
||||
# A definition answers "что такое X?" better than any passage that merely mentions X often.
|
||||
|
||||
Reference in New Issue
Block a user