From 585f0af2731ec6710f5b97e78e1640c573688227 Mon Sep 17 00:00:00 2001 From: Lacky1234union Date: Sat, 3 Oct 2026 15:00:19 +0300 Subject: [PATCH] feat(core): implement scrub() and pseudo_id() with tests Rule-based PII scrubber: phones, e-mails, SNILS (checksum-aware), passport, INN, cards, dates of birth, addresses, URLs/handles and person names (dictionary in config/privacy/names_ru.yaml + declension, patronymic and surname heuristics). Idempotent via fixed-point iteration; false-positive guards for ordinary clinical text. pseudo_id normalises SNILS to 11 digits and returns HMAC-SHA256 truncated to 32 hex chars. Refs #2 Co-Authored-By: Claude Opus 5.5 --- app/core/privacy.py | 441 ++++++++++++++++++++++++++++++++++- config/privacy/names_ru.yaml | 254 ++++++++++++++++++++ tests/test_privacy.py | 199 ++++++++++++++++ 3 files changed, 890 insertions(+), 4 deletions(-) create mode 100644 config/privacy/names_ru.yaml create mode 100644 tests/test_privacy.py diff --git a/app/core/privacy.py b/app/core/privacy.py index 4f0087a..5cd0522 100644 --- a/app/core/privacy.py +++ b/app/core/privacy.py @@ -1,16 +1,449 @@ -"""PII scrubbing and pseudonymisation (issue #2). Contract stub — implementation lands in #2.""" +"""PII scrubbing and pseudonymisation (issue #2, guardrail #1). +Why this module exists +---------------------- +Messages from patients are health data — a special category of personal data under 152-FZ art. 10 — +and are covered by medical secrecy (323-FZ art. 13). Guardrail #1: **only text that passed `scrub()` +may reach the LLM** (`app/llm/gateway.py` accepts `ScrubbedText` only), and only scrubbed text is +stored in `messages.text_scrubbed`. Direct identifiers live solely in the identity DB; the clinical DB +is linked to it by `pseudo_id()` — an HMAC token that cannot be reversed without `PSEUDO_KEY`. + +Design +------ +`scrub()` is a deterministic rule-based pipeline (no ML, works offline, explainable to the jury): +ordered regex passes replace structured identifiers with bracketed placeholders, then a token pass +finds person names using a dictionary (`config/privacy/names_ru.yaml`) plus morphology heuristics. +The order matters: longer / more specific number formats go first so that, e.g., a card number is not +partially eaten by the phone pattern. Placeholders are Latin and bracketed, so no later pass (which +looks for digits or Cyrillic) can match them — this is what makes the function idempotent; on top of +that the pipeline is iterated to a fixed point. + +The trade-off is deliberate: over-scrubbing a city or a capitalised surname-like word is cheap, +leaking an identifier to an external model is not. At the same time ordinary clinical text (doses, +scale scores, durations, times, lowercase complaints) must survive intact, because the LLM extracts +symptom data from it — this is covered by false-positive tests in `tests/test_privacy.py`. +""" + +from __future__ import annotations + +import hashlib +import hmac +import re +from collections.abc import Callable +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path + +import yaml +from pydantic import BaseModel + +from app.config import get_settings from app.core.types import ScrubbedText +# --------------------------------------------------------------------------------------------------- +# Regex passes for structured identifiers +# --------------------------------------------------------------------------------------------------- + +_SOCIAL = re.compile( + r"(?:https?://)?(?:www\.)?" + r"(?:t\.me|telegram\.me|vk\.com|vk\.ru|ok\.ru|instagram\.com|facebook\.com|max\.ru)/[^\s]+", + re.IGNORECASE, +) +_URL = re.compile( + r"(?:https?://|www\.)[^\s]+" + r"|\b[a-z0-9][a-z0-9-]*(?:\.[a-z0-9-]+)*\.(?:ru|com|рф|org|net|su|io|me|info)\b(?:/[^\s]*)?", + re.IGNORECASE, +) +_EMAIL = re.compile(r"[\w.+-]+@[\w-]+(?:\.[\w-]+)+") +_HANDLE = re.compile(r"(? bool: + """Official SNILS control number: weighted sum of the first 9 digits (weights 9..1) mod 101.""" + total = sum(int(d) * w for d, w in zip(digits[:9], range(9, 0, -1), strict=True)) + control = total % 101 + if control == 100: + control = 0 + return control == int(digits[9:]) + + +def _phone_or_snils(m: re.Match[str]) -> str: + # A contiguous 11-digit "8XXXXXXXXXX" could also be a SNILS; the checksum disambiguates. + raw = m.group(0) + if raw.isdigit() and len(raw) == 11 and _snils_checksum_ok(raw): + return "[SNILS]" + return "[PHONE]" + + +def _url_repl(placeholder: str) -> Callable[[re.Match[str]], str]: + def repl(m: re.Match[str]) -> str: + # Keep sentence punctuation that a greedy `\S+` swallowed: "см. t.me/x." → "см. [HANDLE]." + raw = m.group(0) + stripped = raw.rstrip(".,;:!?)»\"'") + return placeholder + raw[len(stripped) :] + + return repl + + +_REGEX_PASSES: list[tuple[re.Pattern[str], str | Callable[[re.Match[str]], str]]] = [ + (_SOCIAL, _url_repl("[HANDLE]")), + (_EMAIL, "[EMAIL]"), + (_URL, _url_repl("[URL]")), + (_HANDLE, "[HANDLE]"), + (_CARD, "[CARD]"), + (_SNILS_FMT, "[SNILS]"), + (_PHONE_FULL, _phone_or_snils), + (_PHONE_MOBILE, "[PHONE]"), + (_PHONE_LOCAL, "[PHONE]"), + (_PASSPORT_KW, "[PASSPORT]"), + (_PASSPORT_BARE, "[PASSPORT]"), + (_SNILS_PLAIN, "[SNILS]"), + (_INN, "[INN]"), + *((p, "[DATE]") for p in _DATES), + (_BIRTH_YEAR, lambda m: m.group(1) + "[DATE]"), + *((p, "[ADDRESS]") for p in _ADDRESSES), + *((p, "[NAME]") for p in _INITIALS), +] + +# --------------------------------------------------------------------------------------------------- +# Person names: dictionary + morphology +# --------------------------------------------------------------------------------------------------- + + +class NameLexicon(BaseModel): + """Schema of `config/privacy/names_ru.yaml` (values may be extended by the clinical zone).""" + + names: list[str] + ambiguous_names: list[str] = [] + ambiguous_forms: list[str] = [] + surname_stopwords: list[str] = [] + + +@dataclass(frozen=True) +class _NameIndex: + forms: frozenset[str] # every known name form, normalised (lowercase, ё→е) + ambiguous: frozenset[str] # forms that are also ordinary Russian words + surname_stop: frozenset[str] + + +def _norm(word: str) -> str: + return word.lower().replace("ё", "е") + + +_HUSHING = set("жшщчц") +_VELAR_HUSH = set("гкхжшщч") + + +def _declensions(name: str) -> set[str]: + """Generate singular case forms of a Russian first name by its ending. + + Covers the regular paradigms (Иван, Алексей, Дмитрий, Игорь, Ирина, Мария, Таня, Саша); irregular + ones (fleeting vowels: Павел→Павла) are listed explicitly in the YAML. + """ + n = _norm(name) + forms = {n} + if n.endswith("ий"): + stem = n[:-2] + forms |= {stem + e for e in ("ия", "ию", "ием", "ии")} + elif n.endswith(("ей", "ай", "ой")): + stem = n[:-1] + forms |= {stem + e for e in ("я", "ю", "ем", "е")} + elif n.endswith("ия"): + stem = n[:-1] + forms |= {stem + e for e in ("и", "ю", "ей")} + elif n.endswith("а"): + stem = n[:-1] + gen = "и" if stem[-1] in _VELAR_HUSH else "ы" + ins = "ей" if stem[-1] in _HUSHING else "ой" + forms |= {stem + gen, stem + "е", stem + "у", stem + ins} + elif n.endswith("я"): + stem = n[:-1] + forms |= {stem + e for e in ("и", "е", "ю", "ей")} + elif n.endswith("ь"): + stem = n[:-1] + forms |= {stem + e for e in ("я", "ю", "ем", "е", "и")} + else: + ins = "ем" if n[-1] in _HUSHING else "ом" + forms |= {n + "а", n + "у", n + ins, n + "е"} + return forms + + +def _lexicon_path() -> Path: + return get_settings().config_dir / "privacy" / "names_ru.yaml" + + +@lru_cache +def _name_index() -> _NameIndex: + lex = NameLexicon.model_validate(yaml.safe_load(_lexicon_path().read_text(encoding="utf-8"))) + forms: set[str] = set() + ambiguous = {_norm(f) for f in lex.ambiguous_forms} + ambiguous_names = {_norm(n) for n in lex.ambiguous_names} + for line in lex.names: + head, *extra = line.split() + name_forms = _declensions(head) | {_norm(e) for e in extra} + forms |= name_forms + if _norm(head) in ambiguous_names: + ambiguous |= name_forms + stop = frozenset(_norm(w) for w in lex.surname_stopwords) + return _NameIndex(frozenset(forms), frozenset(ambiguous), stop) + + +_TOKEN = re.compile(r"[А-ЯЁа-яё]+(?:-[А-ЯЁа-яё]+)*") +_CAPITALISED = re.compile(r"^[А-ЯЁ][а-яё]+(?:-[А-ЯЁ][а-яё]+)*$") +# Patronymics: -ович/-евич/-ьич (+ case endings), -овна/-евна/-ична (+ case endings). +_PATRONYMIC = re.compile( + r"^[а-яё]+(?:(?:(?:ов|ев|ёв)ич|ьич|инич)(?:а|у|ем|е)?|(?:(?:ов|ев|ёв)н|ичн)(?:а|ы|е|у|ой))$" +) +# Strong surname suffixes: -ов/-ев/-ёв, -ский/-цкий, -енко with case endings. +_SURNAME_STRONG = re.compile( + r"^[а-яё]{2,}(?:(?:ов|ев|ёв)(?:а|у|ым|ой|ою|е|ых)?|(?:ск|цк)(?:ий|ая|ого|ому|им|ой|ую|ом)|енко)$" +) +# Weak: -ин/-ын are also drug names (Феварин, Глицин), so they count only next to other name evidence. +_SURNAME_WEAK = re.compile(r"^[а-яё]{2,}(?:ин|ын)(?:а|у|ым|ой|е)?$") +_TRIGGERS = frozenset({"зовут", "звать", "имя", "фамилия", "фио", "отчество"}) +_MAX_NAME_TOKENS = 4 + + +@dataclass +class _Tok: + text: str + start: int + end: int + low: str # normalised + cap: bool + initial: bool # starts a sentence + + +def _sentence_initial(text: str, pos: int) -> bool: + i = pos - 1 + while i >= 0 and text[i] in " \t«\"'(—–-": + i -= 1 + return i < 0 or text[i] in ".!?…\n" + + +def _is_surname(tok: _Tok, idx: _NameIndex, *, weak: bool) -> bool: + if tok.low in idx.surname_stop: + return False + return bool(_SURNAME_STRONG.match(tok.low) or (weak and _SURNAME_WEAK.match(tok.low))) + + +def _is_patronymic(tok: _Tok) -> bool: + return tok.cap and bool(_PATRONYMIC.match(tok.low)) + + +def _dict_hit(tok: _Tok, idx: _NameIndex) -> tuple[bool, bool]: + """(is a known name form, is ambiguous). Hyphenated names match if any part is a known name.""" + if tok.low in idx.forms: + return True, tok.low in idx.ambiguous + parts = tok.low.split("-") + if len(parts) > 1 and tok.cap and any(p in idx.forms and p not in idx.ambiguous for p in parts): + return True, False + return False, False + + +def _name_spans(text: str) -> list[tuple[int, int]]: + idx = _name_index() + toks = [ + _Tok(m.group(0), m.start(), m.end(), _norm(m.group(0)), m.group(0)[0].isupper(), False) + for m in _TOKEN.finditer(text) + ] + for t in toks: + t.initial = _sentence_initial(text, t.start) + + def adjacent(a: _Tok, b: _Tok) -> bool: + gap = text[a.end : b.start] + return gap.isspace() and "\n" not in gap + + def after_trigger(i: int) -> bool: + if i == 0 or toks[i - 1].low not in _TRIGGERS: + return False + return re.fullmatch(r"\s*[:—–-]?\s*", text[toks[i - 1].end : toks[i].start]) is not None + + def strong_neighbour(i: int) -> bool: + if i + 1 < len(toks) and adjacent(toks[i], toks[i + 1]): + nxt = toks[i + 1] + return nxt.cap and (_is_patronymic(nxt) or _is_surname(nxt, idx, weak=True)) + return False + + def is_name_anchor(i: int, t: _Tok) -> bool: + known, ambiguous = _dict_hit(t, idx) + if after_trigger(i): + # "меня зовут Алексей" / "меня зовут алексей" (lowercase only if it is a known name). + return bool(_CAPITALISED.match(t.text)) or known + if known and not ambiguous: + return t.cap or len(t.low) >= 3 + if known and ambiguous: + # "Вера" mid-sentence or before a patronymic is a name; sentence-initial "Вера в себя" is not. + return t.cap and (not t.initial or strong_neighbour(i)) + return _is_patronymic(t) + + anchors: set[int] = set() + for i, t in enumerate(toks): + cap_word = bool(_CAPITALISED.match(t.text)) + if is_name_anchor(i, t): + anchors.add(i) + elif cap_word and not t.initial and _is_surname(t, idx, weak=False): + # Mid-sentence capitalised -ов/-ская word: "был у Петрова". May over-scrub a city (Ростов). + anchors.add(i) + elif ( + cap_word + and not t.initial + and i + 1 < len(toks) + and adjacent(t, toks[i + 1]) + and _CAPITALISED.match(toks[i + 1].text) + ): + # Two capitalised words mid-sentence: "встретил Петрова Ивана". + anchors.update({i, i + 1}) + + spans: list[tuple[int, int]] = [] + covered: set[int] = set() + for i in sorted(anchors): + if i in covered: + continue + lo = hi = i + lower_anchor = not toks[i].cap + # Extend right over the rest of the full name (surname, patronymic). + while hi + 1 < len(toks) and hi - lo + 1 < _MAX_NAME_TOKENS and adjacent(toks[hi], toks[hi + 1]): + nxt = toks[hi + 1] + capitalised = nxt.cap and _CAPITALISED.match(nxt.text) + # "алексей петров" typed in lowercase: extend only over a clearly surname-like word. + lower_tail = lower_anchor and (_is_surname(nxt, idx, weak=False) or _PATRONYMIC.match(nxt.low)) + if capitalised or (hi + 1) in anchors or lower_tail: + hi += 1 + else: + break + # Extend left only over a surname-looking word ("Иванов Иван"), never over "Сегодня Иван". + while lo > 0 and hi - lo + 1 < _MAX_NAME_TOKENS and adjacent(toks[lo - 1], toks[lo]): + prv = toks[lo - 1] + if prv.cap and _CAPITALISED.match(prv.text) and _is_surname(prv, idx, weak=True): + lo -= 1 + else: + break + covered.update(range(lo, hi + 1)) + spans.append((toks[lo].start, toks[hi].end)) + return spans + + +def _scrub_names(text: str) -> str: + out: list[str] = [] + pos = 0 + for start, end in _name_spans(text): + out.append(text[pos:start]) + out.append("[NAME]") + pos = end + out.append(text[pos:]) + # "[NAME] [NAME]" produced by separate spans is one person in practice. + return re.sub(r"\[NAME\](?:[ \t]+\[NAME\])+", "[NAME]", "".join(out)) + + +def _scrub_once(text: str) -> str: + for pattern, repl in _REGEX_PASSES: + text = pattern.sub(repl, text) + return _scrub_names(text) + + +_MAX_PASSES = 5 + def scrub(text: str) -> ScrubbedText: - """Replace personal data (phones, emails, SNILS, names, addresses...) with placeholders.""" - raise NotImplementedError + """Replace personal data (phones, emails, SNILS, names, addresses...) with placeholders. + + Placeholders: [PHONE] [EMAIL] [SNILS] [PASSPORT] [INN] [CARD] [DATE] [ADDRESS] [NAME] [URL] [HANDLE]. + Idempotent: `scrub(scrub(x)) == scrub(x)`. + """ + # Removing one identifier can expose another pattern boundary; iterate to a fixed point. + for _ in range(_MAX_PASSES): + new = _scrub_once(text) + if new == text: + break + text = new + return ScrubbedText(text) def pseudo_id(snils: str, key: str | None = None) -> str: """HMAC-SHA256(PSEUDO_KEY, normalised SNILS) as 32 hex chars. The token is the only link between the identity and clinical databases. + SNILS is normalised to 11 digits (spaces and dashes removed); anything else raises `ValueError`. + The checksum is not validated: demo data is synthetic. """ - raise NotImplementedError + digits = re.sub(r"[\s\-]", "", snils) + if len(digits) != 11 or not digits.isascii() or not digits.isdigit(): + raise ValueError("SNILS must contain exactly 11 digits") + secret = (key if key is not None else get_settings().pseudo_key).encode("utf-8") + return hmac.new(secret, digits.encode("ascii"), hashlib.sha256).hexdigest()[:32] diff --git a/config/privacy/names_ru.yaml b/config/privacy/names_ru.yaml new file mode 100644 index 0000000..f091a0c --- /dev/null +++ b/config/privacy/names_ru.yaml @@ -0,0 +1,254 @@ +# Name lexicon for app.core.privacy.scrub() (issue #2). Synthetic-safe: just a list of common Russian names. +# +# names — nominative forms; case forms (Ивана, Ирине, Сашей...) are generated in code. +# Extra explicit forms may follow on the same line, space-separated +# (needed for fleeting vowels: "Павел Павла Павлу..."). +# ambiguous_names — names that are also ordinary words (вера, надежда, роман...). All their forms are +# treated as names only with context: capitalised mid-sentence or next to a +# patronymic/surname, or after "меня зовут". +# ambiguous_forms — single generated forms that collide with ordinary words (кости, коли, жене...). +# surname_stopwords — capitalised words that look like surnames (-ов/-ова...) but are ordinary words. + +names: + # male + - Александр + - Алексей + - Анатолий + - Андрей + - Антон + - Аркадий + - Арсений + - Артём + - Артем + - Артур + - Борис + - Вадим + - Валентин + - Валерий + - Василий + - Виктор + - Виталий + - Владимир + - Владислав + - Вячеслав + - Геннадий + - Георгий + - Глеб + - Григорий + - Даниил + - Данил + - Данила + - Денис + - Дмитрий + - Евгений + - Егор + - Ефим + - Иван + - Игорь + - Илья + - Ильдар + - Кирилл + - Константин + - Леонид + - Максим + - Марат + - Матвей + - Михаил + - Никита + - Николай + - Олег + - Павел Павла Павлу Павлом Павле + - Пётр Петра Петру Петром Петре + - Петр Петра Петру Петром Петре + - Ринат + - Ростислав + - Руслан + - Рустам + - Святослав + - Семён + - Семен + - Сергей + - Станислав + - Степан + - Тарас + - Тимофей + - Тимур + - Фёдор + - Федор + - Филипп + - Эдуард + - Юрий + - Яков + - Ярослав + # male diminutives + - Алёша + - Алеша + - Лёша + - Леша + - Боря + - Ваня + - Вася + - Витя + - Вова + - Володя + - Влад + - Гоша + - Гриша + - Даня + - Дима + - Егорка + - Женя + - Костя + - Коля + - Лёня + - Леня + - Макс + - Миша + - Митя + - Паша + - Петя + - Рома + - Саша + - Серёжа + - Сережа + - Стас + - Стёпа + - Степа + - Толя + - Федя + - Юра + - Гена + # female + - Алина + - Алиса + - Алла + - Алёна + - Алена + - Анастасия + - Ангелина + - Анна + - Антонина + - Валентина + - Варвара + - Василиса + - Вероника + - Галина + - Дарья + - Диана + - Евгения + - Екатерина + - Елена + - Елизавета + - Жанна + - Зинаида + - Инна + - Ирина + - Карина + - Кристина + - Ксения + - Лариса + - Лидия + - Людмила + - Маргарита + - Марина + - Мария + - Милана + - Наталья + - Наталия + - Нина + - Оксана + - Олеся + - Ольга + - Полина + - Раиса + - Регина + - Светлана + - Софья + - София + - Тамара + - Татьяна + - Ульяна + - Эльвира + - Юлия + - Яна + # female diminutives + - Аня + - Валя + - Варя + - Вика + - Галя + - Даша + - Ира + - Катя + - Ксюша + - Лена + - Лиза + - Люда + - Маша + - Надя + - Настя + - Наташа + - Оля + - Поля + - Рита + - Таня + - Юля + # ambiguous ones are listed in names too, so their case forms are generated + - Вера + - Виктория + - Лев Льва Льву Львом Льве + - Любовь Любови Любовью + - Люба + - Мила + - Надежда + - Ника + - Роза + - Роман + - Света + - Слава + - Соня + - Тома + +ambiguous_names: + - Вера + - Виктория + - Лев + - Любовь + - Люба + - Мила + - Надежда + - Ника + - Роза + - Роман + - Света + - Слава + - Соня + - Тома + - Поля + +ambiguous_forms: + - коли + - вали + - кости + - костей + - жене + - жени + - гены + - машу + - семена + - вари + - кати + +surname_stopwords: + - снова + - готов + - готова + - здоров + - здорова + - слова + - основа + - один + - часов + - разов + - врачей + - сонов diff --git a/tests/test_privacy.py b/tests/test_privacy.py new file mode 100644 index 0000000..31abfc1 --- /dev/null +++ b/tests/test_privacy.py @@ -0,0 +1,199 @@ +"""Tests for app.core.privacy (issue #2, guardrail #1: only scrubbed text reaches the LLM).""" + +import pytest + +from app.core.privacy import pseudo_id, scrub + +# --------------------------------------------------------------------------------------------------- +# scrub(): identifiers are replaced +# --------------------------------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + # phones + ("Мой телефон +7 (916) 123-45-67, звоните", "Мой телефон [PHONE], звоните"), + ("номер 8-916-123-45-67", "номер [PHONE]"), + ("номер 89161234567", "номер [PHONE]"), + ("+79161234567", "[PHONE]"), + ("горячая линия 8 800 2000 122", "горячая линия [PHONE]"), + ("мобильный 916 123 45 67", "мобильный [PHONE]"), + ("домашний 123-45-67", "домашний [PHONE]"), + # e-mail, links, handles + ("пишите на ivan.petrov@mail.ru", "пишите на [EMAIL]"), + ("или почта@мэйл.рф", "или [EMAIL]"), + ("см. https://example.com/page?x=1.", "см. [URL]."), + ("сайт www.clinic.ru", "сайт [URL]"), + ("мой канал t.me/ivan_99", "мой канал [HANDLE]"), + ("страница https://vk.com/id123", "страница [HANDLE]"), + ("ник @ivan_petrov в телеге", "ник [HANDLE] в телеге"), + # documents + ("СНИЛС 112-233-445 95", "СНИЛС [SNILS]"), + ("СНИЛС 11223344595", "СНИЛС [SNILS]"), + ("паспорт серия 4510 номер 123456", "паспорт [PASSPORT]"), + ("паспорт 45 10 123456", "паспорт [PASSPORT]"), + ("паспорт 4510 123456", "паспорт [PASSPORT]"), + ("ИНН 7707083893", "ИНН [INN]"), + ("ИНН 500100732259", "ИНН [INN]"), + ("карта 4276 3800 1234 5678", "карта [CARD]"), + ("карта 4276-3800-1234-5678", "карта [CARD]"), + ("карта 4276380012345678", "карта [CARD]"), + # dates of birth + ("родился 12.03.1985", "родился [DATE]"), + ("д.р. 1985-03-12", "д.р. [DATE]"), + ("родилась 5 мая 1988 г.", "родилась [DATE]"), + ("1985 г.р.", "[DATE]"), + ("дата рождения: 1990", "дата рождения: [DATE]"), + ("родился в 1990 году", "родился в [DATE] году"), + # addresses + ("живу: г. Москва, ул. Ленина, д. 5, кв. 12", "живу: [ADDRESS]"), + ("улица Тверская 7", "[ADDRESS]"), + ("Ленинский проспект, 32", "[ADDRESS]"), + ("пр-т Мира, д. 10, корп. 2, кв. 45", "[ADDRESS]"), + ("ул. 8 Марта 3", "[ADDRESS]"), + ("а потом дом 7 кв. 3", "а потом [ADDRESS]"), + ], +) +def test_scrub_identifiers(text: str, expected: str) -> None: + assert scrub(text) == expected + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("Меня зовут Алексей Петров", "Меня зовут [NAME]"), + ("меня зовут алексей", "меня зовут [NAME]"), + ("Иванов Иван Иванович пришёл", "[NAME] пришёл"), + ("врач д-р Смирнова Е. А., психиатр", "врач д-р [NAME], психиатр"), + ("Е.А. Смирнова назначила", "[NAME] назначила"), + ("Сегодня Саша опять не спала", "Сегодня [NAME] опять не спала"), + ("мой брат саша приехал", "мой брат [NAME] приехал"), + ("звонил Алексею вчера", "звонил [NAME] вчера"), + ("встретил Ирину Сергеевну", "встретил [NAME]"), + ("был у Петрова", "был у [NAME]"), + ("Вера Ивановна сказала", "[NAME] сказала"), + ("Вчера звонила Вера.", "Вчера звонила [NAME]."), + ("Надежда Петрова пришла", "[NAME] пришла"), + ("встретил Петрова Ивана", "встретил [NAME]"), + ("его зовут Тимофей", "его зовут [NAME]"), + ("Анна-Мария приехала", "[NAME] приехала"), + ("с Павлом и Леной", "с [NAME] и [NAME]"), + ], +) +def test_scrub_names(text: str, expected: str) -> None: + assert scrub(text) == expected + + +def test_scrub_full_message() -> None: + text = ( + "Здравствуйте, я Алексей Смирнов, СНИЛС 112-233-445 95, тел. +7 916 123 45 67.\n" + "Живу на ул. Ленина, д. 5, кв. 12. Сплю плохо, тошнота 3 дня." + ) + assert scrub(text) == ( + "Здравствуйте, я [NAME], СНИЛС [SNILS], тел. [PHONE].\nЖиву на [ADDRESS]. Сплю плохо, тошнота 3 дня." + ) + + +# --------------------------------------------------------------------------------------------------- +# scrub(): ordinary clinical text is not mangled (false-positive guards) +# --------------------------------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "text", + [ + "сплю плохо, кошмары почти каждую ночь", + "Сплю плохо, кошмары почти каждую ночь", + "выпил 2 бутылки пива", + "сертралин 50 мг 1 р/сут утром после еды", + "PHQ-9 = 14, п.9 = 0", + "GAD-7: 9, PCL-5 = 45", + "тошнота 3 дня подряд", + "принял таблетку в 08:00, уснул в 23:30", + "сплю 4-6 часов", + "давление 120/80, пульс 90, вес 75.5 кг", + "0.5 мг на ночь, полтаблетки 1/2", + "12 марта был у врача", + "в 2010 г. Мне стало хуже", + "и т.д. 5 раз за ночь", + "на улице страшно, проезд оплатил", + "Снова не спал. Сегодня лучше.", + "Слава богу, лучше. Роман читаю.", + "Надежда есть.", + "Любой ценой. Света нет, боюсь темноты.", + "кости ломит, коли укол", + "Принимаю Феварин и Глицин", + "Не могу уснуть без пива.", + "Всё бессмысленно, устал от всего", + "От таблеток только хуже, брошу", + "Был в ПНД, сказали прийти через 4 недели", + "день 13, настроение 2 из 10", + ], +) +def test_scrub_keeps_clinical_text(text: str) -> None: + assert scrub(text) == text + + +# --------------------------------------------------------------------------------------------------- +# scrub(): idempotency and type +# --------------------------------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "text", + [ + "Меня зовут Алексей Петров, тел. 8 (916) 123-45-67, ivan@mail.ru", + "СНИЛС 112-233-445 95, паспорт 4510 123456, ИНН 7707083893, карта 4276 3800 1234 5678", + "г. Москва, ул. Ленина, д. 5, кв. 12; родился 12.03.1985; t.me/ivan; @ivan_petrov", + "д-р Смирнова Е. А. и Иванов Иван Иванович", + "сертралин 50 мг, PHQ-9 = 14, сплю 4-6 часов", + "[NAME] [PHONE] [EMAIL]", + ], +) +def test_scrub_is_idempotent(text: str) -> None: + once = scrub(text) + assert scrub(once) == once + + +def test_scrub_returns_str_and_handles_empty() -> None: + assert scrub("") == "" + assert isinstance(scrub("Алексей"), str) + + +# --------------------------------------------------------------------------------------------------- +# pseudo_id() +# --------------------------------------------------------------------------------------------------- + +KEY = "test-key-0123456789abcdef0123456789abcdef" + + +def test_pseudo_id_format_and_determinism() -> None: + token = pseudo_id("112-233-445 95", key=KEY) + assert len(token) == 32 + assert all(c in "0123456789abcdef" for c in token) + assert pseudo_id("112-233-445 95", key=KEY) == token + + +@pytest.mark.parametrize("variant", ["11223344595", "112-233-445 95", "112 233 445 95", " 112-233-445-95 "]) +def test_pseudo_id_normalises_snils(variant: str) -> None: + assert pseudo_id(variant, key=KEY) == pseudo_id("11223344595", key=KEY) + + +def test_pseudo_id_depends_on_key_and_snils() -> None: + assert pseudo_id("11223344595", key=KEY) != pseudo_id("11223344595", key=KEY + "x") + assert pseudo_id("11223344595", key=KEY) != pseudo_id("11223344596", key=KEY) + + +def test_pseudo_id_default_key_from_settings() -> None: + from app.config import get_settings + + assert pseudo_id("11223344595") == pseudo_id("11223344595", key=get_settings().pseudo_key) + + +@pytest.mark.parametrize( + "bad", ["", "1234567890", "123456789012", "112-233-445 9X", "１１２２３３４４５９５"] +) +def test_pseudo_id_rejects_invalid_snils(bad: str) -> None: + with pytest.raises(ValueError): + pseudo_id(bad, key=KEY) -- 2.54.0