From e20da1f21cd174a3d454c527a7097ea9f1375b83 Mon Sep 17 00:00:00 2001 From: Lacky1234union Date: Sat, 31 Oct 2026 11:43:36 +0300 Subject: [PATCH] =?UTF-8?q?feat(web):=20voice=20input=20=E2=80=94=20record?= =?UTF-8?q?,=20check=20the=20recognised=20text,=20send?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Microphone button in the chat composer and in free-text steps. The browser records 16 kHz mono WAV (no server-side ffmpeg), /me/{token}/stt transcribes, the text lands in the field for the patient to check and edit — never sent automatically, so safety and scrub() apply on send. Hidden when the server has voice input off or the browser lacks a microphone API. nginx: 4 MB body limit. Refs #117 Co-Authored-By: Claude Opus 5.5 --- web/nginx.conf | 1 + web/src/api/client.ts | 18 +++- web/src/components/Icon.tsx | 6 ++ web/src/components/VoiceInput.tsx | 165 ++++++++++++++++++++++++++++++ web/src/content/ru.ts | 13 +++ web/src/screens/Flow.tsx | 2 + web/src/screens/Messages.tsx | 6 ++ web/src/theme/global.css | 100 ++++++++++++++++++ web/src/voice/recorder.ts | 121 ++++++++++++++++++++++ 9 files changed, 431 insertions(+), 1 deletion(-) create mode 100644 web/src/components/VoiceInput.tsx create mode 100644 web/src/voice/recorder.ts diff --git a/web/nginx.conf b/web/nginx.conf index 35d72de..b1f2e3a 100644 --- a/web/nginx.conf +++ b/web/nginx.conf @@ -14,5 +14,6 @@ server { proxy_http_version 1.1; proxy_buffering off; proxy_read_timeout 120s; + client_max_body_size 4m; # voice input: up to 60 s of 16 kHz WAV (~1.9 MB) } } diff --git a/web/src/api/client.ts b/web/src/api/client.ts index ea6b5ad..f68cdd7 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -46,7 +46,23 @@ export const api = { signInAnonymous: (focus?: Focus) => post("/auth/anonymous", { focus }), // Presentation entry: a fresh private copy of the demo patient, straight to the chat. signInDemoFresh: () => post("/auth/demo/fresh", {}), - patientCapabilities: () => request<{ primary_screening: boolean }>("/patient/capabilities"), + patientCapabilities: () => + request<{ primary_screening: boolean; voice_input?: boolean }>("/patient/capabilities"), + // Voice → text for the patient to check (#117). Raw WAV body; the server only transcribes. + transcribe: async (token: string, wav: Blob): Promise => { + let res: Response; + try { + res = await fetch(`${BASE}/me/${encodeURIComponent(token)}/stt`, { + method: "POST", + headers: { "Content-Type": "audio/wav" }, + body: wav, + }); + } catch { + throw new ApiError(0, "network"); + } + if (!res.ok) throw new ApiError(res.status, res.statusText); + return ((await res.json()) as { text: string }).text; + }, connectDoctor: (token: string) => post<{ has_doctor: boolean }>(`/me/${encodeURIComponent(token)}/doctor/connect`), demoPersonas: () => request("/patient/demo-personas"), crisisResources: () => request("/crisis/resources"), diff --git a/web/src/components/Icon.tsx b/web/src/components/Icon.tsx index 05510a8..83786c1 100644 --- a/web/src/components/Icon.tsx +++ b/web/src/components/Icon.tsx @@ -32,6 +32,12 @@ const paths = { arrowLeft: , close: , check: , + mic: ( + <> + + + + ), x: , info: ( <> diff --git a/web/src/components/VoiceInput.tsx b/web/src/components/VoiceInput.tsx new file mode 100644 index 0000000..6816965 --- /dev/null +++ b/web/src/components/VoiceInput.tsx @@ -0,0 +1,165 @@ +import { useEffect, useRef, useState } from "react"; + +import { ApiError, api } from "../api/client"; +import { ru } from "../content/ru"; +import { useSession } from "../session/SessionContext"; +import { MAX_SECONDS, Recorder, voiceSupported } from "../voice/recorder"; +import { Icon } from "./Icon"; +import { useToast } from "./Toast"; + +/* + * Voice input next to a free-text field (#117, slide 11: «Записать → Проверить»). The recognised text is put into + * the field for the patient to check and edit — never sent automatically: safety and scrub() run when they send it. + */ + +let capability: Promise | null = null; +function voiceEnabled(): Promise { + capability ??= api + .patientCapabilities() + .then((c) => Boolean(c.voice_input) && voiceSupported()) + .catch(() => false); + return capability; +} + +const BARS = 14; +type State = "idle" | "recording" | "transcribing"; + +export function VoiceInput({ + disabled, + onText, + compact, +}: { + disabled?: boolean; + onText: (text: string) => void; + /** Chat composer: a round icon button; the recording panel floats above the composer. */ + compact?: boolean; +}) { + const { session } = useSession(); + const toast = useToast(); + const [enabled, setEnabled] = useState(false); + const [state, setState] = useState("idle"); + const [seconds, setSeconds] = useState(0); + const [levels, setLevels] = useState(() => Array(BARS).fill(0)); + const rec = useRef(null); + const timer = useRef(); + + useEffect(() => { + let alive = true; + void voiceEnabled().then((on) => alive && setEnabled(on)); + return () => { + alive = false; + window.clearInterval(timer.current); + rec.current?.cancel(); + }; + }, []); + + if (!enabled || !session) return null; + + const start = async () => { + const r = new Recorder(); + try { + await r.start((level) => setLevels((prev) => [...prev.slice(1), level])); + } catch { + r.cancel(); + toast(ru.voice.noMic); + return; + } + rec.current = r; + setSeconds(0); + setState("recording"); + const began = Date.now(); + timer.current = window.setInterval(() => { + const s = Math.floor((Date.now() - began) / 1000); + setSeconds(s); + if (s >= MAX_SECONDS) void stop(); + }, 250); + }; + + const reset = () => { + window.clearInterval(timer.current); + rec.current = null; + setLevels(Array(BARS).fill(0)); + }; + + const stop = async () => { + const wav = rec.current?.stop() ?? null; + reset(); + if (!wav) { + setState("idle"); + return; + } + setState("transcribing"); + try { + const text = await api.transcribe(session.token, wav); + onText(text); + toast(ru.voice.check); + } catch (e) { + const status = e instanceof ApiError ? e.status : 0; + toast(status === 422 ? ru.voice.noSpeech : status === 413 ? ru.voice.tooLong : ru.voice.failed); + } finally { + setState("idle"); + } + }; + + const cancel = () => { + rec.current?.cancel(); + reset(); + setState("idle"); + }; + + if (state === "idle" && compact) { + return ( + + ); + } + if (state === "idle") { + return ( + + ); + } + + const mm = String(Math.floor(seconds / 60)).padStart(2, "0"); + const ss = String(seconds % 60).padStart(2, "0"); + return ( +
+ {state === "recording" ? ( + <> +

{ru.voice.recording}

+ +
+ + {mm}:{ss} + + + +
+

{ru.voice.privacy}

+ + ) : ( +

+ {ru.voice.transcribing} +

+ )} +
+ ); +} diff --git a/web/src/content/ru.ts b/web/src/content/ru.ts index b6f2aff..73ece5c 100644 --- a/web/src/content/ru.ts +++ b/web/src/content/ru.ts @@ -251,6 +251,19 @@ export const ru = { drinksLead: "Посчитайте, сколько порций выходит в обычный день, когда вы выпиваете.", noticeTitle: "Опрос приостановлен", }, + voice: { + start: "Сказать голосом", + stop: "Остановить запись", + cancel: "Отменить", + recording: "Говорите — запись идёт", + transcribing: "Распознаём речь…", + check: "Проверьте распознанный текст — его можно исправить перед отправкой.", + privacy: "Запись не сохраняется: её видит только система распознавания, текст — специалисты вашей программы.", + noMic: "Нет доступа к микрофону. Разрешите его в настройках браузера или напишите текстом.", + noSpeech: "Не удалось разобрать речь. Попробуйте ещё раз или напишите текстом.", + failed: "Распознавание сейчас недоступно. Напишите, пожалуйста, текстом.", + tooLong: "Запись слишком длинная — до минуты за раз.", + }, messages: { typing: "Помощник печатает", title: "Сообщения", diff --git a/web/src/screens/Flow.tsx b/web/src/screens/Flow.tsx index 8613b56..099266d 100644 --- a/web/src/screens/Flow.tsx +++ b/web/src/screens/Flow.tsx @@ -5,6 +5,7 @@ import type { Catalog, DialogButton } from "../api/types"; import { AppShell, Notice, TopBar } from "../components/AppShell"; import { HotlineCard, telHref, useHelp } from "../components/HelpSheet"; import { useToast } from "../components/Toast"; +import { VoiceInput } from "../components/VoiceInput"; import { Icon, type IconName } from "../components/Icon"; import { useCatalog } from "../content/reference"; import { ru } from "../content/ru"; @@ -259,6 +260,7 @@ function FreeText({ /> {text.length}/500 + setText((prev) => (prev.trim() ? `${prev.trim()} ${t}` : t).slice(0, 500))} />
{skip ? ( diff --git a/web/src/theme/global.css b/web/src/theme/global.css index 30fce66..0bb3978 100644 --- a/web/src/theme/global.css +++ b/web/src/theme/global.css @@ -1566,6 +1566,106 @@ button.row-card:hover:not(:disabled) { color: var(--c-text-3); } +/* ---------------------------------------------------------------- voice input (#117, slide 11) */ + +.voice-btn { + display: inline-flex; + align-items: center; + gap: var(--s-2); + justify-self: start; + min-height: 44px; + padding: 0 var(--s-4); + border: 1.5px solid var(--c-border-strong); + border-radius: 999px; + background: var(--c-surface); + color: var(--c-teal-strong); + font-weight: 600; + font-size: var(--fs-14); +} + +.voice-btn:hover:not(:disabled) { + background: var(--c-teal-soft); +} + +.mic { + flex: none; + width: 44px; + height: 44px; + display: grid; + place-items: center; + border: 1.5px solid var(--c-border); + border-radius: 50%; + background: var(--c-surface); + color: var(--c-teal-strong); +} + +.mic:hover:not(:disabled) { + background: var(--c-teal-soft); +} + +.voice { + display: grid; + gap: var(--s-3); + padding: var(--s-4); + border: 1px solid var(--c-border); + border-radius: var(--r-tile); + background: #fbf6ee; /* warm, as on the slide */ +} + +.voice--float { + position: absolute; + left: 0; + right: 0; + bottom: calc(100% + var(--s-2)); + box-shadow: 0 8px 24px rgb(33 59 56 / 12%); +} + +.voice__hint { + margin: 0; + font-weight: 600; + color: var(--c-text); +} + +.voice__wave { + display: flex; + align-items: center; + gap: 5px; + height: 44px; +} + +.voice__wave span { + flex: 1; + max-width: 6px; + border-radius: 3px; + background: #b5452f; + transition: height 90ms linear; +} + +.voice__row { + display: grid; + grid-template-columns: auto 1fr 1.4fr; + align-items: center; + gap: var(--s-2); +} + +.voice__time { + font-variant-numeric: tabular-nums; + font-weight: 600; + color: var(--c-text-2); +} + +.voice__row .btn { + min-height: 44px; + padding: 0 var(--s-3); + font-size: var(--fs-14); +} + +.voice__privacy { + margin: 0; + font-size: var(--fs-12); + color: var(--c-text-3); +} + /* ---------------------------------------------------------------- emergency */ .sosblock { diff --git a/web/src/voice/recorder.ts b/web/src/voice/recorder.ts new file mode 100644 index 0000000..74de2fe --- /dev/null +++ b/web/src/voice/recorder.ts @@ -0,0 +1,121 @@ +/* + * Voice recording for the patient's messages (#117, slide 11): microphone → 16 kHz mono 16-bit WAV in the browser, + * so the server needs no ffmpeg and Chrome, Android and iOS Safari produce the same format (ADR-014). + * The audio lives only in memory until it is sent for transcription; nothing is stored on the device. + */ + +export const TARGET_RATE = 16_000; +export const MAX_SECONDS = 60; + +type WindowWithWebkit = Window & { webkitAudioContext?: typeof AudioContext }; + +export function voiceSupported(): boolean { + const w = window as WindowWithWebkit; + return ( + typeof navigator.mediaDevices?.getUserMedia === "function" && Boolean(window.AudioContext || w.webkitAudioContext) + ); +} + +export class Recorder { + private ctx: AudioContext | null = null; + private stream: MediaStream | null = null; + private node: ScriptProcessorNode | null = null; + private chunks: Float32Array[] = []; + private rate = 48_000; + + /** `onLevel` gets the loudness 0..1 of each audio block (for the waveform). */ + async start(onLevel: (level: number) => void): Promise { + this.stream = await navigator.mediaDevices.getUserMedia({ + audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true }, + }); + const Ctx = window.AudioContext || (window as WindowWithWebkit).webkitAudioContext!; + this.ctx = new Ctx(); + await this.ctx.resume(); + this.rate = this.ctx.sampleRate; + const source = this.ctx.createMediaStreamSource(this.stream); + // ScriptProcessorNode is deprecated but works everywhere (incl. older iOS Safari) without a worklet file. + this.node = this.ctx.createScriptProcessor(4096, 1, 1); + this.node.onaudioprocess = (e) => { + const data = new Float32Array(e.inputBuffer.getChannelData(0)); + this.chunks.push(data); + let sum = 0; + for (let i = 0; i < data.length; i++) sum += data[i] * data[i]; + onLevel(Math.min(1, Math.sqrt(sum / data.length) * 4)); + }; + source.connect(this.node); + this.node.connect(this.ctx.destination); + } + + /** Stops the microphone and returns the WAV, or null if nothing was recorded. */ + stop(): Blob | null { + const chunks = this.chunks; + const rate = this.rate; + this.release(); + if (!chunks.length) return null; + return encodeWav(downsample(concat(chunks), rate, TARGET_RATE), TARGET_RATE); + } + + cancel(): void { + this.release(); + } + + private release(): void { + this.node?.disconnect(); + if (this.node) this.node.onaudioprocess = null; + this.stream?.getTracks().forEach((t) => t.stop()); + void this.ctx?.close(); + this.node = null; + this.stream = null; + this.ctx = null; + this.chunks = []; + } +} + +function concat(chunks: Float32Array[]): Float32Array { + const out = new Float32Array(chunks.reduce((n, c) => n + c.length, 0)); + let pos = 0; + for (const c of chunks) { + out.set(c, pos); + pos += c.length; + } + return out; +} + +/** Averaging downsampler: good enough for speech, avoids aliasing better than dropping samples. */ +function downsample(input: Float32Array, from: number, to: number): Float32Array { + if (from === to) return input; + const ratio = from / to; + const out = new Float32Array(Math.floor(input.length / ratio)); + for (let i = 0; i < out.length; i++) { + const start = Math.floor(i * ratio); + const end = Math.min(input.length, Math.floor((i + 1) * ratio)); + let sum = 0; + for (let j = start; j < end; j++) sum += input[j]; + out[i] = sum / Math.max(1, end - start); + } + return out; +} + +function encodeWav(samples: Float32Array, rate: number): Blob { + const buf = new ArrayBuffer(44 + samples.length * 2); + const v = new DataView(buf); + const str = (o: number, s: string) => [...s].forEach((ch, i) => v.setUint8(o + i, ch.charCodeAt(0))); + str(0, "RIFF"); + v.setUint32(4, 36 + samples.length * 2, true); + str(8, "WAVE"); + str(12, "fmt "); + v.setUint32(16, 16, true); + v.setUint16(20, 1, true); // PCM + v.setUint16(22, 1, true); // mono + v.setUint32(24, rate, true); + v.setUint32(28, rate * 2, true); // byte rate + v.setUint16(32, 2, true); // block align + v.setUint16(34, 16, true); + str(36, "data"); + v.setUint32(40, samples.length * 2, true); + for (let i = 0; i < samples.length; i++) { + const s = Math.max(-1, Math.min(1, samples[i])); + v.setInt16(44 + i * 2, s < 0 ? s * 0x8000 : s * 0x7fff, true); + } + return new Blob([buf], { type: "audio/wav" }); +} -- 2.54.0