From b8e3c5dae1020eb943de9617f013081d9d7f6afa Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Wed, 14 Oct 2026 01:43:48 -0700 Subject: [PATCH 01/17] feat(models): a Speech recognition section in Settings, and the set-up popover MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 3 of the local speech recognition proposal, frontend half. Users can now download and choose a speech model, and set its spoken language; nothing reads the choice yet, which selection allows. - `speech_to_text` joins the model types. The local catalog slice (`features/models/local/transcription/`) reads rows whose `selectable_for` names it, by their `transcription` block: size, memory while transcribing, whether the model detects the language, and its listed languages, counted and opened to their names in the interface's language. - Settings › Speech recognition sits beside Audio on `ModelSlotSettings`, with a new `localOnly` option: no server can fill this slot, so neither the list nor Add model offers one. A download installs with `select: true`, because one speech model serves recordings and voice input. The language select reads and writes /llm/selection/speech_to_text/language and offers "Detect automatically" only for a model that detects. - `features/speech-setup/` is the set-up popover the proposal shares between a waiting recording's row and the mic: one card per curated model, Download and set up through the existing install job and its egress consent, its progress, then Start talking or Done, and More choices for Settings. The consent prompt nests inside the popover (a new Base UI `Popover` wrapper hosts app dialogs), so answering it does not dismiss the popover. - Every new string is in all ten catalogs; four `{slot, select}` messages gain a `speech` branch. The glossary gains speech recognition, voice input and transcript. The catalog doc describes both screens. --- .agents/skills/translate/glossary.md | 3 + docs/architecture/local-models/catalog.md | 28 +- .../frontend/src/components/ui/icons.tsx | 2 + .../frontend/src/components/ui/popover.tsx | 101 ++++++ .../src/features/models/local/chat/api.ts | 12 + .../models/local/transcription/api.ts | 70 ++++ .../transcription/describe-speech-model.ts | 64 ++++ .../transcription/download-speech-models.tsx | 208 ++++++++++++ .../local/transcription/fake-speech-rows.ts | 95 ++++++ .../local/transcription/language-names.ts | 28 ++ .../local/transcription/language/api.ts | 33 ++ .../language/speech-language-select.tsx | 78 +++++ .../language/use-speech-language.ts | 24 ++ .../local/transcription/listed-languages.tsx | 54 ++++ .../use-delete-local-speech-model.ts | 12 + .../transcription/use-local-speech-catalog.ts | 11 + .../src/features/models/model-type.ts | 1 + .../models/remote/models/server-models.tsx | 5 + .../models/your-models/in-use-summary.tsx | 4 +- .../models/your-models/use-speech-models.ts | 44 +++ .../settings/models/model-slot-settings.tsx | 92 ++++-- .../models/speech-models-settings.test.tsx | 304 ++++++++++++++++++ .../models/speech-models-settings.tsx | 45 +++ .../settings/settings-dialog.test.tsx | 10 + .../src/features/settings/settings-dialog.tsx | 12 + .../speech-setup/speech-model-card.tsx | 92 ++++++ .../speech-setup-popover.test.tsx | 298 +++++++++++++++++ .../speech-setup/speech-setup-popover.tsx | 178 ++++++++++ .../features/speech-setup/use-speech-setup.ts | 62 ++++ surfsense_local/frontend/translations/de.json | 38 ++- surfsense_local/frontend/translations/en.json | 38 ++- surfsense_local/frontend/translations/es.json | 38 ++- surfsense_local/frontend/translations/fr.json | 38 ++- surfsense_local/frontend/translations/hi.json | 38 ++- surfsense_local/frontend/translations/ja.json | 38 ++- surfsense_local/frontend/translations/ko.json | 38 ++- .../frontend/translations/pt-BR.json | 38 ++- surfsense_local/frontend/translations/ru.json | 38 ++- .../frontend/translations/zh-CN.json | 38 ++- 39 files changed, 2277 insertions(+), 73 deletions(-) create mode 100644 surfsense_local/frontend/src/components/ui/popover.tsx create mode 100644 surfsense_local/frontend/src/features/models/local/transcription/api.ts create mode 100644 surfsense_local/frontend/src/features/models/local/transcription/describe-speech-model.ts create mode 100644 surfsense_local/frontend/src/features/models/local/transcription/download-speech-models.tsx create mode 100644 surfsense_local/frontend/src/features/models/local/transcription/fake-speech-rows.ts create mode 100644 surfsense_local/frontend/src/features/models/local/transcription/language-names.ts create mode 100644 surfsense_local/frontend/src/features/models/local/transcription/language/api.ts create mode 100644 surfsense_local/frontend/src/features/models/local/transcription/language/speech-language-select.tsx create mode 100644 surfsense_local/frontend/src/features/models/local/transcription/language/use-speech-language.ts create mode 100644 surfsense_local/frontend/src/features/models/local/transcription/listed-languages.tsx create mode 100644 surfsense_local/frontend/src/features/models/local/transcription/use-delete-local-speech-model.ts create mode 100644 surfsense_local/frontend/src/features/models/local/transcription/use-local-speech-catalog.ts create mode 100644 surfsense_local/frontend/src/features/models/your-models/use-speech-models.ts create mode 100644 surfsense_local/frontend/src/features/settings/models/speech-models-settings.test.tsx create mode 100644 surfsense_local/frontend/src/features/settings/models/speech-models-settings.tsx create mode 100644 surfsense_local/frontend/src/features/speech-setup/speech-model-card.tsx create mode 100644 surfsense_local/frontend/src/features/speech-setup/speech-setup-popover.test.tsx create mode 100644 surfsense_local/frontend/src/features/speech-setup/speech-setup-popover.tsx create mode 100644 surfsense_local/frontend/src/features/speech-setup/use-speech-setup.ts diff --git a/.agents/skills/translate/glossary.md b/.agents/skills/translate/glossary.md index 123e7cf879..e2fffd1c3b 100644 --- a/.agents/skills/translate/glossary.md +++ b/.agents/skills/translate/glossary.md @@ -41,6 +41,9 @@ SurfSense, Studio, Markdown, llama.cpp, GGUF, Hugging Face, API, MCP, and every | study guide | 学習ガイド | 학습 가이드 | 学习指南 | स्टडी गाइड | Lernzettel † | guía de estudio | Guide de révision | Guia de estudos | конспект | | podcast | ポッドキャスト | 팟캐스트 | 播客 | पॉडकास्ट | Podcast | podcast | Podcast | Podcast | подкаст | | speech (a model reading text aloud) | 音声合成 | 음성 | 语音 | वॉइस | Sprache | voz | voix | voz | речь | +| speech recognition (turning recordings and the voice into text, on this computer) | 音声認識 | 음성 인식 | 语音识别 | स्पीच रिकग्निशन | Spracherkennung | reconocimiento de voz | reconnaissance vocale | reconhecimento de voz | распознавание речи | +| voice input (dictating into the chat composer) | 音声入力 | 음성 입력 | 语音输入 | वॉइस इनपुट | Spracheingabe | entrada de voz | saisie vocale | entrada de voz | голосовой ввод | +| transcript (a recording’s text) | 文字起こし | 대본 | 转录文本 | ट्रांसक्रिप्ट | Transkript | transcripción | transcription | transcrição | расшифровка | | sidebar (the workspace’s left column) | サイドバー | 사이드바 | 侧边栏 | साइडबार | Seitenleiste | barra lateral | barre latérale | barra lateral | боковая панель | | workspace | ワークスペース | 워크스페이스 | 工作区 | वर्कस्पेस | Workspace | espacio de trabajo | espace de travail | Workspace | рабочая область | | artifact (a Studio output) | 生成物 | 생성물 | 产物 | आर्टिफ़ैक्ट | Artefakt | artefacto | artefact | artefato | артефакт | diff --git a/docs/architecture/local-models/catalog.md b/docs/architecture/local-models/catalog.md index 883c34d409..d40da10957 100644 --- a/docs/architecture/local-models/catalog.md +++ b/docs/architecture/local-models/catalog.md @@ -631,7 +631,7 @@ on search and repo reads, and the job's generic error during an install. ## On the screen -Settings has one section per model type, **Chat**, **Image**, **Image editing**, **Video** and **Audio** in the nav, each headed **Text generation models**, **Image generation models**, **Image editing models**, **Video generation models** or **Audio generation models** on its own page. +Settings has one section per model type, **Chat**, **Image**, **Image editing**, **Video**, **Audio** and **Speech recognition** in the nav, each headed **Text generation models**, **Image generation models**, **Image editing models**, **Video generation models**, **Audio generation models** or **Speech recognition models** on its own page. Each names the model in use at the top, then groups the slot's models by source: **This computer**, every build on disk, curated or not, with Use (when its type can fill the slot) or In use and Delete after confirmation; then one group per @@ -714,6 +714,30 @@ Rules the screen holds: audiocpp_server on its next poll. The voice the app ships is the exception: it is part of the install, read in place from the models pack, so it has no Delete, and the API refuses one. +- The speech recognition section is local only: no server can fill + `speech_to_text`, so neither its list nor its Add model page offers one + (`ModelSlotSettings`' `localOnly`). It takes every row whose + `selectable_for` names the slot, described by the row's `transcription`, so + recognisers never reach the Audio page, which drops rows without `voicing`. + A row reads its build, its size, the memory it takes while transcribing, and + "Detects the language" when it does ("Q8_0 · 931 MB · 1.8 GB while + transcribing · Detects the language"), then its languages, counted, which + open to their names in the interface's language, alphabetically + (`Intl.DisplayNames`). A download installs with `select: true`: one speech + model serves recordings and voice input, so downloading one is choosing it. + Under the model in use, **Spoken language** reads and writes + `/llm/selection/speech_to_text/language`: "Detect automatically", only when + the model detects the language, then each language it lists. +- The speech set-up popover ([`speech-setup/`](../../../surfsense_local/frontend/src/features/speech-setup/)) + is the same download where speech is needed, for the mic and a recording + waiting for a model to open: one card per curated recogniser with its size, + its memory while transcribing and "Works well in:" its listed languages. + **Download and set up** starts the same install job with `select: true`, so + egress consent is the usual refused-request prompt, nested in the popover; + the card then shows the job's progress, and once the catalog marks the model + in use the popover reads "Speech recognition is ready", with **Start + talking** where its opener can listen. **More choices** opens this section. + `useSpeechSetup()` holds that state, so an opener can show the download too. ## How it is tested @@ -726,7 +750,7 @@ the service's installs, install jobs (`test_install_jobs.py`), the audio.cpp sli engine's refresh assembly; the routes, audio's `server.json` included, are covered in [`surfsense_local/backend/tests/integration/llm/`](../../../surfsense_local/backend/tests/integration/llm/), the feed over a real socket in `test_install_feed.py`, -and the screen in `download-chat-models.test.tsx`, `install-view.test.tsx`, `install-text.test.ts` and the settings sections' `chat-models-settings.test.tsx`, `image-models-settings.test.tsx` and `audio-models-settings.test.tsx`. +and the screen in `download-chat-models.test.tsx`, `install-view.test.tsx`, `install-text.test.ts`, the settings sections' `chat-models-settings.test.tsx`, `image-models-settings.test.tsx`, `audio-models-settings.test.tsx` and `speech-models-settings.test.tsx`, and the speech set-up in `speech-setup-popover.test.tsx`. ## Known gaps diff --git a/surfsense_local/frontend/src/components/ui/icons.tsx b/surfsense_local/frontend/src/components/ui/icons.tsx index bc5180ac32..a4a11fa927 100644 --- a/surfsense_local/frontend/src/components/ui/icons.tsx +++ b/surfsense_local/frontend/src/components/ui/icons.tsx @@ -53,6 +53,7 @@ import { LinkSquare02Icon as LinkSquare02IconData, LightbulbIcon as LightbulbIconData, Loading03Icon, + Mic01Icon as Mic01IconData, Moon02Icon as Moon02IconData, NetworkIcon as NetworkIconData, NoteAddIcon as NoteAddIconData, @@ -177,6 +178,7 @@ export const LicenseIcon = createIcon(LicenseIconData) export const ExternalLinkIcon = createIcon(LinkSquare02IconData) export const LightbulbIcon = createIcon(LightbulbIconData) export const Loader2Icon = createIcon(Loading03Icon) +export const Mic01Icon = createIcon(Mic01IconData) export const MoonIcon = createIcon(Moon02IconData) export const NetworkIcon = createIcon(NetworkIconData, "rotate-90") export const NoteAddIcon = createIcon(NoteAddIconData) diff --git a/surfsense_local/frontend/src/components/ui/popover.tsx b/surfsense_local/frontend/src/components/ui/popover.tsx new file mode 100644 index 0000000000..d6e768be8c --- /dev/null +++ b/surfsense_local/frontend/src/components/ui/popover.tsx @@ -0,0 +1,101 @@ +"use client" + +import type * as React from "react" +import { Popover as PopoverPrimitive } from "@base-ui/react/popover" + +import { AppDialogHost } from "@/components/ui/app-dialog-slot" +import { cn } from "@/lib/utils" + +function Popover({ ...props }: PopoverPrimitive.Root.Props) { + return +} + +function PopoverTrigger({ ...props }: PopoverPrimitive.Trigger.Props) { + return +} + +function PopoverContent({ + className, + children, + align = "center", + alignOffset = 0, + side = "bottom", + sideOffset = 4, + ...props +}: PopoverPrimitive.Popup.Props & + Pick< + PopoverPrimitive.Positioner.Props, + "align" | "alignOffset" | "side" | "sideOffset" + >) { + return ( + + + + {/* An app dialog asked from inside, such as egress consent, nests + here, so answering it does not dismiss the popover. */} + {children} + + + + ) +} + +function PopoverHeader({ className, ...props }: React.ComponentProps<"div">) { + return ( +
+ ) +} + +function PopoverTitle({ className, ...props }: PopoverPrimitive.Title.Props) { + return ( + + ) +} + +function PopoverDescription({ + className, + ...props +}: PopoverPrimitive.Description.Props) { + return ( + + ) +} + +function PopoverClose({ ...props }: PopoverPrimitive.Close.Props) { + return +} + +export { + Popover, + PopoverClose, + PopoverContent, + PopoverDescription, + PopoverHeader, + PopoverTitle, + PopoverTrigger, +} diff --git a/surfsense_local/frontend/src/features/models/local/chat/api.ts b/surfsense_local/frontend/src/features/models/local/chat/api.ts index 4b6b5aaa72..4abcdba300 100644 --- a/surfsense_local/frontend/src/features/models/local/chat/api.ts +++ b/surfsense_local/frontend/src/features/models/local/chat/api.ts @@ -95,6 +95,16 @@ export type Voicing = { languages: string[] } +export type Transcription = { + /** Measured on the first request, the load included. */ + peak_mb: number + /** The longest stretch of audio one request carries. */ + window_s: number + detects_language: boolean + /** ISO 639-1 codes of the languages the model is listed for. */ + languages: string[] +} + export type LocalRow = { id: string source: "local" @@ -124,6 +134,8 @@ export type LocalRow = { description?: string | null /** An audio model's memory while voicing, voices and languages. */ voicing?: Voicing | null + /** A speech recognition model's memory, languages and language detection. */ + transcription?: Transcription | null /** * The build the row shows and its Download fetches, and why the server chose * it. Absent for a searched repo, which lists every build and leads with none. diff --git a/surfsense_local/frontend/src/features/models/local/transcription/api.ts b/surfsense_local/frontend/src/features/models/local/transcription/api.ts new file mode 100644 index 0000000000..9c096c0974 --- /dev/null +++ b/surfsense_local/frontend/src/features/models/local/transcription/api.ts @@ -0,0 +1,70 @@ +import { + deleteLocalModel, + getModelCatalog, + type LocalBuild, + type LocalRow, +} from "../chat/api" + +/** One speech recognition model, as audio.cpp's catalog row offers it. */ +export type LocalSpeechModel = { + /** The row's id: stable, where an install token may be reissued. */ + id: string + label: string + curated: boolean + quantization: string + size_bytes: number + peak_mb: number + detects_language: boolean + /** ISO 639-1 codes of the languages the model is listed for. */ + languages: string[] + /** Opaque install token for the build a download fetches. */ + catalog_id: string + /** What selection and deletion name; null until it is on disk. */ + installed_as: string | null + /** In use for speech recognition, whatever else names the same build. */ + selected: boolean + /** What the row's action button reads, marked for this slot alone. */ + build: LocalBuild +} + +function toSpeechModel(row: LocalRow): LocalSpeechModel | null { + // The server picks the build a row leads with: in use, installed, default. + const lead = + row.builds.find((b) => b.quantization === row.lead?.quantization) ?? + row.builds[0] + if (!lead || !row.transcription) return null + const selected = lead.selected_for + ? lead.selected_for.includes("speech_to_text") + : lead.selected + return { + id: row.id, + label: row.name, + curated: row.origin === "curated", + quantization: lead.quantization, + size_bytes: lead.footprint_bytes, + peak_mb: row.transcription.peak_mb, + detects_language: row.transcription.detects_language, + languages: row.transcription.languages, + catalog_id: lead.catalog_id, + installed_as: lead.installed_as, + selected, + build: { ...lead, selected }, + } +} + +/** Empty where no audio.cpp shipped, or it cannot transcribe. */ +export async function getLocalSpeechModels( + signal?: AbortSignal +): Promise { + const catalog = await getModelCatalog(signal) + return (catalog.rows ?? []) + .filter((row) => row.selectable_for.includes("speech_to_text")) + .map(toSpeechModel) + .filter((model): model is LocalSpeechModel => model !== null) +} + +export async function deleteLocalSpeechModel( + installedAs: string +): Promise { + await deleteLocalModel(installedAs) +} diff --git a/surfsense_local/frontend/src/features/models/local/transcription/describe-speech-model.ts b/surfsense_local/frontend/src/features/models/local/transcription/describe-speech-model.ts new file mode 100644 index 0000000000..864ecd73a9 --- /dev/null +++ b/surfsense_local/frontend/src/features/models/local/transcription/describe-speech-model.ts @@ -0,0 +1,64 @@ +import { intl } from "@/i18n/intl" + +import type { LocalSpeechModel } from "./api" +import { languageName } from "./language-names" + +const bytes = (value: number) => + value >= 1e9 + ? intl.formatNumber(value / 1e9, { + style: "unit", + unit: "gigabyte", + maximumFractionDigits: 1, + }) + : intl.formatNumber(value / 1e6, { + style: "unit", + unit: "megabyte", + maximumFractionDigits: 0, + }) + +/** What its download takes on disk. */ +export function speechModelSize(model: LocalSpeechModel): string { + return bytes(model.size_bytes) +} + +/** The measured peak, the load included, which admission also reads. */ +export function speechModelMemory(model: LocalSpeechModel): string { + return model.peak_mb >= 1000 + ? intl.formatMessage( + { + id: "models_speech_model_peak_memory_gigabytes_label", + defaultMessage: + "{size, number, ::unit/gigabyte .#} while transcribing", + }, + { size: model.peak_mb / 1000 } + ) + : intl.formatMessage( + { + id: "models_speech_model_peak_memory_megabytes_label", + defaultMessage: + "{size, number, ::unit/megabyte .} while transcribing", + }, + { size: model.peak_mb } + ) +} + +/** + * Build, download, memory while transcribing, language detection, and the one + * language when there is only one; more are counted apart, where they expand. + */ +export function describeSpeechModel(model: LocalSpeechModel): string[] { + return [ + model.quantization, + speechModelSize(model), + speechModelMemory(model), + ...(model.detects_language + ? [ + intl.formatMessage({ + id: "models_speech_model_detects_language_label", + defaultMessage: "Detects the language", + }), + ] + : []), + ...(model.languages.length === 1 ? [languageName(model.languages[0])] : []), + ] +} diff --git a/surfsense_local/frontend/src/features/models/local/transcription/download-speech-models.tsx b/surfsense_local/frontend/src/features/models/local/transcription/download-speech-models.tsx new file mode 100644 index 0000000000..2b5d51f5fe --- /dev/null +++ b/surfsense_local/frontend/src/features/models/local/transcription/download-speech-models.tsx @@ -0,0 +1,208 @@ +import { Fragment, useState } from "react" + +import { CircleAlertIcon, DotIcon, Trash2Icon } from "@/components/ui/icons" +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert" +import { Button } from "@/components/ui/button" +import { intl } from "@/i18n/intl" + +import { useSelect } from "../../selection/use-selection" +import { DeleteModelDialog } from "../../your-models/delete-model-dialog" +import type { YourModelRow } from "../../your-models/your-model-row" +import { useInstall } from "../installs/use-install" +import { jobFor } from "../installs/job-state" +import { BuildAction } from "../chat/build-action" +import { InstallProgress } from "../chat/install-progress" +import type { LocalSpeechModel } from "./api" +import { describeSpeechModel } from "./describe-speech-model" +import { ListedLanguages } from "./listed-languages" +import { useDeleteLocalSpeechModel } from "./use-delete-local-speech-model" +import { useLocalSpeechCatalog } from "./use-local-speech-catalog" + +function messageFrom(error: unknown) { + return error instanceof Error + ? error.message + : intl.formatMessage({ + id: "models_download_speech_request_error", + defaultMessage: "The request failed", + }) +} + +/** The row `DeleteModelDialog` needs; it only reads `name` and `selected`. */ +function deletableRow(model: LocalSpeechModel): YourModelRow | null { + if (!model.installed_as) return null + return { + key: model.installed_as, + name: model.label, + selected: model.selected, + badges: [], + note: null, + target: null, + removeId: model.installed_as, + } +} + +/** + * Speech recognition models audio.cpp can run on this computer. A download + * becomes the model in use: one model serves recordings and voice input. + */ +export function DownloadSpeechModels() { + const catalog = useLocalSpeechCatalog() + const { installs, install, cancel } = useInstall({ + select: true, + modelType: "speech_to_text", + }) + const select = useSelect("speech_to_text") + const remove = useDeleteLocalSpeechModel() + const [deleting, setDeleting] = useState(null) + const [deleteOpen, setDeleteOpen] = useState(false) + const [deleteError, setDeleteError] = useState(null) + + if (catalog.isPending) return null + + if (catalog.isError) { + return ( + + + + {intl.formatMessage({ + id: "models_download_speech_load_error", + defaultMessage: "Could not load speech recognition models", + })} + + {messageFrom(catalog.error)} + + ) + } + + const models = catalog.data + if (models.length === 0) { + return ( + + + + {intl.formatMessage({ + id: "models_download_speech_unsupported_title", + defaultMessage: + "Speech recognition models cannot run on this computer", + })} + + + ) + } + + // A download does not block the rest: another one queues behind it. + const busy = select.isPending || remove.isPending + + const act = (model: LocalSpeechModel) => { + if (busy) return + if (model.installed_as) { + void select + .mutateAsync({ + target: { + provider: "audiocpp", + connection_id: null, + name: model.installed_as, + }, + }) + .catch(() => undefined) + } else { + void install(model.catalog_id) + } + } + + const confirmDelete = () => { + if (!deleting?.removeId) return + setDeleteError(null) + remove + .mutateAsync(deleting.removeId) + .then(() => setDeleteOpen(false)) + .catch((cause: unknown) => setDeleteError(messageFrom(cause))) + } + + return ( +
+
    + {models.map((model) => { + const job = jobFor(installs, model.catalog_id) + return ( +
  • +
    +
    +

    {model.label}

    +

    + {describeSpeechModel(model).map((fact, index) => ( + + {index > 0 ? ( + + ))} +

    +
    +
    + act(model)} + /> + {model.installed_as ? ( + + ) : null} +
    +
    + + {job ? ( + cancel(job.id)} + /> + ) : null} +
    • + ) + })} +
+ + {select.isError ? ( +

{messageFrom(select.error)}

+ ) : null} + + { + if (!open) setDeleting(null) + }} + /> +
+ ) +} diff --git a/surfsense_local/frontend/src/features/models/local/transcription/fake-speech-rows.ts b/surfsense_local/frontend/src/features/models/local/transcription/fake-speech-rows.ts new file mode 100644 index 0000000000..264170fe48 --- /dev/null +++ b/surfsense_local/frontend/src/features/models/local/transcription/fake-speech-rows.ts @@ -0,0 +1,95 @@ +/** The 26 languages Nemotron's card measures at 30% error or better. */ +export const NEMOTRON_LANGUAGES = [ + "it", + "es", + "en", + "pt", + "hi", + "ko", + "de", + "fr", + "ru", + "ja", + "nl", + "vi", + "uk", + "tr", + "ar", + "pl", + "zh", + "nb", + "fi", + "bg", + "cs", + "sk", + "sv", + "hr", + "ro", + "da", +] + +/** Stands in for audio.cpp's catalog row for a recogniser, as the API sends it. */ +export function fakeSpeechRow( + overrides: Record = {}, + build: Record = {} +) { + return { + id: "nemotron-3.5-asr-streaming-0.6b", + source: "local", + origin: "curated", + name: "Nemotron 3.5 ASR", + family: "Nemotron", + types: ["speech_to_text"], + known: true, + approximate: false, + selectable_for: ["speech_to_text"], + support: { + context: null, + reads_images: false, + tools: null, + reasoning: null, + }, + runnable: true, + not_runnable_reason: null, + default_quantization: "Q8_0", + recommended: false, + engine: "audiocpp", + lead: { quantization: "Q8_0", why: "default" }, + voicing: null, + transcription: { + peak_mb: 1776, + window_s: 28, + detects_language: true, + languages: NEMOTRON_LANGUAGES, + }, + builds: [ + { + catalog_id: "opaque-nemotron", + quantization: "Q8_0", + footprint_bytes: 930_600_000, + files: [], + fit: null, + badge: null, + can_install: true, + installed_as: null, + selected: false, + recommended: false, + reads_images: false, + projector_checked: false, + bundled: false, + ...build, + }, + ], + ...overrides, + } +} + +/** What the build reads once on disk, and once chosen for speech recognition. */ +export const SPEECH_INSTALLED = { + installed_as: "nemotron-3.5-asr-streaming-0.6b-q8_0", +} +export const SPEECH_IN_USE = { + ...SPEECH_INSTALLED, + selected: true, + selected_for: ["speech_to_text"], +} diff --git a/surfsense_local/frontend/src/features/models/local/transcription/language-names.ts b/surfsense_local/frontend/src/features/models/local/transcription/language-names.ts new file mode 100644 index 0000000000..bf5c5d55c3 --- /dev/null +++ b/surfsense_local/frontend/src/features/models/local/transcription/language-names.ts @@ -0,0 +1,28 @@ +import { intl } from "@/i18n/intl" + +export type NamedLanguage = { code: string; name: string } + +/** A language code named in the interface's language. */ +export function languageName(code: string): string { + try { + return intl.formatDisplayName(code, { type: "language" }) ?? code + } catch { + // A code Intl cannot parse is still the best name there is. + return code + } +} + +/** Alphabetical in the interface's language, so a reader finds their own. */ +export function namedLanguages(codes: readonly string[]): NamedLanguage[] { + return codes + .map((code) => ({ code, name: languageName(code) })) + .sort((a, b) => a.name.localeCompare(b.name, intl.locale)) +} + +/** The listed languages as one sentence-ready list. */ +export function listLanguages(codes: readonly string[]): string { + return intl.formatList( + namedLanguages(codes).map(({ name }) => name), + { type: "conjunction" } + ) +} diff --git a/surfsense_local/frontend/src/features/models/local/transcription/language/api.ts b/surfsense_local/frontend/src/features/models/local/transcription/language/api.ts new file mode 100644 index 0000000000..075f1cbaa6 --- /dev/null +++ b/surfsense_local/frontend/src/features/models/local/transcription/language/api.ts @@ -0,0 +1,33 @@ +import { ApiError, requestJson } from "@/lib/api" + +const URL = "/llm/selection/speech_to_text/language" + +/** The language intent of the speech model in use: `auto` or a listed code. */ +export type SpeechLanguage = { + language: string + /** ISO 639-1 codes of the languages the model is listed for. */ + languages: string[] + detects_language: boolean +} + +/** Null while no speech model is chosen, which has no intent to read. */ +export async function readSpeechLanguage( + signal?: AbortSignal +): Promise { + try { + return await requestJson(URL, { signal }) + } catch (error) { + if (error instanceof ApiError && error.code === "needs_speech_model") { + return null + } + throw error + } +} + +export function writeSpeechLanguage(language: string): Promise { + return requestJson(URL, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ language }), + }) +} diff --git a/surfsense_local/frontend/src/features/models/local/transcription/language/speech-language-select.tsx b/surfsense_local/frontend/src/features/models/local/transcription/language/speech-language-select.tsx new file mode 100644 index 0000000000..13a1bfbaba --- /dev/null +++ b/surfsense_local/frontend/src/features/models/local/transcription/language/speech-language-select.tsx @@ -0,0 +1,78 @@ +import { useId } from "react" + +import { Select } from "@/components/ui/select" +import { intl } from "@/i18n/intl" + +import { namedLanguages } from "../language-names" +import { useSpeechLanguage } from "./use-speech-language" + +const AUTO = "auto" + +function messageFrom(error: unknown) { + return error instanceof Error + ? error.message + : intl.formatMessage({ + id: "models_speech_language_request_error", + defaultMessage: "The request failed", + }) +} + +/** + * The language recordings and voice input are in. Detection is offered only + * by a model that has it; the interface's language is never assumed. + */ +export function SpeechLanguageSelect() { + const titleId = useId() + const bodyId = useId() + const { language, choose } = useSpeechLanguage() + const intent = language.data + if (!intent) return null + + return ( +
+
+
+

+ {intl.formatMessage({ + id: "models_speech_language_title", + defaultMessage: "Spoken language", + })} +

+

+ {intl.formatMessage({ + id: "models_speech_language_body", + defaultMessage: + "The language your recordings and voice input are in.", + })} +

+
+ +
+ {choose.isError ? ( +

{messageFrom(choose.error)}

+ ) : null} +
+ ) +} diff --git a/surfsense_local/frontend/src/features/models/local/transcription/language/use-speech-language.ts b/surfsense_local/frontend/src/features/models/local/transcription/language/use-speech-language.ts new file mode 100644 index 0000000000..b8f390320a --- /dev/null +++ b/surfsense_local/frontend/src/features/models/local/transcription/language/use-speech-language.ts @@ -0,0 +1,24 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query" + +import { MODELS_QUERY_KEY } from "../../../models-query" +import { readSpeechLanguage, writeSpeechLanguage } from "./api" + +/** Keyed under the models: choosing another model drops the intent. */ +export const speechLanguageQueryKey = [ + ...MODELS_QUERY_KEY, + "speech", + "language", +] as const + +export function useSpeechLanguage() { + const client = useQueryClient() + const language = useQuery({ + queryKey: speechLanguageQueryKey, + queryFn: ({ signal }) => readSpeechLanguage(signal), + }) + const choose = useMutation({ + mutationFn: writeSpeechLanguage, + onSuccess: (saved) => client.setQueryData(speechLanguageQueryKey, saved), + }) + return { language, choose } +} diff --git a/surfsense_local/frontend/src/features/models/local/transcription/listed-languages.tsx b/surfsense_local/frontend/src/features/models/local/transcription/listed-languages.tsx new file mode 100644 index 0000000000..31748f196a --- /dev/null +++ b/surfsense_local/frontend/src/features/models/local/transcription/listed-languages.tsx @@ -0,0 +1,54 @@ +import { useId, useState } from "react" + +import { Button } from "@/components/ui/button" +import { ChevronDownIcon } from "@/components/ui/icons" +import { intl } from "@/i18n/intl" +import { cn } from "@/lib/utils" + +import { listLanguages } from "./language-names" + +/** The languages a model is listed for: counted, named once opened. */ +export function ListedLanguages({ codes }: { codes: readonly string[] }) { + const listId = useId() + const [open, setOpen] = useState(false) + + // One language is named in the row's facts instead. + if (codes.length < 2) return null + + return ( +
+ + {open ? ( +

+ {listLanguages(codes)} +

+ ) : null} +
+ ) +} diff --git a/surfsense_local/frontend/src/features/models/local/transcription/use-delete-local-speech-model.ts b/surfsense_local/frontend/src/features/models/local/transcription/use-delete-local-speech-model.ts new file mode 100644 index 0000000000..7aebb634ec --- /dev/null +++ b/surfsense_local/frontend/src/features/models/local/transcription/use-delete-local-speech-model.ts @@ -0,0 +1,12 @@ +import { useMutation } from "@tanstack/react-query" + +import { useRefreshModels } from "../../models-query" +import { deleteLocalSpeechModel } from "./api" + +export function useDeleteLocalSpeechModel() { + const refresh = useRefreshModels() + return useMutation({ + mutationFn: (installedAs: string) => deleteLocalSpeechModel(installedAs), + onSuccess: () => refresh(), + }) +} diff --git a/surfsense_local/frontend/src/features/models/local/transcription/use-local-speech-catalog.ts b/surfsense_local/frontend/src/features/models/local/transcription/use-local-speech-catalog.ts new file mode 100644 index 0000000000..ce140c38c8 --- /dev/null +++ b/surfsense_local/frontend/src/features/models/local/transcription/use-local-speech-catalog.ts @@ -0,0 +1,11 @@ +import { useQuery } from "@tanstack/react-query" + +import { MODELS_QUERY_KEY } from "../../models-query" +import { getLocalSpeechModels } from "./api" + +export function useLocalSpeechCatalog() { + return useQuery({ + queryKey: [...MODELS_QUERY_KEY, "local", "speech"], + queryFn: ({ signal }) => getLocalSpeechModels(signal), + }) +} diff --git a/surfsense_local/frontend/src/features/models/model-type.ts b/surfsense_local/frontend/src/features/models/model-type.ts index 4b922de35d..c19fed1de1 100644 --- a/surfsense_local/frontend/src/features/models/model-type.ts +++ b/surfsense_local/frontend/src/features/models/model-type.ts @@ -8,6 +8,7 @@ export const MODEL_TYPES = [ "image_edit", "video_gen", "audio_gen", + "speech_to_text", ] as const export type ModelType = (typeof MODEL_TYPES)[number] diff --git a/surfsense_local/frontend/src/features/models/remote/models/server-models.tsx b/surfsense_local/frontend/src/features/models/remote/models/server-models.tsx index 7cd4815803..384d208119 100644 --- a/surfsense_local/frontend/src/features/models/remote/models/server-models.tsx +++ b/surfsense_local/frontend/src/features/models/remote/models/server-models.tsx @@ -57,6 +57,11 @@ const TYPE_LABELS: Record string> = { id: "models_server_models_type_audio_gen_label", defaultMessage: "Audio generation", }), + speech_to_text: () => + intl.formatMessage({ + id: "models_server_models_type_speech_to_text_label", + defaultMessage: "Speech recognition", + }), } function messageFrom(error: unknown) { diff --git a/surfsense_local/frontend/src/features/models/your-models/in-use-summary.tsx b/surfsense_local/frontend/src/features/models/your-models/in-use-summary.tsx index 9b5ec0603d..5ef1a9fbc6 100644 --- a/surfsense_local/frontend/src/features/models/your-models/in-use-summary.tsx +++ b/surfsense_local/frontend/src/features/models/your-models/in-use-summary.tsx @@ -20,7 +20,7 @@ export function InUseSummary({ { id: "models_in_use_summary_aria", defaultMessage: - "{slot, select, chat {chat model in use} image {image model in use} image_edit {image editing model in use} video {video model in use} audio {audio model in use} other {model in use}}", + "{slot, select, chat {chat model in use} image {image model in use} image_edit {image editing model in use} video {video model in use} audio {audio model in use} speech {speech recognition model in use} other {model in use}}", }, { slot } )} @@ -47,7 +47,7 @@ export function InUseSummary({ { id: "models_in_use_summary_empty", defaultMessage: - "{slot, select, chat {No chat model chosen} image {No image model chosen} image_edit {No image editing model chosen} video {No video model chosen} audio {No audio model chosen} other {No model chosen}}", + "{slot, select, chat {No chat model chosen} image {No image model chosen} image_edit {No image editing model chosen} video {No video model chosen} audio {No audio model chosen} speech {No speech recognition model chosen} other {No model chosen}}", }, { slot } )} diff --git a/surfsense_local/frontend/src/features/models/your-models/use-speech-models.ts b/surfsense_local/frontend/src/features/models/your-models/use-speech-models.ts new file mode 100644 index 0000000000..93df77d2d4 --- /dev/null +++ b/surfsense_local/frontend/src/features/models/your-models/use-speech-models.ts @@ -0,0 +1,44 @@ +import { useLocalSpeechCatalog } from "../local/transcription/use-local-speech-catalog" +import { useSelection } from "../selection/use-selection" +import { + describeInUse, + type YourModelRow, + type YourModels, +} from "./your-model-row" + +/** Every speech recognition model on disk, and the one in use. Local only. */ +export function useSpeechModels(): YourModels { + const catalog = useLocalSpeechCatalog() + const selection = useSelection("speech_to_text") + const models = catalog.data ?? [] + + const local: YourModelRow[] = models.flatMap((model) => + model.installed_as === null + ? [] + : [ + { + key: model.installed_as, + name: model.label, + selected: model.selected, + badges: [], + // The server loads a model on its first request; nothing to report. + note: null, + target: { + provider: "audiocpp", + connection_id: null, + name: model.installed_as, + }, + removeId: model.installed_as, + }, + ] + ) + + return { + local, + // No server serves this slot, so there is none to name. + inUse: describeInUse(selection.data, local, undefined), + isPending: catalog.isPending || selection.isPending, + error: selection.error, + canDownload: models.length > 0, + } +} diff --git a/surfsense_local/frontend/src/features/settings/models/model-slot-settings.tsx b/surfsense_local/frontend/src/features/settings/models/model-slot-settings.tsx index 7cdf1f7be6..1bd369174a 100644 --- a/surfsense_local/frontend/src/features/settings/models/model-slot-settings.tsx +++ b/surfsense_local/frontend/src/features/settings/models/model-slot-settings.tsx @@ -29,6 +29,37 @@ import { SettingsSection } from "../settings-section" type Page = "list" | "add" +/** How an empty slot is filled: from this computer, a server, or both. */ +function emptyBody({ + localOnly, + canDownload, +}: { + localOnly: boolean + canDownload: boolean +}) { + if (localOnly) { + return canDownload + ? intl.formatMessage({ + id: "settings_models_empty_local_body", + defaultMessage: "Download one to run on this computer.", + }) + : intl.formatMessage({ + id: "settings_models_empty_unsupported_body", + defaultMessage: "This computer can’t run one.", + }) + } + return canDownload + ? intl.formatMessage({ + id: "settings_models_empty_download_body", + defaultMessage: + "Download one to run on this computer, or use one from a server you already run.", + }) + : intl.formatMessage({ + id: "settings_models_empty_server_body", + defaultMessage: "Use one from a server you already run.", + }) +} + /** * One slot's settings: the model in use, then every source as a group. Adding * a model and editing a server are pages with a way back, never dialogs. @@ -46,6 +77,7 @@ export function ModelSlotSettings({ onSelected, onChatCleared, inUseDetail, + localOnly = false, }: { title: string description: string @@ -60,9 +92,11 @@ export function ModelSlotSettings({ onChatCleared?: () => void /** Said under the model in use, such as what the chat model can do. */ inUseDetail?: ReactNode + /** No server can fill the slot, so none is offered or listed. */ + localOnly?: boolean }) { // Only servers that can fill this slot count, so an empty slot still says so. - const connections = useConnectionsServing(modelType) + const connections = useConnectionsServing(localOnly ? null : modelType) const [page, setPage] = useState("list") // Edited in a dialog over the list; saving leaves the list as it was. const [editing, setEditing] = useState(null) @@ -83,22 +117,29 @@ export function ModelSlotSettings({ { id: "settings_models_add_page_title", defaultMessage: - "{slot, select, audio {Add an audio model} chat {Add a chat model} image {Add an image model} image_edit {Add an image editing model} video {Add a video model} other {Add a model}}", + "{slot, select, audio {Add an audio model} chat {Add a chat model} image {Add an image model} image_edit {Add an image editing model} video {Add a video model} speech {Add a speech recognition model} other {Add a model}}", }, { slot } )} - description={intl.formatMessage({ - id: "settings_models_add_page_body", - defaultMessage: - "Run one on this computer, or use one from a server you already run.", - })} + description={ + localOnly + ? intl.formatMessage({ + id: "settings_models_add_page_local_body", + defaultMessage: "Download one to run on this computer.", + }) + : intl.formatMessage({ + id: "settings_models_add_page_body", + defaultMessage: + "Run one on this computer, or use one from a server you already run.", + }) + } back={back} scrollable="all" > ) @@ -140,22 +181,13 @@ export function ModelSlotSettings({ { id: "settings_models_empty", defaultMessage: - "{slot, select, audio {No audio model yet} chat {No chat model yet} image {No image model yet} image_edit {No image editing model yet} video {No video model yet} other {No model yet}}", + "{slot, select, audio {No audio model yet} chat {No chat model yet} image {No image model yet} image_edit {No image editing model yet} video {No video model yet} speech {No speech recognition model yet} other {No model yet}}", }, { slot } )} - {models.canDownload - ? intl.formatMessage({ - id: "settings_models_empty_download_body", - defaultMessage: - "Download one to run on this computer, or use one from a server you already run.", - }) - : intl.formatMessage({ - id: "settings_models_empty_server_body", - defaultMessage: "Use one from a server you already run.", - })} + {emptyBody({ localOnly, canDownload: models.canDownload })} {add} @@ -179,16 +211,18 @@ export function ModelSlotSettings({ /> ) : null} - { - setEditing(connection) - setEditOpen(true) - }} - onSelected={onSelected} - onChatCleared={onChatCleared} - /> + {localOnly ? null : ( + { + setEditing(connection) + setEditOpen(true) + }} + onSelected={onSelected} + onChatCleared={onChatCleared} + /> + )}
)} { + cleanup() + vi.unstubAllGlobals() +}) + +const budget = { + device_total_bytes: 16_000_000_000, + device_free_bytes: 14_000_000_000, + usable_vram_bytes: 12_900_000_000, + fit_reserve_bytes: 1_073_741_824, + ram_available_bytes: 16_000_000_000, + uma: true, + has_gpu: true, +} + +const voiceRow = { + ...fakeSpeechRow(), + id: "kokoro-82m", + name: "Kokoro 82M", + types: ["audio_gen"], + selectable_for: ["audio_gen"], + transcription: null, + voicing: { peak_mb: 1421, voice_count: 46, languages: ["en-US"] }, +} + +type Extra = (path: string, init?: RequestInit) => Response | null + +function serving(rows: unknown[], extra: Extra = () => null) { + const selected = rows.some((row) => + (row as ReturnType).builds.some( + (build) => (build as { selected_for?: string[] }).selected_for + ) + ) + return vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const path = String(input) + const handled = extra(path, init) + if (handled) return handled + if (path === "/llm/catalog/local") { + return Response.json({ + budget, + gpu_status: "present", + rows, + recommended_id: null, + }) + } + if (path === "/llm/selection/speech_to_text" && init?.method === "PUT") { + return Response.json({ + model_type: "speech_to_text", + ...JSON.parse(String(init.body)), + updated_at: "2026-10-14T00:00:00Z", + }) + } + if (path === "/llm/selection/speech_to_text") { + return selected + ? Response.json({ + model_type: "speech_to_text", + provider: "audiocpp", + connection_id: null, + name: SPEECH_IN_USE.installed_as, + updated_at: "2026-10-14T00:00:00Z", + }) + : Response.json({ detail: "not selected" }, { status: 404 }) + } + if (path === "/llm/connections") return Response.json([]) + return Response.json({ detail: "not found" }, { status: 404 }) + }) +} + +const renderSettings = () => render() + +describe("speech recognition settings", () => { + it("offers no server, since only this computer recognises speech", async () => { + // No audio.cpp that transcribes: the catalog carries no speech rows. + vi.stubGlobal("fetch", serving([voiceRow])) + const user = userEvent.setup() + renderSettings() + + expect( + await screen.findByText("No speech recognition model yet") + ).toBeTruthy() + await user.click(screen.getByRole("button", { name: "Add model" })) + + expect( + screen.getByRole("heading", { + name: "Add a speech recognition model", + }) + ).toBeTruthy() + expect( + await screen.findByText( + "Speech recognition models cannot run on this computer" + ) + ).toBeTruthy() + expect(screen.queryByRole("button", { name: "Connect" })).toBeNull() + expect(screen.queryByText(/Kokoro/)).toBeNull() + }) + + it("describes each model by what transcribing takes and what it understands", async () => { + vi.stubGlobal("fetch", serving([fakeSpeechRow(), voiceRow])) + const user = userEvent.setup() + renderSettings() + + await user.click(await screen.findByRole("button", { name: "Add model" })) + + const line = (await screen.findByText("Detects the language")).parentElement + expect( + [...(line?.querySelectorAll(":scope > span") ?? [])].map( + (part) => part.textContent + ) + ).toEqual([ + "Q8_0", + "931 MB", + "1.8 GB while transcribing", + "Detects the language", + ]) + + // Counted, then named in the interface's language on request. + const count = screen.getByRole("button", { name: "26 languages" }) + expect(count.getAttribute("aria-expanded")).toBe("false") + await user.click(count) + expect(count.getAttribute("aria-expanded")).toBe("true") + expect(screen.getByText(/Arabic, Bulgarian, Chinese/)).toBeTruthy() + expect(screen.getByText(/Norwegian Bokmål/)).toBeTruthy() + }) + + it("downloads a speech model and chooses it in one step", async () => { + const installs = fakeInstallApi({ + modelTypes: { "opaque-nemotron": ["speech_to_text"] }, + }) + vi.stubGlobal("fetch", serving([fakeSpeechRow()], installs.handle)) + const user = userEvent.setup() + renderSettings() + + await user.click(await screen.findByRole("button", { name: "Add model" })) + await user.click( + await screen.findByRole("button", { + name: "Download Nemotron 3.5 ASR Q8_0", + }) + ) + + await waitFor(() => + expect(installs.started).toEqual([ + { + catalog_id: "opaque-nemotron", + select: true, + model_type: "speech_to_text", + }, + ]) + ) + installs.move({ + type: "downloading", + message: "Downloading", + completed: 465_300_000, + total: 930_600_000, + }) + expect((await screen.findAllByText("50%")).length).toBeGreaterThan(0) + }) + + it("uses a downloaded speech model through audio.cpp", async () => { + const fetchMock = serving([fakeSpeechRow({}, SPEECH_INSTALLED)]) + vi.stubGlobal("fetch", fetchMock) + const user = userEvent.setup() + renderSettings() + + await user.click( + await screen.findByRole("button", { name: "Use Nemotron 3.5 ASR" }) + ) + + await waitFor(() => { + const write = fetchMock.mock.calls.find( + ([path, init]) => + path === "/llm/selection/speech_to_text" && init?.method === "PUT" + ) + expect(JSON.parse(String(write?.[1]?.body))).toMatchObject({ + provider: "audiocpp", + connection_id: null, + name: "nemotron-3.5-asr-streaming-0.6b-q8_0", + }) + }) + }) + + it("deletes the speech model in use, after a warning", async () => { + const fetchMock = serving( + [fakeSpeechRow({}, SPEECH_IN_USE)], + (path, init) => { + if (path === "/llm/selection/speech_to_text/language") { + return Response.json({ + language: "auto", + languages: NEMOTRON_LANGUAGES, + detects_language: true, + }) + } + return path === `/llm/models/${SPEECH_IN_USE.installed_as}` && + init?.method === "DELETE" + ? Response.json({ + name: SPEECH_IN_USE.installed_as, + selection_cleared: true, + }) + : null + } + ) + vi.stubGlobal("fetch", fetchMock) + const user = userEvent.setup() + renderSettings() + + await user.click( + await screen.findByRole("button", { name: "Delete Nemotron 3.5 ASR" }) + ) + expect(await screen.findByText(/This is your current model/)).toBeTruthy() + await user.click(screen.getByRole("button", { name: "Delete model" })) + + await waitFor(() => + expect( + fetchMock.mock.calls.some( + ([path, init]) => + path === `/llm/models/${SPEECH_IN_USE.installed_as}` && + init?.method === "DELETE" + ) + ).toBe(true) + ) + }) + + it("offers detecting the language, then each language the model lists", async () => { + const writes: unknown[] = [] + const fetchMock = serving( + [fakeSpeechRow({}, SPEECH_IN_USE)], + (path, init) => { + if (path !== "/llm/selection/speech_to_text/language") return null + if (init?.method === "PUT") { + const body = JSON.parse(String(init.body)) as { language: string } + writes.push(body) + return Response.json({ + language: body.language, + languages: ["ja", "en", "es"], + detects_language: true, + }) + } + return Response.json({ + language: "auto", + languages: ["ja", "en", "es"], + detects_language: true, + }) + } + ) + vi.stubGlobal("fetch", fetchMock) + const user = userEvent.setup() + renderSettings() + + const select = await screen.findByRole("combobox", { + name: "Spoken language", + }) + expect( + within(select) + .getAllByRole("option") + .map((option) => option.textContent) + ).toEqual(["Detect automatically", "English", "Japanese", "Spanish"]) + expect((select as HTMLSelectElement).value).toBe("auto") + + await user.selectOptions(select, "Japanese") + + await waitFor(() => expect(writes).toEqual([{ language: "ja" }])) + await waitFor(() => expect((select as HTMLSelectElement).value).toBe("ja")) + }) + + it("offers only the listed languages when the model cannot detect one", async () => { + vi.stubGlobal( + "fetch", + serving([fakeSpeechRow({}, SPEECH_IN_USE)], (path) => + path === "/llm/selection/speech_to_text/language" + ? Response.json({ + language: "de", + languages: ["de", "fr"], + detects_language: false, + }) + : null + ) + ) + renderSettings() + + const select = await screen.findByRole("combobox", { + name: "Spoken language", + }) + expect( + within(select) + .getAllByRole("option") + .map((option) => option.textContent) + ).toEqual(["French", "German"]) + expect((select as HTMLSelectElement).value).toBe("de") + }) +}) diff --git a/surfsense_local/frontend/src/features/settings/models/speech-models-settings.tsx b/surfsense_local/frontend/src/features/settings/models/speech-models-settings.tsx new file mode 100644 index 0000000000..d140a028e1 --- /dev/null +++ b/surfsense_local/frontend/src/features/settings/models/speech-models-settings.tsx @@ -0,0 +1,45 @@ +import { usePendingInstalls } from "@/features/models/local/installs/pending-installs" +import { DownloadSpeechModels } from "@/features/models/local/transcription/download-speech-models" +import { SpeechLanguageSelect } from "@/features/models/local/transcription/language/speech-language-select" +import { useDeleteLocalSpeechModel } from "@/features/models/local/transcription/use-delete-local-speech-model" +import { useSelect } from "@/features/models/selection/use-selection" +import { useSpeechModels } from "@/features/models/your-models/use-speech-models" +import { intl } from "@/i18n/intl" + +import { ModelSlotSettings } from "./model-slot-settings" + +/** Speech recognition runs on this computer only: no server can fill it. */ +export function SpeechModelsSettings() { + const models = useSpeechModels() + const select = useSelect("speech_to_text") + const remove = useDeleteLocalSpeechModel() + // Only downloads whose model can fill this slot, wherever they started. + const pending = usePendingInstalls("speech_to_text") + + return ( + } + inUseDetail={models.inUse ? : null} + onUse={async (row) => { + if (row.target) await select.mutateAsync({ target: row.target }) + }} + onDelete={async (row) => { + if (row.removeId) await remove.mutateAsync(row.removeId) + }} + /> + ) +} diff --git a/surfsense_local/frontend/src/features/settings/settings-dialog.test.tsx b/surfsense_local/frontend/src/features/settings/settings-dialog.test.tsx index 221985bf28..584d0f7f83 100644 --- a/surfsense_local/frontend/src/features/settings/settings-dialog.test.tsx +++ b/surfsense_local/frontend/src/features/settings/settings-dialog.test.tsx @@ -93,6 +93,7 @@ describe("SettingsDialog", () => { "Image gen", "Image edit", "Audio", + "Speech recognition", "Video", "Embedding", ], @@ -376,6 +377,15 @@ describe("SettingsDialog", () => { name: "audio model in use", }) await waitFor(() => expect(audio.textContent).toContain("tts-1")) + + await user.click(screen.getByRole("button", { name: "Speech recognition" })) + + expect( + await screen.findByRole("heading", { name: "Speech recognition models" }) + ).toBeTruthy() + expect( + await screen.findByText("No speech recognition model yet") + ).toBeTruthy() }) it("shows nothing half-loaded while model data loads", async () => { diff --git a/surfsense_local/frontend/src/features/settings/settings-dialog.tsx b/surfsense_local/frontend/src/features/settings/settings-dialog.tsx index 03583fa62e..35c6b4f8a2 100644 --- a/surfsense_local/frontend/src/features/settings/settings-dialog.tsx +++ b/surfsense_local/frontend/src/features/settings/settings-dialog.tsx @@ -18,6 +18,7 @@ import { Image01Icon, InformationCircleIcon, LicenseIcon, + Mic01Icon, PlugIcon, Settings2Icon, EmbeddingIcon, @@ -52,6 +53,7 @@ import { AudioModelsSettings } from "./models/audio-models-settings" import { ChatModelsSettings } from "./models/chat-models-settings" import { ImageEditModelsSettings } from "./models/image-edit-models-settings" import { ImageModelsSettings } from "./models/image-models-settings" +import { SpeechModelsSettings } from "./models/speech-models-settings" import { VideoModelsSettings } from "./models/video-models-settings" import { SettingsSection } from "./settings-section" @@ -67,6 +69,7 @@ export type SettingsSectionId = | "image-models" | "image-edit-models" | "audio-models" + | "speech-models" | "video-models" | "embedding-model" | "resources" @@ -216,6 +219,7 @@ const SETTINGS_SECTIONS = [ { id: "image-models", group: "models", icon: Image01Icon }, { id: "image-edit-models", group: "models", icon: AiImageEditIcon }, { id: "audio-models", group: "models", icon: AudioWaveformIcon }, + { id: "speech-models", group: "models", icon: Mic01Icon }, { id: "video-models", group: "models", icon: Video01Icon }, { id: "embedding-model", group: "models", icon: EmbeddingIcon }, { id: "resources", group: "system", icon: CpuIcon }, @@ -279,6 +283,11 @@ const SECTION_LABELS: Record string> = { id: "settings_nav_audio_label", defaultMessage: "Audio", }), + "speech-models": () => + intl.formatMessage({ + id: "settings_nav_speech_label", + defaultMessage: "Speech recognition", + }), "embedding-model": () => intl.formatMessage({ id: "settings_nav_embedding_label", @@ -437,6 +446,9 @@ export function SettingsDialog({ {activeSection.id === "audio-models" ? ( ) : null} + {activeSection.id === "speech-models" ? ( + + ) : null} {activeSection.id === "embedding-model" ? ( ) : null} diff --git a/surfsense_local/frontend/src/features/speech-setup/speech-model-card.tsx b/surfsense_local/frontend/src/features/speech-setup/speech-model-card.tsx new file mode 100644 index 0000000000..7132640d96 --- /dev/null +++ b/surfsense_local/frontend/src/features/speech-setup/speech-model-card.tsx @@ -0,0 +1,92 @@ +import { useId } from "react" + +import { Button } from "@/components/ui/button" +import { DotIcon, DownloadIcon } from "@/components/ui/icons" +import { Spinner } from "@/components/ui/spinner" +import { InstallProgress } from "@/features/models/local/chat/install-progress" +import type { InstallJob } from "@/features/models/local/installs/api" +import type { LocalSpeechModel } from "@/features/models/local/transcription/api" +import { + speechModelMemory, + speechModelSize, +} from "@/features/models/local/transcription/describe-speech-model" +import { listLanguages } from "@/features/models/local/transcription/language-names" +import { intl } from "@/i18n/intl" + +/** One curated model: what it downloads, what it takes, where it works well. */ +export function SpeechModelCard({ + model, + download, + busy, + onSetUp, + onCancel, +}: { + model: LocalSpeechModel + /** This model's download, while it runs. */ + download: InstallJob | null + busy: boolean + onSetUp: () => void + onCancel: (jobId: string) => void +}) { + const nameId = useId() + + return ( +
+
+

+ {model.label} +

+

+ {speechModelSize(model)} +

+
+ {model.languages.length > 0 ? ( +

+ {intl.formatMessage( + { + id: "speech_setup_card_languages_body", + defaultMessage: "Works well in: {languages}", + }, + { languages: listLanguages(model.languages) } + )} +

+ ) : null} + {download ? ( + onCancel(download.id)} + /> + ) : ( + + )} +
+ ) +} diff --git a/surfsense_local/frontend/src/features/speech-setup/speech-setup-popover.test.tsx b/surfsense_local/frontend/src/features/speech-setup/speech-setup-popover.test.tsx new file mode 100644 index 0000000000..2ff456f7ae --- /dev/null +++ b/surfsense_local/frontend/src/features/speech-setup/speech-setup-popover.test.tsx @@ -0,0 +1,298 @@ +import { afterEach, describe, expect, it, vi } from "vitest" +import { cleanup, screen, waitFor, within } from "@testing-library/react" +import userEvent from "@testing-library/user-event" + +import { AppDialogs } from "@/components/ui/app-dialog-slot" +import { EgressPrompt } from "@/features/egress/egress-prompt" +import { fakeInstallApi } from "@/features/models/local/installs/fake-install-api" +import { + fakeSpeechRow, + SPEECH_IN_USE, + SPEECH_INSTALLED, +} from "@/features/models/local/transcription/fake-speech-rows" +import { render } from "@/test-utils" + +import { SpeechSetupPopover } from "./speech-setup-popover" + +afterEach(() => { + cleanup() + vi.unstubAllGlobals() +}) + +type Extra = (path: string, init?: RequestInit) => Response | null + +function serving(rows: unknown[], extra: Extra = () => null) { + const calls: string[] = [] + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const path = String(input) + calls.push(`${init?.method ?? "GET"} ${path}`) + const handled = extra(path, init) + if (handled) return handled + if (path === "/llm/catalog/local") { + return Response.json({ rows, recommended_id: null }) + } + if (path === "/llm/selection/speech_to_text" && init?.method === "PUT") { + return Response.json({ + model_type: "speech_to_text", + ...JSON.parse(String(init.body)), + updated_at: "2026-10-14T00:00:00Z", + }) + } + return Response.json({ detail: "not found" }, { status: 404 }) + }) + ) + return calls +} + +function renderSetUp({ + onStart, + onMoreChoices = () => undefined, +}: { onStart?: () => void; onMoreChoices?: () => void } = {}) { + // As main.tsx mounts them, so a consent asked from inside nests there. + return render( + + Voice input} + onStart={onStart} + onMoreChoices={onMoreChoices} + /> + + ) +} + +async function open() { + const user = userEvent.setup() + await user.click(screen.getByRole("button", { name: "Voice input" })) + const popover = await screen.findByRole("dialog", { + name: "Set up speech recognition", + }) + return { user, popover } +} + +describe("speech recognition set-up", () => { + it("offers the model with its download, its memory and where it works well", async () => { + serving([fakeSpeechRow()]) + renderSetUp() + + const { popover } = await open() + + expect(popover.textContent).toContain( + "SurfSense turns your voice into text on this computer. It needs a one-time download." + ) + const card = await within(popover).findByRole("group", { + name: "Nemotron 3.5 ASR", + }) + expect(card.textContent).toContain("931 MB") + expect(card.textContent).toContain("1.8 GB while transcribing") + // Named in the interface's language, alphabetically. + expect( + within(card).getByText( + "Works well in: Arabic, Bulgarian, Chinese, Croatian, Czech, Danish, Dutch, English, Finnish, French, German, Hindi, Italian, Japanese, Korean, Norwegian Bokmål, Polish, Portuguese, Romanian, Russian, Slovak, Spanish, Swedish, Turkish, Ukrainian, and Vietnamese" + ) + ).toBeTruthy() + }) + + it("downloads the model and makes it the one in use, showing how far it got", async () => { + const installs = fakeInstallApi({ + modelTypes: { "opaque-nemotron": ["speech_to_text"] }, + }) + serving([fakeSpeechRow()], installs.handle) + renderSetUp() + const { user, popover } = await open() + + await user.click( + await within(popover).findByRole("button", { + name: "Download and set up", + }) + ) + + await waitFor(() => + expect(installs.started).toEqual([ + { + catalog_id: "opaque-nemotron", + select: true, + model_type: "speech_to_text", + }, + ]) + ) + installs.move({ + type: "downloading", + message: "Downloading", + completed: 465_300_000, + total: 930_600_000, + }) + expect(await within(popover).findByText("50%")).toBeTruthy() + expect( + within(popover).queryByRole("button", { name: "Download and set up" }) + ).toBeNull() + }) + + it("is ready to start talking the moment the download finishes", async () => { + const installs = fakeInstallApi({ + modelTypes: { "opaque-nemotron": ["speech_to_text"] }, + }) + let done = false + serving([], (path, init) => { + if (path === "/llm/catalog/local") { + return Response.json({ + rows: [fakeSpeechRow({}, done ? SPEECH_IN_USE : {})], + recommended_id: null, + }) + } + return installs.handle(path, init) + }) + renderSetUp({ onStart: () => undefined }) + const { user, popover } = await open() + + await user.click( + await within(popover).findByRole("button", { + name: "Download and set up", + }) + ) + await waitFor(() => expect(installs.started).toHaveLength(1)) + installs.move({ type: "selecting", message: "Selecting" }) + done = true + installs.complete() + + expect( + await screen.findByRole("dialog", { + name: "Speech recognition is ready", + }) + ).toBeTruthy() + expect(screen.getByRole("button", { name: "Start talking" })).toBeTruthy() + }) + + it("asks to allow downloads from Hugging Face first, and stays open", async () => { + const installs = fakeInstallApi({ + modelTypes: { "opaque-nemotron": ["speech_to_text"] }, + }) + let allowed = false + const calls = serving([fakeSpeechRow()], (path, init) => { + if (path.startsWith("/egress/")) { + allowed = true + return Response.json({}) + } + if (path === "/llm/installs" && init?.method === "POST" && !allowed) { + return Response.json( + { + detail: { + code: "egress_disabled", + message: "off", + destination: "host:huggingface.co", + host: "huggingface.co", + }, + }, + { status: 403 } + ) + } + return installs.handle(path, init) + }) + renderSetUp() + const { user, popover } = await open() + + await user.click( + await within(popover).findByRole("button", { + name: "Download and set up", + }) + ) + await screen.findByRole("alertdialog", { + name: "Allow searching and downloading models?", + }) + await user.click(screen.getByRole("button", { name: "Allow" })) + + await waitFor(() => + expect(installs.started).toEqual([ + { + catalog_id: "opaque-nemotron", + select: true, + model_type: "speech_to_text", + }, + ]) + ) + expect(calls).toContain("PUT /egress/host:huggingface.co") + expect( + screen.getByRole("dialog", { name: "Set up speech recognition" }) + ).toBeTruthy() + }) + + it("uses a model already on disk rather than downloading it again", async () => { + const calls = serving([fakeSpeechRow({}, SPEECH_INSTALLED)]) + renderSetUp() + const { user, popover } = await open() + + await user.click( + await within(popover).findByRole("button", { name: "Set up" }) + ) + + await waitFor(() => + expect(calls).toContain("PUT /llm/selection/speech_to_text") + ) + expect(calls).not.toContain("POST /llm/installs") + }) + + it("starts talking once a speech model is in use", async () => { + serving([fakeSpeechRow({}, SPEECH_IN_USE)]) + const onStart = vi.fn() + renderSetUp({ onStart }) + const user = userEvent.setup() + + await user.click(screen.getByRole("button", { name: "Voice input" })) + const popover = await screen.findByRole("dialog", { + name: "Speech recognition is ready", + }) + await user.click( + within(popover).getByRole("button", { name: "Start talking" }) + ) + + expect(onStart).toHaveBeenCalledOnce() + await waitFor(() => expect(screen.queryByRole("dialog")).toBeNull()) + }) + + it("says it is ready, with nothing to start, where it was opened for a recording", async () => { + serving([fakeSpeechRow({}, SPEECH_IN_USE)]) + renderSetUp() + const user = userEvent.setup() + + await user.click(screen.getByRole("button", { name: "Voice input" })) + const popover = await screen.findByRole("dialog", { + name: "Speech recognition is ready", + }) + + expect( + within(popover).queryByRole("button", { name: "Start talking" }) + ).toBeNull() + await user.click(within(popover).getByRole("button", { name: "Done" })) + await waitFor(() => expect(screen.queryByRole("dialog")).toBeNull()) + }) + + it("opens Settings for more choices", async () => { + serving([fakeSpeechRow()]) + const onMoreChoices = vi.fn() + renderSetUp({ onMoreChoices }) + const { user, popover } = await open() + + await user.click( + await within(popover).findByRole("button", { name: "More choices" }) + ) + + expect(onMoreChoices).toHaveBeenCalledOnce() + await waitFor(() => expect(screen.queryByRole("dialog")).toBeNull()) + }) + + it("says so where speech recognition cannot run", async () => { + serving([]) + renderSetUp() + const { popover } = await open() + + expect( + await within(popover).findByText( + "Speech recognition can’t run on this computer." + ) + ).toBeTruthy() + expect( + within(popover).queryByRole("button", { name: "Download and set up" }) + ).toBeNull() + }) +}) diff --git a/surfsense_local/frontend/src/features/speech-setup/speech-setup-popover.tsx b/surfsense_local/frontend/src/features/speech-setup/speech-setup-popover.tsx new file mode 100644 index 0000000000..490a70c2f8 --- /dev/null +++ b/surfsense_local/frontend/src/features/speech-setup/speech-setup-popover.tsx @@ -0,0 +1,178 @@ +import type { ReactElement } from "react" + +import { Button } from "@/components/ui/button" +import { + Popover, + PopoverClose, + PopoverContent, + PopoverDescription, + PopoverHeader, + PopoverTitle, + PopoverTrigger, +} from "@/components/ui/popover" +import { Spinner } from "@/components/ui/spinner" +import { intl } from "@/i18n/intl" + +import { SpeechModelCard } from "./speech-model-card" +import { useSpeechSetup } from "./use-speech-setup" + +/** + * Setting up speech recognition where it is needed: a waiting recording's row + * and the composer's mic. Downloading also chooses the model, so one click + * is the whole set-up; Settings holds every other choice. + */ +export function SpeechSetupPopover({ + trigger, + open, + onOpenChange, + onStart, + onMoreChoices, + side = "top", + align = "center", +}: { + /** What opens it, such as the mic or a row's "Set up". */ + trigger: ReactElement + open?: boolean + onOpenChange?: (open: boolean) => void + /** Present for the mic: once ready, it starts listening. */ + onStart?: () => void + /** Opens Settings › Speech recognition. */ + onMoreChoices: () => void + side?: "top" | "bottom" | "left" | "right" + align?: "start" | "center" | "end" +}) { + return ( + + + + + + + ) +} + +function SpeechSetupBody({ + onStart, + onMoreChoices, +}: { + onStart?: () => void + onMoreChoices: () => void +}) { + const setup = useSpeechSetup() + + if (setup.inUse) { + return ( + <> + + + {intl.formatMessage({ + id: "speech_setup_popover_ready_title", + defaultMessage: "Speech recognition is ready", + })} + + {setup.inUse.label} + + {onStart ? ( + } + onClick={onStart} + > + {intl.formatMessage({ + id: "speech_setup_popover_start_button", + defaultMessage: "Start talking", + })} + + ) : ( + + } + > + {intl.formatMessage({ + id: "speech_setup_popover_done_button", + defaultMessage: "Done", + })} + + )} + + ) + } + + const { catalog } = setup + return ( + <> + + + {intl.formatMessage({ + id: "speech_setup_popover_title", + defaultMessage: "Set up speech recognition", + })} + + + {intl.formatMessage({ + id: "speech_setup_popover_body", + defaultMessage: + "SurfSense turns your voice into text on this computer. It needs a one-time download.", + })} + + + + {catalog.isPending ? ( + + ) : catalog.isError ? ( +

+ {intl.formatMessage({ + id: "speech_setup_popover_load_error", + defaultMessage: "Could not load speech recognition models.", + })} +

+ ) : setup.curated.length === 0 ? ( +

+ {intl.formatMessage({ + id: "speech_setup_popover_unsupported_body", + defaultMessage: "Speech recognition can’t run on this computer.", + })} +

+ ) : ( + setup.curated.map((model) => ( + setup.setUp(model)} + onCancel={setup.cancel} + /> + )) + )} + + {setup.chooseError ? ( +

{setup.chooseError.message}

+ ) : null} + + {catalog.isSuccess && setup.curated.length > 0 ? ( + + } + onClick={onMoreChoices} + > + {intl.formatMessage({ + id: "speech_setup_popover_more_button", + defaultMessage: "More choices", + })} + + ) : null} + + ) +} diff --git a/surfsense_local/frontend/src/features/speech-setup/use-speech-setup.ts b/surfsense_local/frontend/src/features/speech-setup/use-speech-setup.ts new file mode 100644 index 0000000000..0d2586238d --- /dev/null +++ b/surfsense_local/frontend/src/features/speech-setup/use-speech-setup.ts @@ -0,0 +1,62 @@ +import { useInstall } from "@/features/models/local/installs/use-install" +import { jobFor, jobsFor } from "@/features/models/local/installs/job-state" +import type { LocalSpeechModel } from "@/features/models/local/transcription/api" +import { useLocalSpeechCatalog } from "@/features/models/local/transcription/use-local-speech-catalog" +import { useSelect } from "@/features/models/selection/use-selection" + +/** + * Where setting up speech recognition stands, and the one step that does it. + * Shared by the set-up popover and whatever opens it, so a waiting + * recording's row and the mic read the same download. + */ +export function useSpeechSetup() { + const catalog = useLocalSpeechCatalog() + const { jobs, installs, install, cancel } = useInstall({ + select: true, + modelType: "speech_to_text", + }) + const select = useSelect("speech_to_text") + const models = catalog.data ?? [] + + // A model on disk and chosen: recordings and voice input can use it. + const inUse = + models.find((model) => model.selected && model.installed_as) ?? null + + const setUp = (model: LocalSpeechModel) => { + if (model.installed_as) { + void select + .mutateAsync({ + target: { + provider: "audiocpp", + connection_id: null, + name: model.installed_as, + }, + }) + .catch(() => undefined) + } else { + void install(model.catalog_id) + } + } + + return { + catalog, + /** One card each; a model the user brought is chosen in Settings. */ + curated: models.filter((model) => model.curated), + inUse, + /** A speech download under way, wherever it was started. */ + download: jobsFor(installs, "speech_to_text")[0] ?? null, + downloadOf: (model: LocalSpeechModel) => + jobFor(installs, model.catalog_id) ?? null, + /** Downloaded and chosen, but the catalog has not said so yet. */ + settling: (model: LocalSpeechModel) => + catalog.isFetching && + jobs.some( + (job) => + job.catalog_id === model.catalog_id && job.event.type === "complete" + ), + setUp, + choosing: select.isPending, + chooseError: select.error, + cancel, + } +} diff --git a/surfsense_local/frontend/translations/de.json b/surfsense_local/frontend/translations/de.json index c7580fbddd..ea0418601a 100644 --- a/surfsense_local/frontend/translations/de.json +++ b/surfsense_local/frontend/translations/de.json @@ -449,6 +449,10 @@ "models_download_image_unexpected_error": "Ein unerwarteter Fehler ist aufgetreten", "models_download_image_unsupported_body": "Diese Version hat keine lokale Bild-Laufzeit. Verwende stattdessen oben einen Server.", "models_download_image_unsupported_title": "Bildmodelle laufen nicht auf diesem Computer", + "models_download_speech_delete_aria": "{model} löschen", + "models_download_speech_load_error": "Spracherkennungsmodelle konnten nicht geladen werden", + "models_download_speech_request_error": "Die Anfrage ist fehlgeschlagen", + "models_download_speech_unsupported_title": "Spracherkennungsmodelle laufen nicht auf diesem Computer", "models_download_video_unsupported_title": "Videomodelle laufen nicht auf diesem Computer", "models_error_install_checksum_mismatch": "Die heruntergeladene Datei stimmt nicht mit der erwarteten überein. Versuche den Download erneut.", "models_error_install_failed": "Das Modell konnte nicht installiert werden. Versuche den Download erneut.", @@ -479,8 +483,8 @@ "models_image_server_running_status": "Läuft", "models_in_use_local_label": "Dieser Computer", "models_in_use_missing_label": "Nicht auf diesem Computer gefunden", - "models_in_use_summary_aria": "{slot, select, chat {Aktives Chat-Modell} image {Aktives Bildmodell} image_edit {Aktives Bildbearbeitungsmodell} video {Aktives Videomodell} audio {Aktives Audiomodell} other {Aktives Modell}}", - "models_in_use_summary_empty": "{slot, select, chat {Kein Chat-Modell gewählt} image {Kein Bildmodell gewählt} image_edit {Kein Bildbearbeitungsmodell gewählt} video {Kein Videomodell gewählt} audio {Kein Audiomodell gewählt} other {Kein Modell gewählt}}", + "models_in_use_summary_aria": "{slot, select, chat {Aktives Chat-Modell} image {Aktives Bildmodell} image_edit {Aktives Bildbearbeitungsmodell} video {Aktives Videomodell} audio {Aktives Audiomodell} speech {Aktives Spracherkennungsmodell} other {Aktives Modell}}", + "models_in_use_summary_empty": "{slot, select, chat {Kein Chat-Modell gewählt} image {Kein Bildmodell gewählt} image_edit {Kein Bildbearbeitungsmodell gewählt} video {Kein Videomodell gewählt} audio {Kein Audiomodell gewählt} speech {Kein Spracherkennungsmodell gewählt} other {Kein Modell gewählt}}", "models_in_use_summary_label": "Aktiv:", "models_in_use_unknown_server_label": "Ein Server", "models_install_cancelled_short_status": "Abgebrochen", @@ -600,6 +604,7 @@ "models_server_models_type_audio_gen_label": "Audioerzeugung", "models_server_models_type_image_edit_label": "Bildbearbeitung", "models_server_models_type_image_gen_label": "Bilderzeugung", + "models_server_models_type_speech_to_text_label": "Spracherkennung", "models_server_models_type_text_gen_label": "Texterzeugung", "models_server_models_type_video_gen_label": "Videoerzeugung", "models_server_models_use_aria": "{model} verwenden", @@ -619,6 +624,14 @@ "models_server_voices_trial_body": "Hallo. So werden deine Podcasts klingen.", "models_server_voices_voice_aria": "Stimmen-ID", "models_server_voices_voice_placeholder": "Stimmen-ID", + "models_speech_language_auto_label": "Automatisch erkennen", + "models_speech_language_body": "Die Sprache deiner Aufnahmen und Spracheingaben.", + "models_speech_language_request_error": "Die Anfrage ist fehlgeschlagen", + "models_speech_language_title": "Gesprochene Sprache", + "models_speech_model_detects_language_label": "Erkennt die Sprache", + "models_speech_model_languages_button": "{count, plural, one {# Sprache} other {# Sprachen}}", + "models_speech_model_peak_memory_gigabytes_label": "{size, number, ::unit/gigabyte .#} beim Transkribieren", + "models_speech_model_peak_memory_megabytes_label": "{size, number, ::unit/megabyte .} beim Transkribieren", "models_try_dialog_cancel_button": "Abbrechen", "models_try_dialog_chat_confirmed_body": "Chat-Unterstützung ist bestätigt. Der Test sendet einen kurzen Prompt und kann Kosten verursachen.", "models_try_dialog_chat_unconfirmed_body": "Dieser Endpunkt veröffentlicht keine Fähigkeiten, daher ist Chat-Unterstützung nicht bestätigt. Der Test sendet einen kurzen Prompt und kann Kosten verursachen.", @@ -974,10 +987,13 @@ "settings_image_models_title": "Modelle für Bilderzeugung", "settings_models_add_button": "Modell hinzufügen", "settings_models_add_page_body": "Führe eines auf diesem Computer aus oder nutze eines von einem Server, den du bereits betreibst.", - "settings_models_add_page_title": "{slot, select, audio {Audiomodell hinzufügen} chat {Chatmodell hinzufügen} image {Bildmodell hinzufügen} image_edit {Bildbearbeitungsmodell hinzufügen} video {Videomodell hinzufügen} other {Modell hinzufügen}}", - "settings_models_empty": "{slot, select, audio {Noch kein Audiomodell} chat {Noch kein Chatmodell} image {Noch kein Bildmodell} image_edit {Noch kein Bildbearbeitungsmodell} video {Noch kein Videomodell} other {Noch kein Modell}}", + "settings_models_add_page_local_body": "Lade eines herunter, um es auf diesem Computer auszuführen.", + "settings_models_add_page_title": "{slot, select, audio {Audiomodell hinzufügen} chat {Chatmodell hinzufügen} image {Bildmodell hinzufügen} image_edit {Bildbearbeitungsmodell hinzufügen} video {Videomodell hinzufügen} speech {Spracherkennungsmodell hinzufügen} other {Modell hinzufügen}}", + "settings_models_empty": "{slot, select, audio {Noch kein Audiomodell} chat {Noch kein Chatmodell} image {Noch kein Bildmodell} image_edit {Noch kein Bildbearbeitungsmodell} video {Noch kein Videomodell} speech {Noch kein Spracherkennungsmodell} other {Noch kein Modell}}", "settings_models_empty_download_body": "Lade eines herunter, um es auf diesem Computer auszuführen, oder nutze eines von einem Server, den du bereits betreibst.", + "settings_models_empty_local_body": "Lade eines herunter, um es auf diesem Computer auszuführen.", "settings_models_empty_server_body": "Nutze eines von einem Server, den du bereits betreibst.", + "settings_models_empty_unsupported_body": "Dieser Computer kann keines ausführen.", "settings_models_load_error": "Modelleinstellungen konnten nicht geladen werden", "settings_nav_about_label": "Über", "settings_nav_app_title": "App", @@ -994,9 +1010,12 @@ "settings_nav_plugins_label": "Plugins", "settings_nav_report_issue_label": "Problem melden", "settings_nav_resources_label": "Ressourcen", + "settings_nav_speech_label": "Spracherkennung", "settings_nav_system_title": "System", "settings_nav_title": "Einstellungen", "settings_nav_video_label": "Video", + "settings_speech_models_body": "Das Modell, das Aufnahmen und Spracheingaben auf diesem Computer in Text umwandelt.", + "settings_speech_models_title": "Modelle für Spracherkennung", "settings_video_models_body": "Das Modell, das Videos erstellt. Führe eines auf diesem Computer aus oder nutze eines von einem Server.", "settings_video_models_title": "Modelle für Videoerzeugung", "source_preview_panel_aria": "Quellenvorschau", @@ -1112,6 +1131,17 @@ "sources_upload_too_large_reason_label": "Über {size, number, ::unit/megabyte}", "sources_upload_unsupported_body": "Nicht unterstützt: {files}", "sources_upload_unsupported_error": "Nicht unterstützter Dateityp: {files}", + "speech_setup_card_download_button": "Herunterladen und einrichten", + "speech_setup_card_languages_body": "Gut geeignet für: {languages}", + "speech_setup_card_use_button": "Einrichten", + "speech_setup_popover_body": "SurfSense wandelt deine Stimme auf diesem Computer in Text um. Dafür ist ein einmaliger Download nötig.", + "speech_setup_popover_done_button": "Fertig", + "speech_setup_popover_load_error": "Spracherkennungsmodelle konnten nicht geladen werden.", + "speech_setup_popover_more_button": "Weitere Optionen", + "speech_setup_popover_ready_title": "Spracherkennung ist bereit", + "speech_setup_popover_start_button": "Jetzt sprechen", + "speech_setup_popover_title": "Spracherkennung einrichten", + "speech_setup_popover_unsupported_body": "Spracherkennung läuft nicht auf diesem Computer.", "studio_artifact_failed_toast": "{name} fehlgeschlagen", "studio_artifact_failed_toast_body": "Dieses Artefakt konnte nicht erstellt werden. Versuche es im Tab „Artefakte“ erneut.", "studio_artifact_list_clear_filter_button": "Filter entfernen", diff --git a/surfsense_local/frontend/translations/en.json b/surfsense_local/frontend/translations/en.json index f2100dbba5..dac573f937 100644 --- a/surfsense_local/frontend/translations/en.json +++ b/surfsense_local/frontend/translations/en.json @@ -449,6 +449,10 @@ "models_download_image_unexpected_error": "An unexpected error occurred", "models_download_image_unsupported_body": "This build has no local image runtime. Use a server above instead.", "models_download_image_unsupported_title": "Image models cannot run on this computer", + "models_download_speech_delete_aria": "Delete {model}", + "models_download_speech_load_error": "Could not load speech recognition models", + "models_download_speech_request_error": "The request failed", + "models_download_speech_unsupported_title": "Speech recognition models cannot run on this computer", "models_download_video_unsupported_title": "Video models cannot run on this computer", "models_error_install_checksum_mismatch": "The downloaded file did not match the expected one. Retry the download.", "models_error_install_failed": "The model could not be installed. Retry the download.", @@ -479,8 +483,8 @@ "models_image_server_running_status": "Running", "models_in_use_local_label": "This computer", "models_in_use_missing_label": "Not found on this computer", - "models_in_use_summary_aria": "{slot, select, chat {chat model in use} image {image model in use} image_edit {image editing model in use} video {video model in use} audio {audio model in use} other {model in use}}", - "models_in_use_summary_empty": "{slot, select, chat {No chat model chosen} image {No image model chosen} image_edit {No image editing model chosen} video {No video model chosen} audio {No audio model chosen} other {No model chosen}}", + "models_in_use_summary_aria": "{slot, select, chat {chat model in use} image {image model in use} image_edit {image editing model in use} video {video model in use} audio {audio model in use} speech {speech recognition model in use} other {model in use}}", + "models_in_use_summary_empty": "{slot, select, chat {No chat model chosen} image {No image model chosen} image_edit {No image editing model chosen} video {No video model chosen} audio {No audio model chosen} speech {No speech recognition model chosen} other {No model chosen}}", "models_in_use_summary_label": "In use:", "models_in_use_unknown_server_label": "A server", "models_install_cancelled_short_status": "Cancelled", @@ -600,6 +604,7 @@ "models_server_models_type_audio_gen_label": "Audio generation", "models_server_models_type_image_edit_label": "Image editing", "models_server_models_type_image_gen_label": "Image generation", + "models_server_models_type_speech_to_text_label": "Speech recognition", "models_server_models_type_text_gen_label": "Text generation", "models_server_models_type_video_gen_label": "Video generation", "models_server_models_use_aria": "Use {model}", @@ -619,6 +624,14 @@ "models_server_voices_trial_body": "Hello. This is how your podcasts will sound.", "models_server_voices_voice_aria": "Voice ID", "models_server_voices_voice_placeholder": "Voice ID", + "models_speech_language_auto_label": "Detect automatically", + "models_speech_language_body": "The language your recordings and voice input are in.", + "models_speech_language_request_error": "The request failed", + "models_speech_language_title": "Spoken language", + "models_speech_model_detects_language_label": "Detects the language", + "models_speech_model_languages_button": "{count, plural, one {# language} other {# languages}}", + "models_speech_model_peak_memory_gigabytes_label": "{size, number, ::unit/gigabyte .#} while transcribing", + "models_speech_model_peak_memory_megabytes_label": "{size, number, ::unit/megabyte .} while transcribing", "models_try_dialog_cancel_button": "Cancel", "models_try_dialog_chat_confirmed_body": "Chat support is confirmed. Testing sends one short prompt and may cost money.", "models_try_dialog_chat_unconfirmed_body": "This endpoint does not publish capabilities, so chat support is unconfirmed. Testing sends one short prompt and may cost money.", @@ -974,10 +987,13 @@ "settings_image_models_title": "Image generation models", "settings_models_add_button": "Add model", "settings_models_add_page_body": "Run one on this computer, or use one from a server you already run.", - "settings_models_add_page_title": "{slot, select, audio {Add an audio model} chat {Add a chat model} image {Add an image model} image_edit {Add an image editing model} video {Add a video model} other {Add a model}}", - "settings_models_empty": "{slot, select, audio {No audio model yet} chat {No chat model yet} image {No image model yet} image_edit {No image editing model yet} video {No video model yet} other {No model yet}}", + "settings_models_add_page_local_body": "Download one to run on this computer.", + "settings_models_add_page_title": "{slot, select, audio {Add an audio model} chat {Add a chat model} image {Add an image model} image_edit {Add an image editing model} video {Add a video model} speech {Add a speech recognition model} other {Add a model}}", + "settings_models_empty": "{slot, select, audio {No audio model yet} chat {No chat model yet} image {No image model yet} image_edit {No image editing model yet} video {No video model yet} speech {No speech recognition model yet} other {No model yet}}", "settings_models_empty_download_body": "Download one to run on this computer, or use one from a server you already run.", + "settings_models_empty_local_body": "Download one to run on this computer.", "settings_models_empty_server_body": "Use one from a server you already run.", + "settings_models_empty_unsupported_body": "This computer can’t run one.", "settings_models_load_error": "Could not load model settings", "settings_nav_about_label": "About", "settings_nav_app_title": "App", @@ -994,9 +1010,12 @@ "settings_nav_plugins_label": "Plugins", "settings_nav_report_issue_label": "Report issue", "settings_nav_resources_label": "Resources", + "settings_nav_speech_label": "Speech recognition", "settings_nav_system_title": "System", "settings_nav_title": "Settings", "settings_nav_video_label": "Video", + "settings_speech_models_body": "The model that turns recordings and voice input into text, on this computer.", + "settings_speech_models_title": "Speech recognition models", "settings_video_models_body": "The model that makes video. Run one on this computer, or use one from a server.", "settings_video_models_title": "Video generation models", "source_preview_panel_aria": "Source preview", @@ -1112,6 +1131,17 @@ "sources_upload_too_large_reason_label": "Over {size, number, ::unit/megabyte}", "sources_upload_unsupported_body": "Not supported: {files}", "sources_upload_unsupported_error": "Unsupported file type: {files}", + "speech_setup_card_download_button": "Download and set up", + "speech_setup_card_languages_body": "Works well in: {languages}", + "speech_setup_card_use_button": "Set up", + "speech_setup_popover_body": "SurfSense turns your voice into text on this computer. It needs a one-time download.", + "speech_setup_popover_done_button": "Done", + "speech_setup_popover_load_error": "Could not load speech recognition models.", + "speech_setup_popover_more_button": "More choices", + "speech_setup_popover_ready_title": "Speech recognition is ready", + "speech_setup_popover_start_button": "Start talking", + "speech_setup_popover_title": "Set up speech recognition", + "speech_setup_popover_unsupported_body": "Speech recognition can’t run on this computer.", "studio_artifact_failed_toast": "{name} failed", "studio_artifact_failed_toast_body": "This artifact couldn’t be generated. Retry it from the artifacts tab.", "studio_artifact_list_clear_filter_button": "Clear filter", diff --git a/surfsense_local/frontend/translations/es.json b/surfsense_local/frontend/translations/es.json index 57286ebcd7..d17af6f3f0 100644 --- a/surfsense_local/frontend/translations/es.json +++ b/surfsense_local/frontend/translations/es.json @@ -449,6 +449,10 @@ "models_download_image_unexpected_error": "Se produjo un error inesperado", "models_download_image_unsupported_body": "Esta versión no incluye un entorno de ejecución de imagen local. Usa en su lugar un servidor de arriba.", "models_download_image_unsupported_title": "Los modelos de imagen no pueden ejecutarse en este equipo", + "models_download_speech_delete_aria": "Eliminar {model}", + "models_download_speech_load_error": "No se pudieron cargar los modelos de reconocimiento de voz", + "models_download_speech_request_error": "La solicitud falló", + "models_download_speech_unsupported_title": "Los modelos de reconocimiento de voz no pueden ejecutarse en este equipo", "models_download_video_unsupported_title": "Los modelos de video no pueden ejecutarse en este equipo", "models_error_install_checksum_mismatch": "El archivo descargado no coincide con el esperado. Reintenta la descarga.", "models_error_install_failed": "No se pudo instalar el modelo. Reintenta la descarga.", @@ -479,8 +483,8 @@ "models_image_server_running_status": "En ejecución", "models_in_use_local_label": "Este equipo", "models_in_use_missing_label": "No se encuentra en este equipo", - "models_in_use_summary_aria": "{slot, select, chat {modelo de chat en uso} image {modelo de imagen en uso} image_edit {modelo de edición de imágenes en uso} video {modelo de video en uso} audio {modelo de audio en uso} other {modelo en uso}}", - "models_in_use_summary_empty": "{slot, select, chat {Sin modelo de chat elegido} image {Sin modelo de imagen elegido} image_edit {Sin modelo de edición de imágenes elegido} video {Sin modelo de video elegido} audio {Sin modelo de audio elegido} other {Sin modelo elegido}}", + "models_in_use_summary_aria": "{slot, select, chat {modelo de chat en uso} image {modelo de imagen en uso} image_edit {modelo de edición de imágenes en uso} video {modelo de video en uso} audio {modelo de audio en uso} speech {modelo de reconocimiento de voz en uso} other {modelo en uso}}", + "models_in_use_summary_empty": "{slot, select, chat {Sin modelo de chat elegido} image {Sin modelo de imagen elegido} image_edit {Sin modelo de edición de imágenes elegido} video {Sin modelo de video elegido} audio {Sin modelo de audio elegido} speech {Sin modelo de reconocimiento de voz elegido} other {Sin modelo elegido}}", "models_in_use_summary_label": "En uso:", "models_in_use_unknown_server_label": "Un servidor", "models_install_cancelled_short_status": "Cancelado", @@ -600,6 +604,7 @@ "models_server_models_type_audio_gen_label": "Generación de audio", "models_server_models_type_image_edit_label": "Edición de imágenes", "models_server_models_type_image_gen_label": "Generación de imágenes", + "models_server_models_type_speech_to_text_label": "Reconocimiento de voz", "models_server_models_type_text_gen_label": "Generación de texto", "models_server_models_type_video_gen_label": "Generación de video", "models_server_models_use_aria": "Usar {model}", @@ -619,6 +624,14 @@ "models_server_voices_trial_body": "Hola. Así sonarán tus podcasts.", "models_server_voices_voice_aria": "ID de voz", "models_server_voices_voice_placeholder": "ID de voz", + "models_speech_language_auto_label": "Detectar automáticamente", + "models_speech_language_body": "El idioma de tus grabaciones y de tu entrada de voz.", + "models_speech_language_request_error": "La solicitud falló", + "models_speech_language_title": "Idioma hablado", + "models_speech_model_detects_language_label": "Detecta el idioma", + "models_speech_model_languages_button": "{count, plural, one {# idioma} many {# idiomas} other {# idiomas}}", + "models_speech_model_peak_memory_gigabytes_label": "{size, number, ::unit/gigabyte .#} al transcribir", + "models_speech_model_peak_memory_megabytes_label": "{size, number, ::unit/megabyte .} al transcribir", "models_try_dialog_cancel_button": "Cancelar", "models_try_dialog_chat_confirmed_body": "La compatibilidad con chat está confirmada. La prueba envía un prompt corto y puede costar dinero.", "models_try_dialog_chat_unconfirmed_body": "Este endpoint no publica sus capacidades, así que la compatibilidad con chat no está confirmada. La prueba envía un prompt corto y puede costar dinero.", @@ -974,10 +987,13 @@ "settings_image_models_title": "Modelos de generación de imágenes", "settings_models_add_button": "Añadir modelo", "settings_models_add_page_body": "Ejecuta uno en este equipo o usa uno de un servidor que ya tengas en marcha.", - "settings_models_add_page_title": "{slot, select, audio {Añadir un modelo de audio} chat {Añadir un modelo de chat} image {Añadir un modelo de imagen} image_edit {Añadir un modelo de edición de imágenes} video {Añadir un modelo de video} other {Añadir un modelo}}", - "settings_models_empty": "{slot, select, audio {Todavía no hay modelo de audio} chat {Todavía no hay modelo de chat} image {Todavía no hay modelo de imagen} image_edit {Todavía no hay modelo de edición de imágenes} video {Todavía no hay modelo de video} other {Todavía no hay modelo}}", + "settings_models_add_page_local_body": "Descarga uno para ejecutarlo en este equipo.", + "settings_models_add_page_title": "{slot, select, audio {Añadir un modelo de audio} chat {Añadir un modelo de chat} image {Añadir un modelo de imagen} image_edit {Añadir un modelo de edición de imágenes} video {Añadir un modelo de video} speech {Añadir un modelo de reconocimiento de voz} other {Añadir un modelo}}", + "settings_models_empty": "{slot, select, audio {Todavía no hay modelo de audio} chat {Todavía no hay modelo de chat} image {Todavía no hay modelo de imagen} image_edit {Todavía no hay modelo de edición de imágenes} video {Todavía no hay modelo de video} speech {Todavía no hay modelo de reconocimiento de voz} other {Todavía no hay modelo}}", "settings_models_empty_download_body": "Descarga uno para ejecutarlo en este equipo o usa uno de un servidor que ya tengas en marcha.", + "settings_models_empty_local_body": "Descarga uno para ejecutarlo en este equipo.", "settings_models_empty_server_body": "Usa uno de un servidor que ya tengas en marcha.", + "settings_models_empty_unsupported_body": "Este equipo no puede ejecutar ninguno.", "settings_models_load_error": "No se pudieron cargar los ajustes del modelo", "settings_nav_about_label": "Acerca de", "settings_nav_app_title": "App", @@ -994,9 +1010,12 @@ "settings_nav_plugins_label": "Plugins", "settings_nav_report_issue_label": "Reportar error", "settings_nav_resources_label": "Recursos", + "settings_nav_speech_label": "Reconoc. de voz", "settings_nav_system_title": "Sistema", "settings_nav_title": "Ajustes", "settings_nav_video_label": "Video", + "settings_speech_models_body": "El modelo que convierte tus grabaciones y tu entrada de voz en texto, en este equipo.", + "settings_speech_models_title": "Modelos de reconocimiento de voz", "settings_video_models_body": "El modelo que crea videos. Ejecuta uno en este equipo o usa uno de un servidor.", "settings_video_models_title": "Modelos de generación de video", "source_preview_panel_aria": "Vista previa de la fuente", @@ -1112,6 +1131,17 @@ "sources_upload_too_large_reason_label": "Más de {size, number, ::unit/megabyte}", "sources_upload_unsupported_body": "No compatibles: {files}", "sources_upload_unsupported_error": "Tipo de archivo no compatible: {files}", + "speech_setup_card_download_button": "Descargar y configurar", + "speech_setup_card_languages_body": "Funciona bien en: {languages}", + "speech_setup_card_use_button": "Configurar", + "speech_setup_popover_body": "SurfSense convierte tu voz en texto en este equipo. Necesita una única descarga.", + "speech_setup_popover_done_button": "Listo", + "speech_setup_popover_load_error": "No se pudieron cargar los modelos de reconocimiento de voz.", + "speech_setup_popover_more_button": "Más opciones", + "speech_setup_popover_ready_title": "El reconocimiento de voz está listo", + "speech_setup_popover_start_button": "Empezar a hablar", + "speech_setup_popover_title": "Configurar el reconocimiento de voz", + "speech_setup_popover_unsupported_body": "El reconocimiento de voz no puede ejecutarse en este equipo.", "studio_artifact_failed_toast": "{name} falló", "studio_artifact_failed_toast_body": "No se pudo generar este artefacto. Reinténtalo desde la pestaña de artefactos.", "studio_artifact_list_clear_filter_button": "Quitar el filtro", diff --git a/surfsense_local/frontend/translations/fr.json b/surfsense_local/frontend/translations/fr.json index f631027ff0..def6e54d05 100644 --- a/surfsense_local/frontend/translations/fr.json +++ b/surfsense_local/frontend/translations/fr.json @@ -449,6 +449,10 @@ "models_download_image_unexpected_error": "Une erreur inattendue s’est produite", "models_download_image_unsupported_body": "Cette version n’a pas de moteur d’exécution d’image local. Utilisez plutôt un serveur ci-dessus.", "models_download_image_unsupported_title": "Les modèles d’image ne peuvent pas fonctionner sur cet ordinateur", + "models_download_speech_delete_aria": "Supprimer {model}", + "models_download_speech_load_error": "Impossible de charger les modèles de reconnaissance vocale", + "models_download_speech_request_error": "La requête a échoué", + "models_download_speech_unsupported_title": "Les modèles de reconnaissance vocale ne peuvent pas fonctionner sur cet ordinateur", "models_download_video_unsupported_title": "Les modèles vidéo ne peuvent pas fonctionner sur cet ordinateur", "models_error_install_checksum_mismatch": "Le fichier téléchargé ne correspond pas à celui attendu. Relancez le téléchargement.", "models_error_install_failed": "Impossible d’installer le modèle. Relancez le téléchargement.", @@ -479,8 +483,8 @@ "models_image_server_running_status": "En cours d’exécution", "models_in_use_local_label": "Cet ordinateur", "models_in_use_missing_label": "Introuvable sur cet ordinateur", - "models_in_use_summary_aria": "{slot, select, chat {modèle de chat actif} image {modèle d’image actif} image_edit {modèle de modification d’images actif} video {modèle vidéo actif} audio {modèle audio actif} other {modèle actif}}", - "models_in_use_summary_empty": "{slot, select, chat {Aucun modèle de chat choisi} image {Aucun modèle d’image choisi} image_edit {Aucun modèle de modification d’images choisi} video {Aucun modèle vidéo choisi} audio {Aucun modèle audio choisi} other {Aucun modèle choisi}}", + "models_in_use_summary_aria": "{slot, select, chat {modèle de chat actif} image {modèle d’image actif} image_edit {modèle de modification d’images actif} video {modèle vidéo actif} audio {modèle audio actif} speech {modèle de reconnaissance vocale actif} other {modèle actif}}", + "models_in_use_summary_empty": "{slot, select, chat {Aucun modèle de chat choisi} image {Aucun modèle d’image choisi} image_edit {Aucun modèle de modification d’images choisi} video {Aucun modèle vidéo choisi} audio {Aucun modèle audio choisi} speech {Aucun modèle de reconnaissance vocale choisi} other {Aucun modèle choisi}}", "models_in_use_summary_label": "Actif :", "models_in_use_unknown_server_label": "Un serveur", "models_install_cancelled_short_status": "Annulé", @@ -600,6 +604,7 @@ "models_server_models_type_audio_gen_label": "Génération audio", "models_server_models_type_image_edit_label": "Modification d’images", "models_server_models_type_image_gen_label": "Génération d’images", + "models_server_models_type_speech_to_text_label": "Reconnaissance vocale", "models_server_models_type_text_gen_label": "Génération de texte", "models_server_models_type_video_gen_label": "Génération de vidéos", "models_server_models_use_aria": "Utiliser {model}", @@ -619,6 +624,14 @@ "models_server_voices_trial_body": "Bonjour. Voici comment vos podcasts sonneront.", "models_server_voices_voice_aria": "ID de voix", "models_server_voices_voice_placeholder": "ID de voix", + "models_speech_language_auto_label": "Détecter automatiquement", + "models_speech_language_body": "La langue de vos enregistrements et de votre saisie vocale.", + "models_speech_language_request_error": "La requête a échoué", + "models_speech_language_title": "Langue parlée", + "models_speech_model_detects_language_label": "Détecte la langue", + "models_speech_model_languages_button": "{count, plural, one {# langue} many {# langues} other {# langues}}", + "models_speech_model_peak_memory_gigabytes_label": "{size, number, ::unit/gigabyte .#} pendant la transcription", + "models_speech_model_peak_memory_megabytes_label": "{size, number, ::unit/megabyte .} pendant la transcription", "models_try_dialog_cancel_button": "Annuler", "models_try_dialog_chat_confirmed_body": "La prise en charge du chat est confirmée. Le test envoie un prompt court et peut être facturé.", "models_try_dialog_chat_unconfirmed_body": "Cet endpoint ne publie pas ses capacités, la prise en charge du chat n’est donc pas confirmée. Le test envoie un prompt court et peut être facturé.", @@ -974,10 +987,13 @@ "settings_image_models_title": "Modèles de génération d’images", "settings_models_add_button": "Ajouter un modèle", "settings_models_add_page_body": "Exécutez-en un sur cet ordinateur ou utilisez-en un depuis un serveur que vous exploitez déjà.", - "settings_models_add_page_title": "{slot, select, audio {Ajouter un modèle audio} chat {Ajouter un modèle de chat} image {Ajouter un modèle d’image} image_edit {Ajouter un modèle de modification d’images} video {Ajouter un modèle vidéo} other {Ajouter un modèle}}", - "settings_models_empty": "{slot, select, audio {Aucun modèle audio pour l’instant} chat {Aucun modèle de chat pour l’instant} image {Aucun modèle d’image pour l’instant} image_edit {Aucun modèle de modification d’images pour l’instant} video {Aucun modèle vidéo pour l’instant} other {Aucun modèle pour l’instant}}", + "settings_models_add_page_local_body": "Téléchargez-en un pour l’exécuter sur cet ordinateur.", + "settings_models_add_page_title": "{slot, select, audio {Ajouter un modèle audio} chat {Ajouter un modèle de chat} image {Ajouter un modèle d’image} image_edit {Ajouter un modèle de modification d’images} video {Ajouter un modèle vidéo} speech {Ajouter un modèle de reconnaissance vocale} other {Ajouter un modèle}}", + "settings_models_empty": "{slot, select, audio {Aucun modèle audio pour l’instant} chat {Aucun modèle de chat pour l’instant} image {Aucun modèle d’image pour l’instant} image_edit {Aucun modèle de modification d’images pour l’instant} video {Aucun modèle vidéo pour l’instant} speech {Aucun modèle de reconnaissance vocale pour l’instant} other {Aucun modèle pour l’instant}}", "settings_models_empty_download_body": "Téléchargez-en un pour l’exécuter sur cet ordinateur ou utilisez-en un depuis un serveur que vous exploitez déjà.", + "settings_models_empty_local_body": "Téléchargez-en un pour l’exécuter sur cet ordinateur.", "settings_models_empty_server_body": "Utilisez-en un depuis un serveur que vous exploitez déjà.", + "settings_models_empty_unsupported_body": "Cet ordinateur ne peut en exécuter aucun.", "settings_models_load_error": "Impossible de charger les paramètres des modèles", "settings_nav_about_label": "À propos", "settings_nav_app_title": "Application", @@ -994,9 +1010,12 @@ "settings_nav_plugins_label": "Plugins", "settings_nav_report_issue_label": "Signaler un bug", "settings_nav_resources_label": "Ressources", + "settings_nav_speech_label": "Reconn. vocale", "settings_nav_system_title": "Système", "settings_nav_title": "Paramètres", "settings_nav_video_label": "Vidéo", + "settings_speech_models_body": "Le modèle qui transforme vos enregistrements et votre saisie vocale en texte, sur cet ordinateur.", + "settings_speech_models_title": "Modèles de reconnaissance vocale", "settings_video_models_body": "Le modèle qui crée les vidéos. Exécutez-en un sur cet ordinateur ou utilisez-en un depuis un serveur.", "settings_video_models_title": "Modèles de génération de vidéos", "source_preview_panel_aria": "Aperçu de la source", @@ -1112,6 +1131,17 @@ "sources_upload_too_large_reason_label": "Plus de {size, number, ::unit/megabyte}", "sources_upload_unsupported_body": "Non pris en charge : {files}", "sources_upload_unsupported_error": "Type de fichier non pris en charge : {files}", + "speech_setup_card_download_button": "Télécharger et configurer", + "speech_setup_card_languages_body": "Fonctionne bien en : {languages}", + "speech_setup_card_use_button": "Configurer", + "speech_setup_popover_body": "SurfSense transforme votre voix en texte sur cet ordinateur. Un téléchargement unique est nécessaire.", + "speech_setup_popover_done_button": "Terminé", + "speech_setup_popover_load_error": "Impossible de charger les modèles de reconnaissance vocale.", + "speech_setup_popover_more_button": "Plus de choix", + "speech_setup_popover_ready_title": "La reconnaissance vocale est prête", + "speech_setup_popover_start_button": "Commencer à parler", + "speech_setup_popover_title": "Configurer la reconnaissance vocale", + "speech_setup_popover_unsupported_body": "La reconnaissance vocale ne peut pas fonctionner sur cet ordinateur.", "studio_artifact_failed_toast": "Échec de {name}", "studio_artifact_failed_toast_body": "Cet artefact n’a pas pu être généré. Réessayez depuis l’onglet Artefacts.", "studio_artifact_list_clear_filter_button": "Effacer le filtre", diff --git a/surfsense_local/frontend/translations/hi.json b/surfsense_local/frontend/translations/hi.json index c4d2bb2eb2..348b649419 100644 --- a/surfsense_local/frontend/translations/hi.json +++ b/surfsense_local/frontend/translations/hi.json @@ -449,6 +449,10 @@ "models_download_image_unexpected_error": "एक अनपेक्षित त्रुटि हुई", "models_download_image_unsupported_body": "इस वर्शन में लोकल इमेज रनटाइम नहीं है। इसके बजाय ऊपर दिया कोई सर्वर इस्तेमाल करें।", "models_download_image_unsupported_title": "इमेज मॉडल इस कंप्यूटर पर नहीं चल सकते", + "models_download_speech_delete_aria": "{model} हटाएं", + "models_download_speech_load_error": "स्पीच रिकग्निशन मॉडल लोड नहीं हो सके", + "models_download_speech_request_error": "अनुरोध विफल रहा", + "models_download_speech_unsupported_title": "स्पीच रिकग्निशन मॉडल इस कंप्यूटर पर नहीं चल सकते", "models_download_video_unsupported_title": "वीडियो मॉडल इस कंप्यूटर पर नहीं चल सकते", "models_error_install_checksum_mismatch": "डाउनलोड की गई फ़ाइल अपेक्षित फ़ाइल से मेल नहीं खाती। डाउनलोड की फिर से कोशिश करें।", "models_error_install_failed": "मॉडल इंस्टॉल नहीं हो सका। डाउनलोड की फिर से कोशिश करें।", @@ -479,8 +483,8 @@ "models_image_server_running_status": "चल रहा है", "models_in_use_local_label": "यह कंप्यूटर", "models_in_use_missing_label": "इस कंप्यूटर पर नहीं मिला", - "models_in_use_summary_aria": "{slot, select, chat {उपयोग में चैट मॉडल} image {उपयोग में इमेज मॉडल} image_edit {उपयोग में इमेज एडिटिंग मॉडल} video {उपयोग में वीडियो मॉडल} audio {उपयोग में ऑडियो मॉडल} other {उपयोग में मॉडल}}", - "models_in_use_summary_empty": "{slot, select, chat {कोई चैट मॉडल नहीं चुना गया} image {कोई इमेज मॉडल नहीं चुना गया} image_edit {कोई इमेज एडिटिंग मॉडल नहीं चुना गया} video {कोई वीडियो मॉडल नहीं चुना गया} audio {कोई ऑडियो मॉडल नहीं चुना गया} other {कोई मॉडल नहीं चुना गया}}", + "models_in_use_summary_aria": "{slot, select, chat {उपयोग में चैट मॉडल} image {उपयोग में इमेज मॉडल} image_edit {उपयोग में इमेज एडिटिंग मॉडल} video {उपयोग में वीडियो मॉडल} audio {उपयोग में ऑडियो मॉडल} speech {उपयोग में स्पीच रिकग्निशन मॉडल} other {उपयोग में मॉडल}}", + "models_in_use_summary_empty": "{slot, select, chat {कोई चैट मॉडल नहीं चुना गया} image {कोई इमेज मॉडल नहीं चुना गया} image_edit {कोई इमेज एडिटिंग मॉडल नहीं चुना गया} video {कोई वीडियो मॉडल नहीं चुना गया} audio {कोई ऑडियो मॉडल नहीं चुना गया} speech {कोई स्पीच रिकग्निशन मॉडल नहीं चुना गया} other {कोई मॉडल नहीं चुना गया}}", "models_in_use_summary_label": "उपयोग में:", "models_in_use_unknown_server_label": "एक सर्वर", "models_install_cancelled_short_status": "रद्द किया गया", @@ -600,6 +604,7 @@ "models_server_models_type_audio_gen_label": "ऑडियो जनरेशन", "models_server_models_type_image_edit_label": "इमेज एडिटिंग", "models_server_models_type_image_gen_label": "इमेज जनरेशन", + "models_server_models_type_speech_to_text_label": "स्पीच रिकग्निशन", "models_server_models_type_text_gen_label": "टेक्स्ट जनरेशन", "models_server_models_type_video_gen_label": "वीडियो जनरेशन", "models_server_models_use_aria": "{model} इस्तेमाल करें", @@ -619,6 +624,14 @@ "models_server_voices_trial_body": "नमस्ते। आपके पॉडकास्ट ऐसे सुनाई देंगे।", "models_server_voices_voice_aria": "आवाज़ ID", "models_server_voices_voice_placeholder": "आवाज़ ID", + "models_speech_language_auto_label": "अपने-आप पहचानें", + "models_speech_language_body": "आपकी रिकॉर्डिंग और वॉइस इनपुट की भाषा।", + "models_speech_language_request_error": "अनुरोध विफल रहा", + "models_speech_language_title": "बोली जाने वाली भाषा", + "models_speech_model_detects_language_label": "भाषा पहचानता है", + "models_speech_model_languages_button": "{count, plural, one {# भाषा} other {# भाषाएं}}", + "models_speech_model_peak_memory_gigabytes_label": "ट्रांसक्राइब करते समय {size, number, ::unit/gigabyte .#}", + "models_speech_model_peak_memory_megabytes_label": "ट्रांसक्राइब करते समय {size, number, ::unit/megabyte .}", "models_try_dialog_cancel_button": "रद्द करें", "models_try_dialog_chat_confirmed_body": "चैट सपोर्ट की पुष्टि हो चुकी है। जांच में एक छोटा प्रॉम्प्ट भेजा जाता है, जिसका शुल्क लग सकता है।", "models_try_dialog_chat_unconfirmed_body": "यह एंडपॉइंट अपनी क्षमताएं प्रकाशित नहीं करता, इसलिए चैट सपोर्ट की पुष्टि नहीं हुई है। जांच में एक छोटा प्रॉम्प्ट भेजा जाता है, जिसका शुल्क लग सकता है।", @@ -974,10 +987,13 @@ "settings_image_models_title": "इमेज जनरेशन मॉडल", "settings_models_add_button": "मॉडल जोड़ें", "settings_models_add_page_body": "इसे इस कंप्यूटर पर चलाएं, या पहले से चल रहे किसी सर्वर से इस्तेमाल करें।", - "settings_models_add_page_title": "{slot, select, audio {ऑडियो मॉडल जोड़ें} chat {चैट मॉडल जोड़ें} image {इमेज मॉडल जोड़ें} image_edit {इमेज एडिटिंग मॉडल जोड़ें} video {वीडियो मॉडल जोड़ें} other {मॉडल जोड़ें}}", - "settings_models_empty": "{slot, select, audio {अभी कोई ऑडियो मॉडल नहीं} chat {अभी कोई चैट मॉडल नहीं} image {अभी कोई इमेज मॉडल नहीं} image_edit {अभी कोई इमेज एडिटिंग मॉडल नहीं} video {अभी कोई वीडियो मॉडल नहीं} other {अभी कोई मॉडल नहीं}}", + "settings_models_add_page_local_body": "इस कंप्यूटर पर चलाने के लिए कोई डाउनलोड करें।", + "settings_models_add_page_title": "{slot, select, audio {ऑडियो मॉडल जोड़ें} chat {चैट मॉडल जोड़ें} image {इमेज मॉडल जोड़ें} image_edit {इमेज एडिटिंग मॉडल जोड़ें} video {वीडियो मॉडल जोड़ें} speech {स्पीच रिकग्निशन मॉडल जोड़ें} other {मॉडल जोड़ें}}", + "settings_models_empty": "{slot, select, audio {अभी कोई ऑडियो मॉडल नहीं} chat {अभी कोई चैट मॉडल नहीं} image {अभी कोई इमेज मॉडल नहीं} image_edit {अभी कोई इमेज एडिटिंग मॉडल नहीं} video {अभी कोई वीडियो मॉडल नहीं} speech {अभी कोई स्पीच रिकग्निशन मॉडल नहीं} other {अभी कोई मॉडल नहीं}}", "settings_models_empty_download_body": "इस कंप्यूटर पर चलाने के लिए कोई डाउनलोड करें, या पहले से चल रहे किसी सर्वर से इस्तेमाल करें।", + "settings_models_empty_local_body": "इस कंप्यूटर पर चलाने के लिए कोई डाउनलोड करें।", "settings_models_empty_server_body": "पहले से चल रहे किसी सर्वर से इस्तेमाल करें।", + "settings_models_empty_unsupported_body": "यह कंप्यूटर कोई नहीं चला सकता।", "settings_models_load_error": "मॉडल सेटिंग्स लोड नहीं हो सकीं", "settings_nav_about_label": "जानकारी", "settings_nav_app_title": "ऐप", @@ -994,9 +1010,12 @@ "settings_nav_plugins_label": "प्लगइन", "settings_nav_report_issue_label": "समस्या रिपोर्ट करें", "settings_nav_resources_label": "संसाधन", + "settings_nav_speech_label": "स्पीच रिकग्निशन", "settings_nav_system_title": "सिस्टम", "settings_nav_title": "सेटिंग्स", "settings_nav_video_label": "वीडियो", + "settings_speech_models_body": "वह मॉडल जो इस कंप्यूटर पर रिकॉर्डिंग और वॉइस इनपुट को टेक्स्ट में बदलता है।", + "settings_speech_models_title": "स्पीच रिकग्निशन मॉडल", "settings_video_models_body": "वह मॉडल जो वीडियो बनाता है। इसे इस कंप्यूटर पर चलाएं, या किसी सर्वर से इस्तेमाल करें।", "settings_video_models_title": "वीडियो जनरेशन मॉडल", "source_preview_panel_aria": "स्रोत का पूर्वावलोकन", @@ -1112,6 +1131,17 @@ "sources_upload_too_large_reason_label": "{size, number, ::unit/megabyte} से बड़ी", "sources_upload_unsupported_body": "समर्थित नहीं: {files}", "sources_upload_unsupported_error": "असमर्थित फ़ाइल प्रकार: {files}", + "speech_setup_card_download_button": "डाउनलोड और सेट अप करें", + "speech_setup_card_languages_body": "इनमें अच्छा काम करता है: {languages}", + "speech_setup_card_use_button": "सेट अप करें", + "speech_setup_popover_body": "SurfSense इस कंप्यूटर पर आपकी आवाज़ को टेक्स्ट में बदलता है। इसके लिए एक बार डाउनलोड करना होगा।", + "speech_setup_popover_done_button": "हो गया", + "speech_setup_popover_load_error": "स्पीच रिकग्निशन मॉडल लोड नहीं हो सके।", + "speech_setup_popover_more_button": "और विकल्प", + "speech_setup_popover_ready_title": "स्पीच रिकग्निशन तैयार है", + "speech_setup_popover_start_button": "बोलना शुरू करें", + "speech_setup_popover_title": "स्पीच रिकग्निशन सेट अप करें", + "speech_setup_popover_unsupported_body": "स्पीच रिकग्निशन इस कंप्यूटर पर नहीं चल सकता।", "studio_artifact_failed_toast": "{name} विफल रहा", "studio_artifact_failed_toast_body": "यह आर्टिफ़ैक्ट जनरेट नहीं हो सका। आर्टिफ़ैक्ट टैब से फिर कोशिश करें।", "studio_artifact_list_clear_filter_button": "फ़िल्टर हटाएं", diff --git a/surfsense_local/frontend/translations/ja.json b/surfsense_local/frontend/translations/ja.json index 0554be99ff..330479a0e8 100644 --- a/surfsense_local/frontend/translations/ja.json +++ b/surfsense_local/frontend/translations/ja.json @@ -449,6 +449,10 @@ "models_download_image_unexpected_error": "予期しないエラーが発生しました", "models_download_image_unsupported_body": "このビルドにはローカル画像ランタイムがありません。代わりに上のサーバーを使用してください。", "models_download_image_unsupported_title": "このコンピューターでは画像モデルを実行できません", + "models_download_speech_delete_aria": "{model}を削除", + "models_download_speech_load_error": "音声認識モデルを読み込めませんでした", + "models_download_speech_request_error": "リクエストに失敗しました", + "models_download_speech_unsupported_title": "このコンピューターでは音声認識モデルを実行できません", "models_download_video_unsupported_title": "このコンピューターでは動画モデルを実行できません", "models_error_install_checksum_mismatch": "ダウンロードしたファイルが想定のファイルと一致しませんでした。ダウンロードを再試行してください。", "models_error_install_failed": "モデルをインストールできませんでした。ダウンロードを再試行してください。", @@ -479,8 +483,8 @@ "models_image_server_running_status": "実行中", "models_in_use_local_label": "このコンピューター", "models_in_use_missing_label": "このコンピューターに見つかりません", - "models_in_use_summary_aria": "{slot, select, chat {使用中のチャットモデル} image {使用中の画像モデル} image_edit {使用中の画像編集モデル} video {使用中の動画モデル} audio {使用中の音声モデル} other {使用中のモデル}}", - "models_in_use_summary_empty": "{slot, select, chat {チャットモデルが選択されていません} image {画像モデルが選択されていません} image_edit {画像編集モデルが選択されていません} video {動画モデルが選択されていません} audio {音声モデルが選択されていません} other {モデルが選択されていません}}", + "models_in_use_summary_aria": "{slot, select, chat {使用中のチャットモデル} image {使用中の画像モデル} image_edit {使用中の画像編集モデル} video {使用中の動画モデル} audio {使用中の音声モデル} speech {使用中の音声認識モデル} other {使用中のモデル}}", + "models_in_use_summary_empty": "{slot, select, chat {チャットモデルが選択されていません} image {画像モデルが選択されていません} image_edit {画像編集モデルが選択されていません} video {動画モデルが選択されていません} audio {音声モデルが選択されていません} speech {音声認識モデルが選択されていません} other {モデルが選択されていません}}", "models_in_use_summary_label": "使用中:", "models_in_use_unknown_server_label": "サーバー", "models_install_cancelled_short_status": "キャンセル済み", @@ -600,6 +604,7 @@ "models_server_models_type_audio_gen_label": "音声生成", "models_server_models_type_image_edit_label": "画像編集", "models_server_models_type_image_gen_label": "画像生成", + "models_server_models_type_speech_to_text_label": "音声認識", "models_server_models_type_text_gen_label": "テキスト生成", "models_server_models_type_video_gen_label": "動画生成", "models_server_models_use_aria": "{model}を使用", @@ -619,6 +624,14 @@ "models_server_voices_trial_body": "こんにちは。ポッドキャストはこのように聞こえます。", "models_server_voices_voice_aria": "声のID", "models_server_voices_voice_placeholder": "声のID", + "models_speech_language_auto_label": "自動検出", + "models_speech_language_body": "録音と音声入力の言語です。", + "models_speech_language_request_error": "リクエストに失敗しました", + "models_speech_language_title": "話す言語", + "models_speech_model_detects_language_label": "言語を自動検出", + "models_speech_model_languages_button": "{count, plural, other {#言語}}", + "models_speech_model_peak_memory_gigabytes_label": "文字起こし時{size, number, ::unit/gigabyte .#}", + "models_speech_model_peak_memory_megabytes_label": "文字起こし時{size, number, ::unit/megabyte .}", "models_try_dialog_cancel_button": "キャンセル", "models_try_dialog_chat_confirmed_body": "チャットへの対応は確認済みです。テストでは短いプロンプトを1回送信するため、料金が発生する場合があります。", "models_try_dialog_chat_unconfirmed_body": "このエンドポイントは機能を公開していないため、チャットへの対応は未確認です。テストでは短いプロンプトを1回送信するため、料金が発生する場合があります。", @@ -974,10 +987,13 @@ "settings_image_models_title": "画像生成モデル", "settings_models_add_button": "モデルを追加", "settings_models_add_page_body": "このコンピューターで実行するか、すでに運用しているサーバー上のモデルを使います。", - "settings_models_add_page_title": "{slot, select, audio {音声モデルを追加} chat {チャットモデルを追加} image {画像モデルを追加} image_edit {画像編集モデルを追加} video {動画モデルを追加} other {モデルを追加}}", - "settings_models_empty": "{slot, select, audio {音声モデルはまだありません} chat {チャットモデルはまだありません} image {画像モデルはまだありません} image_edit {画像編集モデルはまだありません} video {動画モデルはまだありません} other {モデルはまだありません}}", + "settings_models_add_page_local_body": "ダウンロードしてこのコンピューターで実行します。", + "settings_models_add_page_title": "{slot, select, audio {音声モデルを追加} chat {チャットモデルを追加} image {画像モデルを追加} image_edit {画像編集モデルを追加} video {動画モデルを追加} speech {音声認識モデルを追加} other {モデルを追加}}", + "settings_models_empty": "{slot, select, audio {音声モデルはまだありません} chat {チャットモデルはまだありません} image {画像モデルはまだありません} image_edit {画像編集モデルはまだありません} video {動画モデルはまだありません} speech {音声認識モデルはまだありません} other {モデルはまだありません}}", "settings_models_empty_download_body": "ダウンロードしてこのコンピューターで実行するか、すでに運用しているサーバー上のモデルを使います。", + "settings_models_empty_local_body": "ダウンロードしてこのコンピューターで実行します。", "settings_models_empty_server_body": "すでに運用しているサーバー上のモデルを使います。", + "settings_models_empty_unsupported_body": "このコンピューターでは実行できません。", "settings_models_load_error": "モデル設定を読み込めませんでした", "settings_nav_about_label": "情報", "settings_nav_app_title": "アプリ", @@ -994,9 +1010,12 @@ "settings_nav_plugins_label": "プラグイン", "settings_nav_report_issue_label": "問題を報告", "settings_nav_resources_label": "リソース", + "settings_nav_speech_label": "音声認識", "settings_nav_system_title": "システム", "settings_nav_title": "設定", "settings_nav_video_label": "動画", + "settings_speech_models_body": "このコンピューター上で、録音や音声入力をテキストに変換するモデルです。", + "settings_speech_models_title": "音声認識モデル", "settings_video_models_body": "動画を生成するモデルです。このコンピューターで実行するか、サーバー上のモデルを使います。", "settings_video_models_title": "動画生成モデル", "source_preview_panel_aria": "ソースのプレビュー", @@ -1112,6 +1131,17 @@ "sources_upload_too_large_reason_label": "{size, number, ::unit/megabyte}超", "sources_upload_unsupported_body": "非対応: {files}", "sources_upload_unsupported_error": "非対応のファイル形式: {files}", + "speech_setup_card_download_button": "ダウンロードして設定", + "speech_setup_card_languages_body": "得意な言語：{languages}", + "speech_setup_card_use_button": "設定", + "speech_setup_popover_body": "SurfSenseは、このコンピューター上で声をテキストに変換します。初回のみダウンロードが必要です。", + "speech_setup_popover_done_button": "完了", + "speech_setup_popover_load_error": "音声認識モデルを読み込めませんでした。", + "speech_setup_popover_more_button": "その他の選択肢", + "speech_setup_popover_ready_title": "音声認識の準備ができました", + "speech_setup_popover_start_button": "話し始める", + "speech_setup_popover_title": "音声認識を設定", + "speech_setup_popover_unsupported_body": "このコンピューターでは音声認識を実行できません。", "studio_artifact_failed_toast": "{name}の生成に失敗しました", "studio_artifact_failed_toast_body": "この生成物を生成できませんでした。生成物タブから再試行してください。", "studio_artifact_list_clear_filter_button": "フィルターを解除", diff --git a/surfsense_local/frontend/translations/ko.json b/surfsense_local/frontend/translations/ko.json index 533f2c793a..11a352f73f 100644 --- a/surfsense_local/frontend/translations/ko.json +++ b/surfsense_local/frontend/translations/ko.json @@ -449,6 +449,10 @@ "models_download_image_unexpected_error": "예기치 않은 오류가 발생했습니다", "models_download_image_unsupported_body": "이 빌드에는 로컬 이미지 런타임이 없습니다. 대신 위의 서버를 사용하세요.", "models_download_image_unsupported_title": "이 컴퓨터에서는 이미지 모델을 실행할 수 없습니다", + "models_download_speech_delete_aria": "{model} 삭제", + "models_download_speech_load_error": "음성 인식 모델을 불러오지 못했습니다", + "models_download_speech_request_error": "요청이 실패했습니다", + "models_download_speech_unsupported_title": "이 컴퓨터에서는 음성 인식 모델을 실행할 수 없습니다", "models_download_video_unsupported_title": "이 컴퓨터에서는 동영상 모델을 실행할 수 없습니다", "models_error_install_checksum_mismatch": "다운로드한 파일이 예상한 파일과 일치하지 않습니다. 다운로드를 다시 시도하세요.", "models_error_install_failed": "모델을 설치하지 못했습니다. 다운로드를 다시 시도하세요.", @@ -479,8 +483,8 @@ "models_image_server_running_status": "실행 중", "models_in_use_local_label": "이 컴퓨터", "models_in_use_missing_label": "이 컴퓨터에서 찾을 수 없음", - "models_in_use_summary_aria": "{slot, select, chat {사용 중인 채팅 모델} image {사용 중인 이미지 모델} image_edit {사용 중인 이미지 편집 모델} video {사용 중인 동영상 모델} audio {사용 중인 음성 모델} other {사용 중인 모델}}", - "models_in_use_summary_empty": "{slot, select, chat {선택된 채팅 모델이 없습니다} image {선택된 이미지 모델이 없습니다} image_edit {선택된 이미지 편집 모델이 없습니다} video {선택된 동영상 모델이 없습니다} audio {선택된 음성 모델이 없습니다} other {선택된 모델이 없습니다}}", + "models_in_use_summary_aria": "{slot, select, chat {사용 중인 채팅 모델} image {사용 중인 이미지 모델} image_edit {사용 중인 이미지 편집 모델} video {사용 중인 동영상 모델} audio {사용 중인 음성 모델} speech {사용 중인 음성 인식 모델} other {사용 중인 모델}}", + "models_in_use_summary_empty": "{slot, select, chat {선택된 채팅 모델이 없습니다} image {선택된 이미지 모델이 없습니다} image_edit {선택된 이미지 편집 모델이 없습니다} video {선택된 동영상 모델이 없습니다} audio {선택된 음성 모델이 없습니다} speech {선택된 음성 인식 모델이 없습니다} other {선택된 모델이 없습니다}}", "models_in_use_summary_label": "사용 중:", "models_in_use_unknown_server_label": "서버", "models_install_cancelled_short_status": "취소됨", @@ -600,6 +604,7 @@ "models_server_models_type_audio_gen_label": "음성 생성", "models_server_models_type_image_edit_label": "이미지 편집", "models_server_models_type_image_gen_label": "이미지 생성", + "models_server_models_type_speech_to_text_label": "음성 인식", "models_server_models_type_text_gen_label": "텍스트 생성", "models_server_models_type_video_gen_label": "동영상 생성", "models_server_models_use_aria": "{model} 사용", @@ -619,6 +624,14 @@ "models_server_voices_trial_body": "안녕하세요. 팟캐스트는 이렇게 들립니다.", "models_server_voices_voice_aria": "목소리 ID", "models_server_voices_voice_placeholder": "목소리 ID", + "models_speech_language_auto_label": "자동 감지", + "models_speech_language_body": "녹음과 음성 입력의 언어입니다.", + "models_speech_language_request_error": "요청이 실패했습니다", + "models_speech_language_title": "말하는 언어", + "models_speech_model_detects_language_label": "언어 자동 감지", + "models_speech_model_languages_button": "{count, plural, other {#개 언어}}", + "models_speech_model_peak_memory_gigabytes_label": "음성 인식 시 {size, number, ::unit/gigabyte .#}", + "models_speech_model_peak_memory_megabytes_label": "음성 인식 시 {size, number, ::unit/megabyte .}", "models_try_dialog_cancel_button": "취소", "models_try_dialog_chat_confirmed_body": "채팅 지원이 확인되었습니다. 테스트는 짧은 프롬프트를 한 번 전송하며 비용이 발생할 수 있습니다.", "models_try_dialog_chat_unconfirmed_body": "이 엔드포인트는 기능을 공개하지 않으므로 채팅 지원 여부가 확인되지 않았습니다. 테스트는 짧은 프롬프트를 한 번 전송하며 비용이 발생할 수 있습니다.", @@ -974,10 +987,13 @@ "settings_image_models_title": "이미지 생성 모델", "settings_models_add_button": "모델 추가", "settings_models_add_page_body": "이 컴퓨터에서 실행하거나, 이미 운영 중인 서버의 모델을 사용하세요.", - "settings_models_add_page_title": "{slot, select, audio {음성 모델 추가} chat {채팅 모델 추가} image {이미지 모델 추가} image_edit {이미지 편집 모델 추가} video {동영상 모델 추가} other {모델 추가}}", - "settings_models_empty": "{slot, select, audio {아직 음성 모델이 없습니다} chat {아직 채팅 모델이 없습니다} image {아직 이미지 모델이 없습니다} image_edit {아직 이미지 편집 모델이 없습니다} video {아직 동영상 모델이 없습니다} other {아직 모델이 없습니다}}", + "settings_models_add_page_local_body": "다운로드해 이 컴퓨터에서 실행하세요.", + "settings_models_add_page_title": "{slot, select, audio {음성 모델 추가} chat {채팅 모델 추가} image {이미지 모델 추가} image_edit {이미지 편집 모델 추가} video {동영상 모델 추가} speech {음성 인식 모델 추가} other {모델 추가}}", + "settings_models_empty": "{slot, select, audio {아직 음성 모델이 없습니다} chat {아직 채팅 모델이 없습니다} image {아직 이미지 모델이 없습니다} image_edit {아직 이미지 편집 모델이 없습니다} video {아직 동영상 모델이 없습니다} speech {아직 음성 인식 모델이 없습니다} other {아직 모델이 없습니다}}", "settings_models_empty_download_body": "다운로드해 이 컴퓨터에서 실행하거나, 이미 운영 중인 서버의 모델을 사용하세요.", + "settings_models_empty_local_body": "다운로드해 이 컴퓨터에서 실행하세요.", "settings_models_empty_server_body": "이미 운영 중인 서버의 모델을 사용하세요.", + "settings_models_empty_unsupported_body": "이 컴퓨터에서는 실행할 수 없습니다.", "settings_models_load_error": "모델 설정을 불러오지 못했습니다", "settings_nav_about_label": "정보", "settings_nav_app_title": "앱", @@ -994,9 +1010,12 @@ "settings_nav_plugins_label": "플러그인", "settings_nav_report_issue_label": "문제 신고", "settings_nav_resources_label": "리소스", + "settings_nav_speech_label": "음성 인식", "settings_nav_system_title": "시스템", "settings_nav_title": "설정", "settings_nav_video_label": "동영상", + "settings_speech_models_body": "이 컴퓨터에서 녹음과 음성 입력을 텍스트로 변환하는 모델입니다.", + "settings_speech_models_title": "음성 인식 모델", "settings_video_models_body": "동영상을 만드는 모델입니다. 이 컴퓨터에서 실행하거나, 서버의 모델을 사용하세요.", "settings_video_models_title": "동영상 생성 모델", "source_preview_panel_aria": "소스 미리보기", @@ -1112,6 +1131,17 @@ "sources_upload_too_large_reason_label": "{size, number, ::unit/megabyte} 초과", "sources_upload_unsupported_body": "지원하지 않음: {files}", "sources_upload_unsupported_error": "지원하지 않는 파일 형식: {files}", + "speech_setup_card_download_button": "다운로드 및 설정", + "speech_setup_card_languages_body": "잘 지원하는 언어: {languages}", + "speech_setup_card_use_button": "설정", + "speech_setup_popover_body": "SurfSense는 이 컴퓨터에서 음성을 텍스트로 변환합니다. 한 번만 다운로드하면 됩니다.", + "speech_setup_popover_done_button": "완료", + "speech_setup_popover_load_error": "음성 인식 모델을 불러오지 못했습니다.", + "speech_setup_popover_more_button": "다른 선택지", + "speech_setup_popover_ready_title": "음성 인식 준비 완료", + "speech_setup_popover_start_button": "말하기 시작", + "speech_setup_popover_title": "음성 인식 설정", + "speech_setup_popover_unsupported_body": "이 컴퓨터에서는 음성 인식을 실행할 수 없습니다.", "studio_artifact_failed_toast": "{name} 실패", "studio_artifact_failed_toast_body": "이 생성물을 생성하지 못했습니다. 생성물 탭에서 다시 시도하세요.", "studio_artifact_list_clear_filter_button": "필터 해제", diff --git a/surfsense_local/frontend/translations/pt-BR.json b/surfsense_local/frontend/translations/pt-BR.json index a2ef6577b9..908204ab58 100644 --- a/surfsense_local/frontend/translations/pt-BR.json +++ b/surfsense_local/frontend/translations/pt-BR.json @@ -449,6 +449,10 @@ "models_download_image_unexpected_error": "Ocorreu um erro inesperado", "models_download_image_unsupported_body": "Esta versão não tem runtime de imagem local. Use um servidor acima.", "models_download_image_unsupported_title": "Modelos de imagem não podem ser executados neste computador", + "models_download_speech_delete_aria": "Excluir {model}", + "models_download_speech_load_error": "Não foi possível carregar os modelos de reconhecimento de voz", + "models_download_speech_request_error": "A solicitação falhou", + "models_download_speech_unsupported_title": "Modelos de reconhecimento de voz não podem ser executados neste computador", "models_download_video_unsupported_title": "Modelos de vídeo não podem ser executados neste computador", "models_error_install_checksum_mismatch": "O arquivo baixado não corresponde ao esperado. Tente baixar novamente.", "models_error_install_failed": "Não foi possível instalar o modelo. Tente baixar novamente.", @@ -479,8 +483,8 @@ "models_image_server_running_status": "Em execução", "models_in_use_local_label": "Este computador", "models_in_use_missing_label": "Não encontrado neste computador", - "models_in_use_summary_aria": "{slot, select, chat {modelo de chat em uso} image {modelo de imagem em uso} image_edit {modelo de edição de imagem em uso} video {modelo de vídeo em uso} audio {modelo de áudio em uso} other {modelo em uso}}", - "models_in_use_summary_empty": "{slot, select, chat {Nenhum modelo de chat escolhido} image {Nenhum modelo de imagem escolhido} image_edit {Nenhum modelo de edição de imagem escolhido} video {Nenhum modelo de vídeo escolhido} audio {Nenhum modelo de áudio escolhido} other {Nenhum modelo escolhido}}", + "models_in_use_summary_aria": "{slot, select, chat {modelo de chat em uso} image {modelo de imagem em uso} image_edit {modelo de edição de imagem em uso} video {modelo de vídeo em uso} audio {modelo de áudio em uso} speech {modelo de reconhecimento de voz em uso} other {modelo em uso}}", + "models_in_use_summary_empty": "{slot, select, chat {Nenhum modelo de chat escolhido} image {Nenhum modelo de imagem escolhido} image_edit {Nenhum modelo de edição de imagem escolhido} video {Nenhum modelo de vídeo escolhido} audio {Nenhum modelo de áudio escolhido} speech {Nenhum modelo de reconhecimento de voz escolhido} other {Nenhum modelo escolhido}}", "models_in_use_summary_label": "Em uso:", "models_in_use_unknown_server_label": "Um servidor", "models_install_cancelled_short_status": "Cancelado", @@ -600,6 +604,7 @@ "models_server_models_type_audio_gen_label": "Geração de áudio", "models_server_models_type_image_edit_label": "Edição de imagem", "models_server_models_type_image_gen_label": "Geração de imagem", + "models_server_models_type_speech_to_text_label": "Reconhecimento de voz", "models_server_models_type_text_gen_label": "Geração de texto", "models_server_models_type_video_gen_label": "Geração de vídeo", "models_server_models_use_aria": "Usar {model}", @@ -619,6 +624,14 @@ "models_server_voices_trial_body": "Olá. É assim que seus podcasts vão soar.", "models_server_voices_voice_aria": "ID de voz", "models_server_voices_voice_placeholder": "ID de voz", + "models_speech_language_auto_label": "Detectar automaticamente", + "models_speech_language_body": "O idioma das suas gravações e da sua entrada de voz.", + "models_speech_language_request_error": "A solicitação falhou", + "models_speech_language_title": "Idioma falado", + "models_speech_model_detects_language_label": "Detecta o idioma", + "models_speech_model_languages_button": "{count, plural, one {# idioma} many {# idiomas} other {# idiomas}}", + "models_speech_model_peak_memory_gigabytes_label": "{size, number, ::unit/gigabyte .#} ao transcrever", + "models_speech_model_peak_memory_megabytes_label": "{size, number, ::unit/megabyte .} ao transcrever", "models_try_dialog_cancel_button": "Cancelar", "models_try_dialog_chat_confirmed_body": "O suporte a chat está confirmado. O teste envia um prompt curto e pode gerar custos.", "models_try_dialog_chat_unconfirmed_body": "Este endpoint não publica suas capacidades, então o suporte a chat não está confirmado. O teste envia um prompt curto e pode gerar custos.", @@ -974,10 +987,13 @@ "settings_image_models_title": "Modelos de geração de imagem", "settings_models_add_button": "Adicionar modelo", "settings_models_add_page_body": "Execute um neste computador ou use um de um servidor que você já mantém.", - "settings_models_add_page_title": "{slot, select, audio {Adicionar um modelo de áudio} chat {Adicionar um modelo de chat} image {Adicionar um modelo de imagem} image_edit {Adicionar um modelo de edição de imagem} video {Adicionar um modelo de vídeo} other {Adicionar um modelo}}", - "settings_models_empty": "{slot, select, audio {Ainda sem modelo de áudio} chat {Ainda sem modelo de chat} image {Ainda sem modelo de imagem} image_edit {Ainda sem modelo de edição de imagem} video {Ainda sem modelo de vídeo} other {Ainda sem modelo}}", + "settings_models_add_page_local_body": "Baixe um para executar neste computador.", + "settings_models_add_page_title": "{slot, select, audio {Adicionar um modelo de áudio} chat {Adicionar um modelo de chat} image {Adicionar um modelo de imagem} image_edit {Adicionar um modelo de edição de imagem} video {Adicionar um modelo de vídeo} speech {Adicionar um modelo de reconhecimento de voz} other {Adicionar um modelo}}", + "settings_models_empty": "{slot, select, audio {Ainda sem modelo de áudio} chat {Ainda sem modelo de chat} image {Ainda sem modelo de imagem} image_edit {Ainda sem modelo de edição de imagem} video {Ainda sem modelo de vídeo} speech {Ainda sem modelo de reconhecimento de voz} other {Ainda sem modelo}}", "settings_models_empty_download_body": "Baixe um para executar neste computador ou use um de um servidor que você já mantém.", + "settings_models_empty_local_body": "Baixe um para executar neste computador.", "settings_models_empty_server_body": "Use um de um servidor que você já mantém.", + "settings_models_empty_unsupported_body": "Este computador não consegue executar nenhum.", "settings_models_load_error": "Não foi possível carregar as configurações de modelos", "settings_nav_about_label": "Sobre", "settings_nav_app_title": "App", @@ -994,9 +1010,12 @@ "settings_nav_plugins_label": "Plugins", "settings_nav_report_issue_label": "Relatar problema", "settings_nav_resources_label": "Recursos", + "settings_nav_speech_label": "Reconhec. de voz", "settings_nav_system_title": "Sistema", "settings_nav_title": "Configurações", "settings_nav_video_label": "Vídeo", + "settings_speech_models_body": "O modelo que transforma suas gravações e sua entrada de voz em texto, neste computador.", + "settings_speech_models_title": "Modelos de reconhecimento de voz", "settings_video_models_body": "O modelo que cria vídeos. Execute um neste computador ou use um de um servidor.", "settings_video_models_title": "Modelos de geração de vídeo", "source_preview_panel_aria": "Prévia da fonte", @@ -1112,6 +1131,17 @@ "sources_upload_too_large_reason_label": "Acima de {size, number, ::unit/megabyte}", "sources_upload_unsupported_body": "Sem suporte: {files}", "sources_upload_unsupported_error": "Tipo de arquivo sem suporte: {files}", + "speech_setup_card_download_button": "Baixar e configurar", + "speech_setup_card_languages_body": "Funciona bem em: {languages}", + "speech_setup_card_use_button": "Configurar", + "speech_setup_popover_body": "O SurfSense transforma sua voz em texto neste computador. Basta baixar uma vez.", + "speech_setup_popover_done_button": "Concluído", + "speech_setup_popover_load_error": "Não foi possível carregar os modelos de reconhecimento de voz.", + "speech_setup_popover_more_button": "Mais opções", + "speech_setup_popover_ready_title": "O reconhecimento de voz está pronto", + "speech_setup_popover_start_button": "Começar a falar", + "speech_setup_popover_title": "Configurar o reconhecimento de voz", + "speech_setup_popover_unsupported_body": "O reconhecimento de voz não pode ser executado neste computador.", "studio_artifact_failed_toast": "{name} falhou", "studio_artifact_failed_toast_body": "Não foi possível gerar este artefato. Tente novamente na aba Artefatos.", "studio_artifact_list_clear_filter_button": "Limpar filtro", diff --git a/surfsense_local/frontend/translations/ru.json b/surfsense_local/frontend/translations/ru.json index a12ca6cac3..24a1538e86 100644 --- a/surfsense_local/frontend/translations/ru.json +++ b/surfsense_local/frontend/translations/ru.json @@ -449,6 +449,10 @@ "models_download_image_unexpected_error": "Произошла непредвиденная ошибка", "models_download_image_unsupported_body": "В этой сборке нет локальной среды выполнения для изображений. Используйте вместо неё сервер выше.", "models_download_image_unsupported_title": "Модели для изображений не запускаются на этом компьютере", + "models_download_speech_delete_aria": "Удалить {model}", + "models_download_speech_load_error": "Не удалось загрузить модели распознавания речи", + "models_download_speech_request_error": "Запрос не выполнен", + "models_download_speech_unsupported_title": "Модели распознавания речи не запускаются на этом компьютере", "models_download_video_unsupported_title": "Модели для видео не запускаются на этом компьютере", "models_error_install_checksum_mismatch": "Скачанный файл не совпадает с ожидаемым. Повторите скачивание.", "models_error_install_failed": "Не удалось установить модель. Повторите скачивание.", @@ -479,8 +483,8 @@ "models_image_server_running_status": "Работает", "models_in_use_local_label": "Этот компьютер", "models_in_use_missing_label": "Не найдена на этом компьютере", - "models_in_use_summary_aria": "{slot, select, chat {используемая модель для чата} image {используемая модель для изображений} image_edit {используемая модель для редактирования изображений} video {используемая модель для видео} audio {используемая модель для аудио} other {используемая модель}}", - "models_in_use_summary_empty": "{slot, select, chat {Модель для чата не выбрана} image {Модель для изображений не выбрана} image_edit {Модель для редактирования изображений не выбрана} video {Модель для видео не выбрана} audio {Модель для аудио не выбрана} other {Модель не выбрана}}", + "models_in_use_summary_aria": "{slot, select, chat {используемая модель для чата} image {используемая модель для изображений} image_edit {используемая модель для редактирования изображений} video {используемая модель для видео} audio {используемая модель для аудио} speech {используемая модель распознавания речи} other {используемая модель}}", + "models_in_use_summary_empty": "{slot, select, chat {Модель для чата не выбрана} image {Модель для изображений не выбрана} image_edit {Модель для редактирования изображений не выбрана} video {Модель для видео не выбрана} audio {Модель для аудио не выбрана} speech {Модель распознавания речи не выбрана} other {Модель не выбрана}}", "models_in_use_summary_label": "Используется:", "models_in_use_unknown_server_label": "Сервер", "models_install_cancelled_short_status": "Отменено", @@ -600,6 +604,7 @@ "models_server_models_type_audio_gen_label": "Генерация аудио", "models_server_models_type_image_edit_label": "Редактирование изображений", "models_server_models_type_image_gen_label": "Генерация изображений", + "models_server_models_type_speech_to_text_label": "Распознавание речи", "models_server_models_type_text_gen_label": "Генерация текста", "models_server_models_type_video_gen_label": "Генерация видео", "models_server_models_use_aria": "Использовать {model}", @@ -619,6 +624,14 @@ "models_server_voices_trial_body": "Здравствуйте. Так будут звучать ваши подкасты.", "models_server_voices_voice_aria": "ID голоса", "models_server_voices_voice_placeholder": "ID голоса", + "models_speech_language_auto_label": "Определять автоматически", + "models_speech_language_body": "Язык ваших записей и голосового ввода.", + "models_speech_language_request_error": "Запрос не выполнен", + "models_speech_language_title": "Язык речи", + "models_speech_model_detects_language_label": "Определяет язык", + "models_speech_model_languages_button": "{count, plural, one {# язык} few {# языка} many {# языков} other {# языка}}", + "models_speech_model_peak_memory_gigabytes_label": "{size, number, ::unit/gigabyte .#} при распознавании", + "models_speech_model_peak_memory_megabytes_label": "{size, number, ::unit/megabyte .} при распознавании", "models_try_dialog_cancel_button": "Отмена", "models_try_dialog_chat_confirmed_body": "Поддержка чата подтверждена. При тесте отправляется один короткий промпт, это может стоить денег.", "models_try_dialog_chat_unconfirmed_body": "Эта конечная точка не сообщает о своих возможностях, поэтому поддержка чата не подтверждена. При тесте отправляется один короткий промпт, это может стоить денег.", @@ -974,10 +987,13 @@ "settings_image_models_title": "Модели генерации изображений", "settings_models_add_button": "Добавить модель", "settings_models_add_page_body": "Запустите модель на этом компьютере или используйте модель с сервера, который у вас уже работает.", - "settings_models_add_page_title": "{slot, select, audio {Добавить модель для аудио} chat {Добавить модель для чата} image {Добавить модель для изображений} image_edit {Добавить модель для редактирования изображений} video {Добавить модель для видео} other {Добавить модель}}", - "settings_models_empty": "{slot, select, audio {Модели для аудио пока нет} chat {Модели для чата пока нет} image {Модели для изображений пока нет} image_edit {Модели для редактирования изображений пока нет} video {Модели для видео пока нет} other {Модели пока нет}}", + "settings_models_add_page_local_body": "Скачайте модель, чтобы запустить её на этом компьютере.", + "settings_models_add_page_title": "{slot, select, audio {Добавить модель для аудио} chat {Добавить модель для чата} image {Добавить модель для изображений} image_edit {Добавить модель для редактирования изображений} video {Добавить модель для видео} speech {Добавить модель распознавания речи} other {Добавить модель}}", + "settings_models_empty": "{slot, select, audio {Модели для аудио пока нет} chat {Модели для чата пока нет} image {Модели для изображений пока нет} image_edit {Модели для редактирования изображений пока нет} video {Модели для видео пока нет} speech {Модели распознавания речи пока нет} other {Модели пока нет}}", "settings_models_empty_download_body": "Скачайте модель, чтобы запустить её на этом компьютере, или используйте модель с сервера, который у вас уже работает.", + "settings_models_empty_local_body": "Скачайте модель, чтобы запустить её на этом компьютере.", "settings_models_empty_server_body": "Используйте модель с сервера, который у вас уже работает.", + "settings_models_empty_unsupported_body": "На этом компьютере такая модель не запустится.", "settings_models_load_error": "Не удалось загрузить настройки моделей", "settings_nav_about_label": "О приложении", "settings_nav_app_title": "Приложение", @@ -994,9 +1010,12 @@ "settings_nav_plugins_label": "Плагины", "settings_nav_report_issue_label": "Отчёт об ошибке", "settings_nav_resources_label": "Ресурсы", + "settings_nav_speech_label": "Распозн. речи", "settings_nav_system_title": "Система", "settings_nav_title": "Настройки", "settings_nav_video_label": "Видео", + "settings_speech_models_body": "Модель, которая превращает записи и голосовой ввод в текст на этом компьютере.", + "settings_speech_models_title": "Модели распознавания речи", "settings_video_models_body": "Модель, которая создаёт видео. Запустите её на этом компьютере или используйте модель с сервера.", "settings_video_models_title": "Модели генерации видео", "source_preview_panel_aria": "Предпросмотр источника", @@ -1112,6 +1131,17 @@ "sources_upload_too_large_reason_label": "Больше {size, number, ::unit/megabyte}", "sources_upload_unsupported_body": "Не поддерживается: {files}", "sources_upload_unsupported_error": "Неподдерживаемый тип файла: {files}", + "speech_setup_card_download_button": "Скачать и настроить", + "speech_setup_card_languages_body": "Хорошо работает с языками: {languages}", + "speech_setup_card_use_button": "Настроить", + "speech_setup_popover_body": "SurfSense превращает ваш голос в текст на этом компьютере. Модель нужно скачать один раз.", + "speech_setup_popover_done_button": "Готово", + "speech_setup_popover_load_error": "Не удалось загрузить модели распознавания речи.", + "speech_setup_popover_more_button": "Другие варианты", + "speech_setup_popover_ready_title": "Распознавание речи готово", + "speech_setup_popover_start_button": "Начать говорить", + "speech_setup_popover_title": "Настройка распознавания речи", + "speech_setup_popover_unsupported_body": "Распознавание речи не работает на этом компьютере.", "studio_artifact_failed_toast": "{name}: ошибка", "studio_artifact_failed_toast_body": "Этот артефакт не удалось создать. Повторите попытку на вкладке «Артефакты».", "studio_artifact_list_clear_filter_button": "Сбросить фильтр", diff --git a/surfsense_local/frontend/translations/zh-CN.json b/surfsense_local/frontend/translations/zh-CN.json index 960647de77..ea30a0849a 100644 --- a/surfsense_local/frontend/translations/zh-CN.json +++ b/surfsense_local/frontend/translations/zh-CN.json @@ -449,6 +449,10 @@ "models_download_image_unexpected_error": "发生了意外错误", "models_download_image_unsupported_body": "这个版本不含本地图像运行时。请改用上面的服务器。", "models_download_image_unsupported_title": "这台电脑无法运行图像模型", + "models_download_speech_delete_aria": "删除 {model}", + "models_download_speech_load_error": "无法加载语音识别模型", + "models_download_speech_request_error": "请求失败", + "models_download_speech_unsupported_title": "这台电脑无法运行语音识别模型", "models_download_video_unsupported_title": "这台电脑无法运行视频模型", "models_error_install_checksum_mismatch": "下载的文件与预期的文件不一致。请重试下载。", "models_error_install_failed": "无法安装模型。请重试下载。", @@ -479,8 +483,8 @@ "models_image_server_running_status": "运行中", "models_in_use_local_label": "这台电脑", "models_in_use_missing_label": "在这台电脑上找不到", - "models_in_use_summary_aria": "{slot, select, chat {使用中的对话模型} image {使用中的图像模型} image_edit {使用中的图像编辑模型} video {使用中的视频模型} audio {使用中的语音模型} other {使用中的模型}}", - "models_in_use_summary_empty": "{slot, select, chat {未选择对话模型} image {未选择图像模型} image_edit {未选择图像编辑模型} video {未选择视频模型} audio {未选择语音模型} other {未选择模型}}", + "models_in_use_summary_aria": "{slot, select, chat {使用中的对话模型} image {使用中的图像模型} image_edit {使用中的图像编辑模型} video {使用中的视频模型} audio {使用中的语音模型} speech {使用中的语音识别模型} other {使用中的模型}}", + "models_in_use_summary_empty": "{slot, select, chat {未选择对话模型} image {未选择图像模型} image_edit {未选择图像编辑模型} video {未选择视频模型} audio {未选择语音模型} speech {未选择语音识别模型} other {未选择模型}}", "models_in_use_summary_label": "使用中：", "models_in_use_unknown_server_label": "某个服务器", "models_install_cancelled_short_status": "已取消", @@ -600,6 +604,7 @@ "models_server_models_type_audio_gen_label": "语音生成", "models_server_models_type_image_edit_label": "图像编辑", "models_server_models_type_image_gen_label": "图像生成", + "models_server_models_type_speech_to_text_label": "语音识别", "models_server_models_type_text_gen_label": "文本生成", "models_server_models_type_video_gen_label": "视频生成", "models_server_models_use_aria": "使用 {model}", @@ -619,6 +624,14 @@ "models_server_voices_trial_body": "你好。你的播客听起来会是这样。", "models_server_voices_voice_aria": "音色 ID", "models_server_voices_voice_placeholder": "音色 ID", + "models_speech_language_auto_label": "自动检测", + "models_speech_language_body": "你的录音和语音输入所用的语言。", + "models_speech_language_request_error": "请求失败", + "models_speech_language_title": "说话语言", + "models_speech_model_detects_language_label": "自动检测语言", + "models_speech_model_languages_button": "{count, plural, other {# 种语言}}", + "models_speech_model_peak_memory_gigabytes_label": "转写时 {size, number, ::unit/gigabyte .#}", + "models_speech_model_peak_memory_megabytes_label": "转写时 {size, number, ::unit/megabyte .}", "models_try_dialog_cancel_button": "取消", "models_try_dialog_chat_confirmed_body": "已确认支持对话。测试会发送一条简短的提示词，可能产生费用。", "models_try_dialog_chat_unconfirmed_body": "这个端点没有公布能力，因此是否支持对话尚未确认。测试会发送一条简短的提示词，可能产生费用。", @@ -974,10 +987,13 @@ "settings_image_models_title": "图像生成模型", "settings_models_add_button": "添加模型", "settings_models_add_page_body": "可以在这台电脑上运行一个，也可以使用你已经在运行的服务器上的模型。", - "settings_models_add_page_title": "{slot, select, audio {添加语音模型} chat {添加对话模型} image {添加图像模型} image_edit {添加图像编辑模型} video {添加视频模型} other {添加模型}}", - "settings_models_empty": "{slot, select, audio {还没有语音模型} chat {还没有对话模型} image {还没有图像模型} image_edit {还没有图像编辑模型} video {还没有视频模型} other {还没有模型}}", + "settings_models_add_page_local_body": "下载一个在这台电脑上运行。", + "settings_models_add_page_title": "{slot, select, audio {添加语音模型} chat {添加对话模型} image {添加图像模型} image_edit {添加图像编辑模型} video {添加视频模型} speech {添加语音识别模型} other {添加模型}}", + "settings_models_empty": "{slot, select, audio {还没有语音模型} chat {还没有对话模型} image {还没有图像模型} image_edit {还没有图像编辑模型} video {还没有视频模型} speech {还没有语音识别模型} other {还没有模型}}", "settings_models_empty_download_body": "下载一个在这台电脑上运行，也可以使用你已经在运行的服务器上的模型。", + "settings_models_empty_local_body": "下载一个在这台电脑上运行。", "settings_models_empty_server_body": "使用你已经在运行的服务器上的模型。", + "settings_models_empty_unsupported_body": "这台电脑无法运行。", "settings_models_load_error": "无法加载模型设置", "settings_nav_about_label": "关于", "settings_nav_app_title": "应用", @@ -994,9 +1010,12 @@ "settings_nav_plugins_label": "插件", "settings_nav_report_issue_label": "报告问题", "settings_nav_resources_label": "资源", + "settings_nav_speech_label": "语音识别", "settings_nav_system_title": "系统", "settings_nav_title": "设置", "settings_nav_video_label": "视频", + "settings_speech_models_body": "在这台电脑上把录音和语音输入转为文字的模型。", + "settings_speech_models_title": "语音识别模型", "settings_video_models_body": "生成视频的模型。可以在这台电脑上运行一个，也可以使用服务器上的。", "settings_video_models_title": "视频生成模型", "source_preview_panel_aria": "来源预览", @@ -1112,6 +1131,17 @@ "sources_upload_too_large_reason_label": "超过 {size, number, ::unit/megabyte}", "sources_upload_unsupported_body": "不支持：{files}", "sources_upload_unsupported_error": "不支持的文件类型：{files}", + "speech_setup_card_download_button": "下载并设置", + "speech_setup_card_languages_body": "擅长的语言：{languages}", + "speech_setup_card_use_button": "设置", + "speech_setup_popover_body": "SurfSense 在这台电脑上把你的语音转为文字。只需下载一次。", + "speech_setup_popover_done_button": "完成", + "speech_setup_popover_load_error": "无法加载语音识别模型。", + "speech_setup_popover_more_button": "更多选择", + "speech_setup_popover_ready_title": "语音识别已就绪", + "speech_setup_popover_start_button": "开始说话", + "speech_setup_popover_title": "设置语音识别", + "speech_setup_popover_unsupported_body": "这台电脑无法运行语音识别。", "studio_artifact_failed_toast": "{name} 失败", "studio_artifact_failed_toast_body": "这个产物没能生成。请在“产物”标签页中重试。", "studio_artifact_list_clear_filter_button": "清除筛选", From 5d4dc25312b573285be6015bd393a3f901abdcf4 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Wed, 14 Oct 2026 01:54:13 -0700 Subject: [PATCH 02/17] feat(local): stage speech recognition's audio.cpp server, not yet started Step 2 of the local speech recognition proposal. Nemotron 3.5 ASR runs on the audiocpp_server the app already ships for voices, as a second process with its own config, port and crash restart, so a voice install, a podcast's unload or a busy voice model never touches a transcription. - pins.mjs builds nemotron_asr beside the voice families, so its spec is staged too; staging now starts the server once per curated family on an empty model package and fails unless that family's loader runs, which --list-devices could not show. It also prints how long staging took. - build-audiocpp.yml gains a macOS job that stages upstream's archive the same way; the Linux and Windows gates re-run because the pin changes the cache key. - Electron has the audiocpp-transcription sidecar (transcription.json, its own free port, the voices' flags without eSpeak) and a watcher that starts, restarts and stops it with that file, and restarts it after a crash at most once per 10 s, giving up after 3 crashes in 5 minutes until the file changes. - The API alone receives SURFSENSE_LOCAL_TRANSCRIPTION_BASE_URL, read into LLMSettings.transcription_base_url; workers will go through the API. Nothing writes transcription.json yet, so the server never starts. ADR 0055 records the decision; packaging.md and overview.md describe it. --- .github/workflows/build-audiocpp.yml | 29 +- ...nition-runs-on-a-second-audiocpp-server.md | 33 ++ docs/adr/README.md | 1 + docs/architecture/overview.md | 4 +- docs/architecture/packaging.md | 10 +- docs/proposals/local-speech-to-text.md | 2 +- surfsense_local/backend/shared/config.py | 5 + .../tests/unit/shared/test_llm_settings.py | 23 ++ .../scripts/audiocpp/built-families.mjs | 76 +++++ .../scripts/audiocpp/built-families.test.mjs | 23 ++ .../electron/scripts/audiocpp/pins.mjs | 3 +- .../electron/scripts/audiocpp/recipe.test.mjs | 7 + .../scripts/audiocpp/server-files.test.mjs | 37 +++ .../electron/scripts/audiocpp/stage.mjs | 9 +- .../sidecars/audiocpp-transcription.test.ts | 147 +++++++++ .../main/sidecars/audiocpp-transcription.ts | 63 ++++ .../electron/src/main/sidecars/boot.ts | 8 + .../electron/src/main/sidecars/python.test.ts | 31 ++ .../electron/src/main/sidecars/python.ts | 12 +- .../electron/src/main/sidecars/types.ts | 3 + .../watch-transcription-models.test.ts | 298 ++++++++++++++++++ .../sidecars/watch-transcription-models.ts | 96 ++++++ 22 files changed, 909 insertions(+), 11 deletions(-) create mode 100644 docs/adr/0055-speech-recognition-runs-on-a-second-audiocpp-server.md create mode 100644 surfsense_local/backend/tests/unit/shared/test_llm_settings.py create mode 100644 surfsense_local/electron/scripts/audiocpp/built-families.mjs create mode 100644 surfsense_local/electron/scripts/audiocpp/built-families.test.mjs create mode 100644 surfsense_local/electron/scripts/audiocpp/server-files.test.mjs create mode 100644 surfsense_local/electron/src/main/sidecars/audiocpp-transcription.test.ts create mode 100644 surfsense_local/electron/src/main/sidecars/audiocpp-transcription.ts create mode 100644 surfsense_local/electron/src/main/sidecars/watch-transcription-models.test.ts create mode 100644 surfsense_local/electron/src/main/sidecars/watch-transcription-models.ts diff --git a/.github/workflows/build-audiocpp.yml b/.github/workflows/build-audiocpp.yml index 26c7721db7..4053fa889f 100644 --- a/.github/workflows/build-audiocpp.yml +++ b/.github/workflows/build-audiocpp.yml @@ -1,8 +1,9 @@ # Compile audio.cpp's server for Windows and Linux with the recipe in # surfsense_local/electron/scripts/audiocpp/, check it against the app's # floors, and hand the staged folder to the desktop release as an artifact. -# A pull request that changes the recipe, or the app code that drives the -# server, runs it too. +# Check that upstream's macOS archive, which the release ships as it is, has +# every curated family too. A pull request that changes the recipe, or the +# app code that drives the server, runs it too. # # TODO: ask upstream (0xShug0/audio.cpp) to publish Linux archives built on # Ubuntu 22.04 and Windows ones with per-CPU ggml libraries, as llama.cpp does. @@ -71,6 +72,8 @@ jobs: echo "CC=gcc-13" >> "$GITHUB_ENV" echo "CXX=g++-13" >> "$GITHUB_ENV" + # Staging starts the server once per curated family and fails unless + # that family's loader runs, so a family the recipe drops never ships. - name: Compile and stage if: steps.cache.outputs.cache-hit != 'true' run: node scripts/audiocpp/stage.mjs --strict @@ -148,3 +151,25 @@ jobs: name: audiocpp-${{ runner.os }} path: surfsense_local/electron/audiocpp.tar retention-days: 7 + + # macOS ships upstream's archive, which the recipe does not build, so a + # family added to the pin is checked against the archive itself. The release + # job stages it again; this only fails earlier, on the pull request. + macos-archive: + name: macOS arm64, upstream's archive + runs-on: macos-15 + defaults: + run: + working-directory: surfsense_local/electron + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-node@v5 + with: + node-version: 24 + package-manager-cache: false + + # Downloads the pinned archive, copies each curated family's spec, which + # fails on one it lacks, and starts the server once per family. + - name: Stage and check every curated family + run: node scripts/audiocpp/stage.mjs --strict diff --git a/docs/adr/0055-speech-recognition-runs-on-a-second-audiocpp-server.md b/docs/adr/0055-speech-recognition-runs-on-a-second-audiocpp-server.md new file mode 100644 index 0000000000..4a267758eb --- /dev/null +++ b/docs/adr/0055-speech-recognition-runs-on-a-second-audiocpp-server.md @@ -0,0 +1,33 @@ +# ADR 0055: Speech recognition runs on a second audio.cpp server that only the API calls + +- **Status:** Proposed, with the [local speech recognition proposal](../proposals/local-speech-to-text.md) +- **Date:** 2026-10-14 +- **Source:** [Local speech recognition proposal, the runtime L119–147](https://github.com/MODSetter/SurfSense/blob/55b3ddc2e9d6423f194ff7e5ef0c62910cece95f/docs/proposals/local-speech-to-text.md#L119-L147) + +## Context + +Recordings as sources and a mic in the chat composer need a speech recogniser that runs on the user's CPU and takes as little of the machine as it can. The app already ships audio.cpp's `audiocpp_server` for podcast voices ([packaging](../architecture/packaging.md)), and its pinned v0.8.2 source has NVIDIA's Nemotron 3.5 ASR, which held 1.19 GB loaded and ran at 11.5× real time on that server's CPU path. Compiling its family adds 9 source files and no new library. + +The voices' server cannot also serve it. Under its flags a dictation would evict Kokoro between a podcast's turns; a request that finds the other model busy fails at once with 503; the podcast's closing `unload_all_models` would unload a warm speech model and wait without bound on a transcription in flight; and every voice install, delete and API start rewrites `server.json`, which restarts the server and kills whatever it is transcribing. + +Text models already reach their runtime only through the API, so one gate can order every request ([ADR 0048](0048-the-api-is-the-only-path-to-a-text-model.md)). A recording's windows from the ingest worker and a speaker's phrases from the composer compete for one model the same way. + +## Decision + +- Speech recognition runs on `audiocpp_server`, the binary staged for voices, compiled with `nemotron_asr` beside the voice families and started a second time as its own sidecar, `audiocpp-transcription`. +- It reads its own config, `/audio/transcription.json`, which the API writes, and listens on its own port. Electron passes the flags: the CPU backend, half the logical cores up to 8, one loaded model, an unload after 5 idle minutes, 1 GiB kept free, no web UI. It gets no eSpeak. +- Electron starts it when that file exists, restarts it when the file changes and stops it when the file goes. After a crash it restarts it, at most once every 10 seconds, and leaves it stopped after 3 crashes in 5 minutes, until the file changes again. +- Only the API is told where it is. Every other process reaches speech recognition through the API. +- [transcribe.cpp](https://github.com/handy-computer/transcribe.cpp) is the fallback, replacing the API's client, the sidecar spec and the config writer, not the routes. + +## Consequences + +- Voices and speech recognition never unload, evict, restart or wait on each other. The second process costs 7 MiB idle and 10 MiB after its model is unloaded (measured). +- A dictation does not wait for a recording behind the server's own queue, because the API decides the order. +- The languages are those of the families in the pin. audio.cpp will not run Whisper, so the long tail of languages needs a later model or the fallback. +- One binary serves both, so a pin bump is measured for both: the voices' peaks and the speech model's. +- Settings › Resources counts the second process under its audio.cpp row with no change, since it counts every `audiocpp_server`. + +## Where the code stands + +The family is compiled and checked in staging, and Electron has the sidecar, its watcher with the crash restart, and the API's address (`sidecars/audiocpp-transcription.ts`, `sidecars/watch-transcription-models.ts`, `sidecars/python.ts`). Nothing writes `transcription.json` yet, so the server never starts, and the API has no client, gate or routes for it ([proposal](../proposals/local-speech-to-text.md#order-of-work), steps 3 and 4). diff --git a/docs/adr/README.md b/docs/adr/README.md index 1cdc3a0e17..f6e1c69b64 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -82,3 +82,4 @@ The Date is the day the decision was first written down. Source links are permal | 0052 | [The chat engine's model never calls tools; SurfSense chooses a plugin tool through structured output and calls it](0052-the-chat-model-never-calls-tools.md) | Proposed, deferred | | 0053 | [First-party plugins are listed in the app, third-party plugins in a registry in this repository, and third-party plugins start off](0053-plugins-are-listed-in-a-registry-and-third-party-ones-start-off.md) | Proposed | | 0054 | [SurfSense's own plugins, free and paid, run on one plugin host, with the registry served beside it](0054-surfsenses-plugins-run-on-one-plugin-host.md) | Proposed | +| 0055 | [Speech recognition runs on a second audio.cpp server that only the API calls](0055-speech-recognition-runs-on-a-second-audiocpp-server.md) | Proposed | diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 988cf9ab76..c02db69c41 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -22,6 +22,7 @@ Electron main ─┬─ api FastAPI on 127.0.0.1, free port ─┐ ├─ llamacpp llama-server in router mode ├─ sdcpp sd-server, started once an image model is chosen ├─ audiocpp audiocpp_server, started once the API names an audio model + ├─ audiocpp-transcription the same binary, started once the API names a speech model └─ opencode opencode serve, started once the API writes its configuration BrowserWindow (Vite SPA) ── HTTP ──> api @@ -35,11 +36,12 @@ worker-ingest, worker-studio ── POST /internal/events ──> api ── SSE - llama-server starts whenever its pinned build is staged, in dev too (`pnpm build:llamacpp`, which `predev` runs). It reads per-model arguments from a preset file once at startup, so Electron restarts it when the API rewrites that file ([`local-models/runtime.md`](local-models/runtime.md)). - sd-server takes its model as startup arguments, so it cannot start at boot. Whenever its pinned build is staged, in dev too (`pnpm build:sdcpp`, which `predev` runs), `watchImageModel` asks the API every 5 seconds which files to run, and starts, restarts or stops it on a change. The API names a model only while a Studio job needs one and for up to 5 minutes after, so its weights are not held beside the chat model all session ([`studio.md`](studio.md#the-image-path)). - audiocpp_server refuses an empty model list, so it starts only once the API has written `audio/server.json` under the data directory, in dev too (`pnpm build:audiocpp`, which `predev` runs). On Windows and Linux that script compiles audio.cpp, and without a C++ toolchain the app runs without local audio ([packaging](packaging.md)). Electron checks that file every 5 seconds and restarts the server when it changes. Electron sets the machine-wide flags: the CPU backend, half the logical cores up to 8, one loaded model, and an unload after 5 idle minutes. The API writes that file whenever an audio model is installed or deleted, and at startup ([`local-models/catalog.md`](local-models/catalog.md)). The Studio worker voices podcasts there, at `SURFSENSE_LOCAL_AUDIO_BASE_URL`, and unloads the model when a podcast ends ([`studio.md`](studio.md)). +- The same `audiocpp_server` runs a second time as `audiocpp-transcription`, for speech recognition, from its own `audio/transcription.json` and on its own port, so a voice install, a podcast's unload or a busy voice model never touches it ([ADR 0055](../adr/0055-speech-recognition-runs-on-a-second-audiocpp-server.md)). Electron checks that file every 5 seconds: it starts the server once the file is there, restarts it when the file changes and stops it when the file goes. It takes the voices' flags, with no eSpeak. Unlike the voices' server it is restarted after a crash, at most once every 10 seconds, and left stopped after 3 crashes in 5 minutes until the file changes. The API alone is told its address, `SURFSENSE_LOCAL_TRANSCRIPTION_BASE_URL`. Nothing writes that file yet, so it does not run ([proposal](../proposals/local-speech-to-text.md)). - Only the API gates the window. Electron waits up to 60 seconds for `/health` and gives up at once if the API exits. llama-server is best-effort; its state shows through `/llm/providers`. - opencode, the agent's engine, starts only once the API has written `agent/opencode.json` under the data directory, in dev too (`pnpm build:opencode`, which `predev` runs, stages it unless `SURFSENSE_LOCAL_OPENCODE_ENABLED=0`: [agent](agent.md)). Electron checks for that file every 2 seconds: it starts opencode once the file is there, stops it when the file goes, and restarts it after a crash, at most once every 10 seconds. A rewrite does not restart it, because opencode reads the file per folder the first time the folder is used; the API makes it read the file again through `POST /global/dispose` ([agent proposal](../proposals/agent/03-opencode.md)). Electron removes the file at boot, so each run's API writes its own. At boot Electron picks opencode's port and a random password, which it passes to the API alone as `SURFSENSE_LOCAL_OPENCODE_URL` and `SURFSENSE_LOCAL_OPENCODE_PASSWORD`. opencode's environment is built from a few system variables rather than inherited, with its home and XDG folders under `agent/opencode/`, its own fetches switched off and its proxy pointed at the discard port ([`sidecars/opencode.ts`](../../surfsense_local/electron/src/main/sidecars/opencode.ts)). `serve` keeps running when its parent is killed, so Electron records the pid, port and password in `agent/opencode-process.json`, and the next boot stops a recorded process that still accepts that password. - Before a quit, whether from the menu or by closing the last window, Electron asks the API how many chat replies are being written; with any, it asks the person, and on Quit has the API store each reply before anything is stopped ([chat](chat.md#runs)). Cancel keeps the app open. - On macOS and Linux each child runs in its own process group. On quit Electron sends SIGTERM and, after 5 seconds, SIGKILL; the group is also killed as soon as the sidecar itself exits, because a child that ignores SIGTERM would outlive the app. On Windows it kills the process tree. A single-instance lock hands a second launch to the first window, because two sets of sidecars would fight over the SQLite file. -- The Python sidecars are configured through `SURFSENSE_LOCAL_*` variables: the API's host and port, the data and models directories, the llama-server and sd-server addresses, the images folder, audio.cpp's address and folder where it is staged, and `SURFSENSE_LOCAL_SECRET`, the key that encrypts stored API keys ([`connections.md`](connections.md)). No other sidecar receives the secret. The API alone also gets `SURFSENSE_LOCAL_SHELL_PID`, Electron's own pid, the root of what Settings › Resources counts as the app ([`resource-usage.md`](resource-usage.md)). +- The Python sidecars are configured through `SURFSENSE_LOCAL_*` variables: the API's host and port, the data and models directories, the llama-server and sd-server addresses, the images folder, audio.cpp's address and folder where it is staged, and `SURFSENSE_LOCAL_SECRET`, the key that encrypts stored API keys ([`connections.md`](connections.md)). No other sidecar receives the secret. The API alone also gets `SURFSENSE_LOCAL_SHELL_PID`, Electron's own pid, the root of what Settings › Resources counts as the app ([`resource-usage.md`](resource-usage.md)), and the speech recognition server's address. ## Layer boundary diff --git a/docs/architecture/packaging.md b/docs/architecture/packaging.md index 32a79cbe2d..165f972998 100644 --- a/docs/architecture/packaging.md +++ b/docs/architecture/packaging.md @@ -3,7 +3,7 @@ One build per platform (Linux ships two packages) carries everything the desktop app needs to run offline: the API and the worker frozen into PyInstaller binaries, the renderer, the embedding, voice and parser model packs, the llama.cpp, sd.cpp and audio.cpp runtimes, and opencode with ripgrep for the agent. PyInstaller follows only `import` statements, so anything the app reaches by a path or a string must be named in a spec, and each omission shows up only in a frozen build on a clean machine; the specs name those files, and packaging tests freeze real binaries to catch the ones that slip. Generation weights are never bundled; the app downloads them when the user asks ([egress](egress.md)). **Code:** [`surfsense_local/backend/bundling/`](../../surfsense_local/backend/bundling/), [`surfsense_local/backend/scripts/build_binaries.py`](../../surfsense_local/backend/scripts/build_binaries.py), [`surfsense_local/electron/electron-builder.yml`](../../surfsense_local/electron/electron-builder.yml), [`surfsense_local/electron/scripts/`](../../surfsense_local/electron/scripts/), [`.github/workflows/release-local.yml`](../../.github/workflows/release-local.yml), [`surfsense_local/backend/tests/packaging/`](../../surfsense_local/backend/tests/packaging/) -**Decisions:** [ADR 0021](../adr/0021-no-intel-mac-build.md), [ADR 0012](../adr/0012-vulkan-only-gpu-backend.md) +**Decisions:** [ADR 0021](../adr/0021-no-intel-mac-build.md), [ADR 0012](../adr/0012-vulkan-only-gpu-backend.md), [ADR 0055](../adr/0055-speech-recognition-runs-on-a-second-audiocpp-server.md) How to cut a release is in [`surfsense_local/RELEASE.md`](../../surfsense_local/RELEASE.md); how a release reaches installed apps is in [updates](updates.md). @@ -18,7 +18,7 @@ The asar holds only the Electron main and preload bundles. Everything else rides | `models` | `backend/models` | the embedding model and the Docling parser pack | | `llamacpp` | `electron/llamacpp` | `llama-server` and the libraries it links | | `sdcpp` | `electron/sdcpp` | `sd-server`, for local image generation | -| `audiocpp` | `electron/audiocpp` | `audiocpp_server`, its libraries, the curated model specs and eSpeak-ng, for podcast voices | +| `audiocpp` | `electron/audiocpp` | `audiocpp_server`, its libraries, the curated model specs and eSpeak-ng, for podcast voices and speech recognition | | `opencode` | `electron/opencode` | `opencode` and `rg`, for the agent, with their licences; absent only from a build staged with `SURFSENSE_LOCAL_OPENCODE_ENABLED=0` | The app icon lives in `electron/build/icons/`: `packaged/` holds the `.icns`, `.ico` and `.png` that `electron-builder.yml` names per OS, and `dev/` a variant with a "DEV" badge, which `electron/src/main/dev-app-identity.ts` sets on the Dock, taskbar, window and About panel only while unpackaged, alongside the name "SurfSense Dev", because development runs inside Electron's own bundle and would otherwise show Electron's icon. The macOS menu bar name and the About panel icon stay Electron’s in development; only packaging changes them. Artwork on Windows and Linux fills its canvas; on macOS it sits at 824 of 1024 px with a transparent margin, Apple's icon grid, so `icon.icns` and `dev/icon-macos.png` carry that margin and the `.ico` and `.png` files do not. @@ -70,7 +70,7 @@ The release workflow runs the three scripts directly. Without the parser pack, D - Windows downloads upstream's Vulkan archive, checked against its pinned SHA-256, and copies the MSVC and OpenMP runtimes beside it, which the archive needs and does not carry. - Linux and macOS compile the pinned commit, because upstream's Linux archives need glibc 2.38 and its macOS archive needs macOS 26.0. The recipe, `scripts/sdcpp/recipe.mjs`, builds Vulkan on Linux, with one ggml library per micro-architecture, and Metal on macOS, for 13.3. - Compiling needs CMake, a C++ compiler and the Vulkan SDK (`glslc` and headers) on Linux, or Xcode's command line tools on macOS; Windows needs Visual Studio 2022 or newer with the C++ tools, for the runtime it ships. Without them the script stages an empty folder and says why, and the app runs without local images. Release CI and `pnpm dist` pass `--strict`, which fails instead, naming everything missing and the command that installs it with the machine's package manager. -- `scripts/audiocpp/stage.mjs`, which `build:audiocpp` runs, stages audio.cpp's `audiocpp_server` into `electron/audiocpp/` with the model specs of the three curated families. It adds eSpeak-ng 1.52.0 from the pinned `espeakng-loader` wheel, which Kokoro and Kitten phonemise through (Kokoro finds it through the server's environment, Kitten through `server.json`, [`local-models/catalog.md`](local-models/catalog.md)), and eSpeak-ng's GPL licence text, which the wheel does not carry. It runs `--list-devices` from the staged folder before it swaps the folder into place. +- `scripts/audiocpp/stage.mjs`, which `build:audiocpp` runs, stages audio.cpp's `audiocpp_server` into `electron/audiocpp/` with the model specs of the four curated families, `FAMILIES` in `scripts/audiocpp/pins.mjs`: Kokoro, Supertonic and Kitten for voices, and Nemotron's `nemotron_asr` for speech recognition, which the same binary serves from a second process ([ADR 0055](../adr/0055-speech-recognition-runs-on-a-second-audiocpp-server.md)). It adds eSpeak-ng 1.52.0 from the pinned `espeakng-loader` wheel, which Kokoro and Kitten phonemise through (Kokoro finds it through the server's environment, Kitten through `server.json`, [`local-models/catalog.md`](local-models/catalog.md)), and eSpeak-ng's GPL licence text, which the wheel does not carry. Before it swaps the folder into place, it runs `--list-devices` from it, then starts the server once per curated family on an empty model package (`scripts/audiocpp/built-families.mjs`): a family the build has fails inside its own loader, and one it lacks is refused by name first, so a family missing from the build stops the stage. It prints the staged size and how long staging took. A server staged before a family was added lacks it, and `stage.mjs` keeps a staged server as it is, so delete `electron/audiocpp/` once to pick the family up. - macOS downloads upstream's archive, checked against its pinned SHA-256. - Windows and Linux compile the pinned commit, because upstream's Linux archives need glibc 2.38 and its Windows archive compiles AVX-512 into the executable. The recipe, `scripts/audiocpp/recipe.mjs`, builds only the CPU backend, since the server runs with `--backend cpu`, with one ggml library per micro-architecture and only the curated model families. - Compiling needs CMake and GCC 13 or newer on Linux, or Visual Studio 2022 or newer with the C++ tools on Windows. Without them the script stages an empty folder and says why, and the app runs without local audio. Release CI and `pnpm dist` pass `--strict`, which fails instead, naming everything missing and the command that installs it with the machine's package manager. @@ -99,6 +99,9 @@ It is a stopgap. Once upstream publishes archives that meet both floors, the app - OpenMP is off, so no `libgomp` ships. - Linux gates on no symbol newer than `GLIBC_2.34` or `GCC_7.0.0`, which RHEL 9's glibc and libgcc provide, and on no file that needs libstdc++, libgomp or Vulkan. The runner's own libstdc++ is newer than a user's, so only that gate, not the voicing test, catches a dynamic one. - Windows copies the MSVC runtime beside the executable, so a clean Windows needs no redistributable. +- Staging checks that the build has every curated family, and a change to `FAMILIES` changes the cache key, so adding a family compiles again and runs both floors' gates on the new build. +- A third job stages upstream's macOS archive on `macos-15` the same way, so a family added to the pin that the archive lacks fails on the pull request; the release's macOS job stages it again. +- With the four families, the Windows stage is 32.6 MB, and cloning and compiling it took 4.8 minutes on a 16-thread Ryzen 7 5800X; `nemotron_asr` added 0.24 MB to the executable (**measured**, 14 Oct 2026). - Both run `test_audiocpp_voicing.py` on the staged folder, cached or not, with each curated model's pinned file downloaded from the URL its manifest entry names ([Packaging tests](#packaging-tests)). - An artifact drops symlinks and file modes, so the staged folder travels as a tar. @@ -155,6 +158,7 @@ Four run in CI: `test_license_key.py` inside the release workflow, `test_frozen_ ## Known gaps - No issue on audio.cpp asks for archives that meet the app's floors yet, so `build-audiocpp.yml` has no end date. +- `nemotron_asr` is checked only for a loader: no packaging test transcribes with it yet, so a staged server that cannot transcribe would still ship. - No packaging test checks that the frozen API carries the agent's `SKILL.md`; `api.spec` collects it, and it was checked by hand. - [`desktop-tests.yml`](../../.github/workflows/desktop-tests.yml) runs on pull requests and by hand (`workflow_dispatch`), never on a push to `dev`, and only on Linux, so a break that lands on `dev`, or shows only on Windows or macOS, shows first in a release build ([#2226](https://github.com/MODSetter/SurfSense/issues/2226)). - The packaged `llama-server`, `audiocpp_server` and `sd-server` are started only on the Linux runner ([#1986](https://github.com/MODSetter/SurfSense/issues/1986)). diff --git a/docs/proposals/local-speech-to-text.md b/docs/proposals/local-speech-to-text.md index 7745405d74..963376e5e9 100644 --- a/docs/proposals/local-speech-to-text.md +++ b/docs/proposals/local-speech-to-text.md @@ -1,5 +1,5 @@ --- -status: proposed +status: in-progress code: - surfsense_local/backend/modules/llm/ - surfsense_local/backend/modules/documents/ diff --git a/surfsense_local/backend/shared/config.py b/surfsense_local/backend/shared/config.py index 9e4fdc4c2f..eb98e6f825 100644 --- a/surfsense_local/backend/shared/config.py +++ b/surfsense_local/backend/shared/config.py @@ -124,6 +124,11 @@ class LLMSettings(BaseSettings): audio_espeak_library: Path | None = None audio_espeak_data: Path | None = None + # The same audio.cpp build run again for speech recognition. Electron tells + # the API alone, and workers reach it through the API; absent where no + # audio.cpp is staged. + transcription_base_url: str | None = None + class AgentSettings(BaseSettings): """Where opencode will listen and its password, chosen by Electron at boot. diff --git a/surfsense_local/backend/tests/unit/shared/test_llm_settings.py b/surfsense_local/backend/tests/unit/shared/test_llm_settings.py new file mode 100644 index 0000000000..42d5a6ade8 --- /dev/null +++ b/surfsense_local/backend/tests/unit/shared/test_llm_settings.py @@ -0,0 +1,23 @@ +import pytest + +from shared.config import LLMSettings + +pytestmark = pytest.mark.unit + + +def test_reads_the_speech_recognition_server_electron_names( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Electron gives the API alone this address (sidecars/python.ts).""" + monkeypatch.setenv( + "SURFSENSE_LOCAL_TRANSCRIPTION_BASE_URL", "http://127.0.0.1:9997" + ) + assert LLMSettings().transcription_base_url == "http://127.0.0.1:9997" + + +def test_names_no_speech_recognition_server_unless_told( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """No audio.cpp staged, or a worker's environment: there is none to call.""" + monkeypatch.delenv("SURFSENSE_LOCAL_TRANSCRIPTION_BASE_URL", raising=False) + assert LLMSettings().transcription_base_url is None diff --git a/surfsense_local/electron/scripts/audiocpp/built-families.mjs b/surfsense_local/electron/scripts/audiocpp/built-families.mjs new file mode 100644 index 0000000000..ecf09e8a69 --- /dev/null +++ b/surfsense_local/electron/scripts/audiocpp/built-families.mjs @@ -0,0 +1,76 @@ +// Proof that a staged server was built with every curated family, which +// --list-devices cannot give. Started on an empty model package, the server +// fails inside the family's loader; a family left out of the build is refused +// by name first. Measured on v0.8.2 builds with and without each family. +import { execFile } from "node:child_process" +import { mkdtempSync, rmSync, writeFileSync } from "node:fs" +import { createServer } from "node:net" +import { tmpdir } from "node:os" +import { join } from "node:path" + +import { FAMILIES } from "./pins.mjs" +import { SERVER } from "./server-files.mjs" + +// A GGUF v3 header with no tensors and no metadata: enough to reach the loader. +const EMPTY_GGUF = Buffer.concat([ + Buffer.from("GGUF"), + Buffer.from([3, 0, 0, 0]), + Buffer.alloc(16), +]) + +/** Whether the server's output shows `family`'s own loader reading the package. */ +export function loaderRan(family, output) { + return ( + output.includes(`model spec for family '${family}'`) && + !output.includes("unsupported model family") + ) +} + +/** A port nothing listens on, in case a server gets as far as listening. */ +function freePort() { + return new Promise((resolve, reject) => { + const server = createServer() + server.once("error", reject) + server.listen(0, "127.0.0.1", () => { + const { port } = server.address() + server.close(() => resolve(port)) + }) + }) +} + +/** Everything the server in `stage` printed while loading one empty `family` model. */ +async function loadOutput(stage, work, family) { + const config = join(work, `${family}.json`) + writeFileSync( + config, + JSON.stringify({ + lazy_load: false, + models: [{ id: "probe", family, path: join(work, "empty.gguf") }], + }) + ) + const args = ["--config", config, "--host", "127.0.0.1", "--port", String(await freePort())] + return new Promise((resolve) => { + execFile( + join(stage, SERVER), + [...args, "--backend", "cpu", "--no-ui"], + { cwd: stage, timeout: 120_000, windowsHide: true }, + (_error, stdout, stderr) => resolve(`${stdout}${stderr}`) + ) + }) +} + +/** Throw unless the server in `stage` has a loader for every curated family. */ +export async function verifyBuiltFamilies(stage) { + const work = mkdtempSync(join(tmpdir(), "surfsense-audiocpp-families-")) + try { + writeFileSync(join(work, "empty.gguf"), EMPTY_GGUF) + for (const family of FAMILIES) { + const output = await loadOutput(stage, work, family) + if (!loaderRan(family, output)) { + throw new Error(`audiocpp_server was not built with ${family}: ${output}`) + } + } + } finally { + rmSync(work, { recursive: true, force: true }) + } +} diff --git a/surfsense_local/electron/scripts/audiocpp/built-families.test.mjs b/surfsense_local/electron/scripts/audiocpp/built-families.test.mjs new file mode 100644 index 0000000000..96eeb63a66 --- /dev/null +++ b/surfsense_local/electron/scripts/audiocpp/built-families.test.mjs @@ -0,0 +1,23 @@ +import assert from "node:assert/strict" +import test from "node:test" + +import { loaderRan } from "./built-families.mjs" + +// What audiocpp_server v0.8.2 printed when started on an empty model package, +// measured on Windows builds with and without each family. +const BUILT = + "audiocpp_server failed: failed to load model resources using builtin model spec for family " + + "'nemotron_asr' source 'gguf': missing model package file 'config': C:\\probe\\config.json" +const LEFT_OUT = "audiocpp_server failed: unsupported model family hint: nemotron_asr" + +test("a family's loader reaching the empty package proves it was built", () => { + assert.equal(loaderRan("nemotron_asr", BUILT), true) +}) + +test("a family the build left out is refused by name before any loader runs", () => { + assert.equal(loaderRan("nemotron_asr", LEFT_OUT), false) +}) + +test("another family's loader proves nothing about this one", () => { + assert.equal(loaderRan("kokoro_tts", BUILT), false) +}) diff --git a/surfsense_local/electron/scripts/audiocpp/pins.mjs b/surfsense_local/electron/scripts/audiocpp/pins.mjs index f846ed7404..2a9ca98a98 100644 --- a/surfsense_local/electron/scripts/audiocpp/pins.mjs +++ b/surfsense_local/electron/scripts/audiocpp/pins.mjs @@ -6,7 +6,8 @@ export const COMMIT = "4d88768fbcae4e6eb3352c6ab1422dabb7d90b58" // pragma: allo export const SOURCE = "https://github.com/0xShug0/audio.cpp.git" // The families the app curates: the only ones built, and their specs ship. -export const FAMILIES = ["kokoro_tts", "supertonic", "kitten_tts"] +// The voices', then Nemotron's for speech recognition. +export const FAMILIES = ["kokoro_tts", "supertonic", "kitten_tts", "nemotron_asr"] // macOS takes upstream's archive, which meets the app's floor there. export const MAC_ARCHIVE = { diff --git a/surfsense_local/electron/scripts/audiocpp/recipe.test.mjs b/surfsense_local/electron/scripts/audiocpp/recipe.test.mjs index 17e6fab1d8..39d4ef4620 100644 --- a/surfsense_local/electron/scripts/audiocpp/recipe.test.mjs +++ b/surfsense_local/electron/scripts/audiocpp/recipe.test.mjs @@ -45,6 +45,13 @@ for (const platform of ["linux", "win32"]) { assert.equal(defs.get("AUDIOCPP_MODEL_SET"), "custom") assert.equal(defs.get("AUDIOCPP_MODELS"), FAMILIES.join(",")) }) + + test(`${platform}: builds Nemotron's family beside the voices, for speech recognition`, () => { + const families = definitions(configureArgs(platform)).get("AUDIOCPP_MODELS").split(",") + for (const family of ["kokoro_tts", "supertonic", "kitten_tts", "nemotron_asr"]) { + assert.ok(families.includes(family), family) + } + }) } test("linux: links libstdc++ into every file, ggml's backend modules included", () => { diff --git a/surfsense_local/electron/scripts/audiocpp/server-files.test.mjs b/surfsense_local/electron/scripts/audiocpp/server-files.test.mjs new file mode 100644 index 0000000000..4c51b3ef65 --- /dev/null +++ b/surfsense_local/electron/scripts/audiocpp/server-files.test.mjs @@ -0,0 +1,37 @@ +import assert from "node:assert/strict" +import { mkdirSync, mkdtempSync, readdirSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import test from "node:test" + +import { copyServerFiles, SERVER } from "./server-files.mjs" + +/** A build folder and a source checkout carrying more model specs than the app curates. */ +function built() { + const root = mkdtempSync(join(tmpdir(), "audiocpp-files-")) + const binaries = join(root, "build", "bin", "Release") + const source = join(root, "source") + const stage = join(root, "stage") + mkdirSync(binaries, { recursive: true }) + mkdirSync(join(source, "model_specs"), { recursive: true }) + mkdirSync(stage) + writeFileSync(join(binaries, SERVER), "") + writeFileSync(join(source, "LICENSE"), "MIT") + for (const family of ["kokoro_tts", "supertonic", "kitten_tts", "nemotron_asr", "qwen3_asr"]) { + writeFileSync(join(source, "model_specs", `${family}.json`), "{}") + } + return { files: { binaries, source }, stage } +} + +test("ships the voices' specs and Nemotron's, and no other family's", () => { + const { files, stage } = built() + + copyServerFiles(files, stage) + + assert.deepEqual(readdirSync(join(stage, "model_specs")).sort(), [ + "kitten_tts.json", + "kokoro_tts.json", + "nemotron_asr.json", + "supertonic.json", + ]) +}) diff --git a/surfsense_local/electron/scripts/audiocpp/stage.mjs b/surfsense_local/electron/scripts/audiocpp/stage.mjs index 770e71218e..6c374f2bc7 100644 --- a/surfsense_local/electron/scripts/audiocpp/stage.mjs +++ b/surfsense_local/electron/scripts/audiocpp/stage.mjs @@ -1,5 +1,6 @@ // `pnpm build:audiocpp`: stage audio.cpp's server and eSpeak-ng into -// electron/audiocpp, swapped in whole once the server lists its devices. +// electron/audiocpp, swapped in whole once the server lists its devices and +// has a loader for every curated family. // macOS downloads upstream's archive; Windows and Linux compile the pinned // source. Without a toolchain the app runs without local audio, unless // --strict, which release CI and `pnpm dist` pass. @@ -9,6 +10,7 @@ import { tmpdir } from "node:os" import { join } from "node:path" import { fileURLToPath } from "node:url" +import { verifyBuiltFamilies } from "./built-families.mjs" import { compile, missingToolchain } from "./compile.mjs" import { stageEspeak } from "./espeak.mjs" import { copyMsvcRuntime } from "../msvc-runtime.mjs" @@ -69,6 +71,7 @@ async function main() { return } + const started = Date.now() const work = mkdtempSync(join(tmpdir(), "surfsense-audiocpp-")) const stage = `${OUT}.stage-${process.pid}` const backup = `${OUT}.old-${process.pid}` @@ -82,6 +85,7 @@ async function main() { if (process.platform === "win32") copyMsvcRuntime(stage) await stageEspeak(work, stage) verifyStage(stage) + await verifyBuiltFamilies(stage) rmSync(backup, { recursive: true, force: true }) if (existsSync(OUT)) renameSync(OUT, backup) @@ -92,7 +96,8 @@ async function main() { throw error } rmSync(backup, { recursive: true, force: true }) - console.log(`audio.cpp ${TAG} staged in ${OUT}: ${megabytes(OUT)} MB`) + const minutes = ((Date.now() - started) / 60_000).toFixed(1) + console.log(`audio.cpp ${TAG} staged in ${OUT}: ${megabytes(OUT)} MB, in ${minutes} min`) } finally { rmSync(work, { recursive: true, force: true }) rmSync(stage, { recursive: true, force: true }) diff --git a/surfsense_local/electron/src/main/sidecars/audiocpp-transcription.test.ts b/surfsense_local/electron/src/main/sidecars/audiocpp-transcription.test.ts new file mode 100644 index 0000000000..d60be7158d --- /dev/null +++ b/surfsense_local/electron/src/main/sidecars/audiocpp-transcription.test.ts @@ -0,0 +1,147 @@ +import assert from "node:assert/strict" +import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs" +import os, { tmpdir } from "node:os" +import { join } from "node:path" +import test from "node:test" + +import { AUDIOCPP_SIDECAR, audiocppSpec, SERVER_CONFIG } from "./audiocpp.ts" +import { + AUDIOCPP_TRANSCRIPTION_SIDECAR, + audiocppTranscriptionSpec, + TRANSCRIPTION_CONFIG, +} from "./audiocpp-transcription.ts" +import { exe } from "./platform.ts" +import type { SidecarContext } from "./types.ts" + +/** A staged build and an audio folder, with the API's transcription config written or not. */ +function staged({ config = true, voices = false } = {}): { + ctx: SidecarContext + audio: string +} { + const root = mkdtempSync(join(tmpdir(), "audiocpp-transcription-")) + const runtime = join(root, "resources", "audiocpp") + const audio = join(root, "audio") + mkdirSync(runtime, { recursive: true }) + mkdirSync(audio, { recursive: true }) + writeFileSync(join(runtime, exe("audiocpp_server")), "") + if (config) writeFileSync(join(audio, TRANSCRIPTION_CONFIG), '{"models":[]}') + if (voices) writeFileSync(join(audio, SERVER_CONFIG), '{"models":[]}') + return { + ctx: { + packaged: true, + backendDir: root, + binariesDir: join(root, "resources"), + host: "127.0.0.1", + apiPort: 1, + dataDir: root, + secret: "s", + audioPort: 9998, + transcriptionPort: 9997, + audioModelsDir: audio, + audioBinariesDir: runtime, + }, + audio, + } +} + +/** The value that follows a flag in a spec's args. */ +function flag(args: string[], name: string): string | undefined { + const at = args.indexOf(name) + return at === -1 ? undefined : args[at + 1] +} + +test("runs the binary the voices' server runs, staged once", () => { + const { ctx } = staged({ voices: true }) + const spec = audiocppTranscriptionSpec(ctx) + assert.ok(spec) + assert.equal(spec.cmd, audiocppSpec(ctx)?.cmd) +}) + +test("runs beside the voices' server under a name of its own", () => { + // The supervisor keys children by name: one name would make each replace the other. + const { ctx } = staged({ voices: true }) + const spec = audiocppTranscriptionSpec(ctx) + assert.ok(spec) + assert.equal(spec.name, AUDIOCPP_TRANSCRIPTION_SIDECAR) + assert.notEqual(spec.name, AUDIOCPP_SIDECAR) +}) + +test("runs in dev too, because only the binary's path differs", () => { + const { ctx } = staged() + assert.ok(audiocppTranscriptionSpec({ ...ctx, packaged: false })) +}) + +test("does not run when the pinned build is not staged", () => { + const { ctx } = staged() + assert.equal( + audiocppTranscriptionSpec({ + ...ctx, + audioBinariesDir: join(ctx.binariesDir, "absent"), + }), + null + ) +}) + +test("does not run until the API names a speech model, whatever voices are installed", () => { + // The server refuses an empty model list, and the voices' config is not its own. + const { ctx } = staged({ config: false, voices: true }) + assert.equal(audiocppTranscriptionSpec(ctx), null) +}) + +test("reads its own config on its own port, so a voice install never restarts it", () => { + const { ctx, audio } = staged({ voices: true }) + const spec = audiocppTranscriptionSpec(ctx) + assert.ok(spec) + assert.equal(flag(spec.args, "--config"), join(audio, TRANSCRIPTION_CONFIG)) + assert.equal(flag(spec.args, "--host"), "127.0.0.1") + assert.equal(flag(spec.args, "--port"), "9997") +}) + +test("holds one speech model, unloads it after 5 idle minutes, and keeps 1 GiB free", () => { + const { ctx } = staged() + const spec = audiocppTranscriptionSpec(ctx) + assert.ok(spec) + assert.equal(flag(spec.args, "--max-loaded-models"), "1") + assert.equal(flag(spec.args, "--idle-unload-ms"), "300000") + assert.equal(flag(spec.args, "--min-free-memory-mb"), "1024") + assert.ok(spec.args.includes("--no-ui")) +}) + +test("runs on the CPU, never on its CUDA default", () => { + // The GPU is the chat model's (ADR 0012). + const { ctx } = staged() + const spec = audiocppTranscriptionSpec(ctx) + assert.ok(spec) + assert.equal(flag(spec.args, "--backend"), "cpu") +}) + +test("passes half of this machine's logical cores, capped at 8", (t) => { + const { ctx } = staged() + const threads = (cores: number) => { + t.mock.method(os, "availableParallelism", () => cores) + const spec = audiocppTranscriptionSpec(ctx) + assert.ok(spec) + return flag(spec.args, "--threads") + } + assert.equal(threads(12), "6") + assert.equal(threads(32), "8") +}) + +test("is not pointed at eSpeak, which only the voices phonemise through", () => { + const { ctx } = staged() + const spec = audiocppTranscriptionSpec(ctx) + assert.ok(spec) + assert.deepEqual(spec.env, {}) +}) + +test("runs from its own directory, where its ggml libraries are", () => { + const { ctx } = staged() + const spec = audiocppTranscriptionSpec(ctx) + assert.ok(spec) + assert.equal(spec.cwd, ctx.audioBinariesDir) +}) + +test("does not run without a port of its own", () => { + const { ctx } = staged() + assert.equal(audiocppTranscriptionSpec({ ...ctx, transcriptionPort: undefined }), null) +}) diff --git a/surfsense_local/electron/src/main/sidecars/audiocpp-transcription.ts b/surfsense_local/electron/src/main/sidecars/audiocpp-transcription.ts new file mode 100644 index 0000000000..01be6853d6 --- /dev/null +++ b/surfsense_local/electron/src/main/sidecars/audiocpp-transcription.ts @@ -0,0 +1,63 @@ +/** + * audio.cpp's server a second time, for speech recognition. Its own process, + * config and port keep a voice install, a podcast's unload or a busy voice + * model from touching a transcription, and the reverse. Only the API calls it. + */ +import { existsSync } from "node:fs" +import os from "node:os" +import { join } from "node:path" + +import { audioThreads, binaryPath } from "./audiocpp.ts" +import type { SidecarContext, SidecarSpec } from "./types.ts" + +export const AUDIOCPP_TRANSCRIPTION_SIDECAR = "audiocpp-transcription" + +/** The speech models the API writes, beside the voices' server.json. */ +export const TRANSCRIPTION_CONFIG = "transcription.json" + +export function audiocppTranscriptionSpec(ctx: SidecarContext): SidecarSpec | null { + if ( + ctx.transcriptionPort == null || + ctx.audioModelsDir == null || + ctx.audioBinariesDir == null + ) { + return null + } + const binary = binaryPath(ctx) + const config = join(ctx.audioModelsDir, TRANSCRIPTION_CONFIG) + // It refuses an empty model list, so it waits for the API to name one. + if (!existsSync(binary) || !existsSync(config)) return null + + return { + name: AUDIOCPP_TRANSCRIPTION_SIDECAR, + cmd: binary, + // These win over the config file, as for the voices' server. + args: [ + "--config", + config, + "--host", + ctx.host, + "--port", + String(ctx.transcriptionPort), + // The GPU is the chat model's. + "--backend", + "cpu", + "--threads", + String(audioThreads(os.availableParallelism())), + // One speech model at a time; the API unloads it when a file is done and + // no dictation is open, and five idle minutes are the backstop. + "--max-loaded-models", + "1", + "--idle-unload-ms", + "300000", + // A backstop behind the API's check against the model's measured peak. + "--min-free-memory-mb", + "1024", + "--no-ui", + ], + // ggml loads its backend libraries from beside the executable. + cwd: ctx.audioBinariesDir, + // No eSpeak: only the voices phonemise. + env: {}, + } +} diff --git a/surfsense_local/electron/src/main/sidecars/boot.ts b/surfsense_local/electron/src/main/sidecars/boot.ts index 60c4c1364a..5535053b0a 100644 --- a/surfsense_local/electron/src/main/sidecars/boot.ts +++ b/surfsense_local/electron/src/main/sidecars/boot.ts @@ -13,6 +13,7 @@ import { join } from "node:path" import { getFreePort, waitForHealth } from "../net.ts" import { audiocppSpec } from "./audiocpp.ts" +import { audiocppTranscriptionSpec } from "./audiocpp-transcription.ts" import { llamacppSpec } from "./llamacpp.ts" import { binaryPath as opencodeBinaryPath, @@ -34,6 +35,7 @@ import type { CrashHandler, SidecarContext, SidecarSpec } from "./types.ts" import { watchAudioModels } from "./watch-audio-models.ts" import { watchGenerationPreset } from "./watch-generation-preset.ts" import { watchImageModel } from "./watch-image-model.ts" +import { watchTranscriptionModels } from "./watch-transcription-models.ts" /** What only the host knows: its paths, the packaged flag and the secret. */ export interface BootContext { @@ -111,6 +113,9 @@ export async function bootSidecars( ctx.audioUrl = `http://${host}:${ctx.audioPort}` ctx.audioModelsDir = join(dataDir, "audio") mkdirSync(ctx.audioModelsDir, { recursive: true }) + // The same build again for speech recognition, on a port of its own. + ctx.transcriptionPort = await getFreePort(host) + ctx.transcriptionUrl = `http://${host}:${ctx.transcriptionPort}` // Only a host with a staged sd-server gets an images dir: without it the API // offers no image models, rather than downloads that can never run. @@ -151,6 +156,8 @@ export async function bootSidecars( llamacppSpec(ctx), // Null until the API's config names an audio model. audiocppSpec(ctx), + // Null until the API's transcription config names a speech model. + audiocppTranscriptionSpec(ctx), ].filter((s): s is SidecarSpec => s !== null) const sidecars = startAll(specs, onCrash) const control = sidecarControl(sidecars, onCrash) @@ -160,6 +167,7 @@ export async function bootSidecars( watchImageModel({ ctx, control, stopping: () => stopping }), watchGenerationPreset({ ctx, control, stopping: () => stopping }), watchAudioModels({ ctx, control, stopping: () => stopping }), + watchTranscriptionModels({ ctx, control, stopping: () => stopping }), watchAgentConfig({ ctx, control, stopping: () => stopping }), ] diff --git a/surfsense_local/electron/src/main/sidecars/python.test.ts b/surfsense_local/electron/src/main/sidecars/python.test.ts index 9e1b4e6db5..4d0c2f907a 100644 --- a/surfsense_local/electron/src/main/sidecars/python.test.ts +++ b/surfsense_local/electron/src/main/sidecars/python.test.ts @@ -128,3 +128,34 @@ test("without a staged opencode the API is told of no agent", () => { assert.equal(api.SURFSENSE_LOCAL_OPENCODE_URL, undefined) assert.equal(api.SURFSENSE_LOCAL_OPENCODE_PASSWORD, undefined) }) + +/** The context boot builds on a host where audio.cpp is staged. */ +function withTranscription(): SidecarContext { + return { + ...withAudio(), + transcriptionPort: 9997, + transcriptionUrl: "http://127.0.0.1:9997", + } +} + +test("the API is told where the speech recognition server is", () => { + const api = apiSpec(withTranscription()).env + + assert.equal(api.SURFSENSE_LOCAL_TRANSCRIPTION_BASE_URL, "http://127.0.0.1:9997") +}) + +test("a worker is never told where the speech recognition server is", () => { + // Every request goes through the API's gate, which puts dictation first. + for (const queue of ["ingest", "studio"] as const) { + const worker = workerSpec(withTranscription(), queue).env + + assert.equal(worker.SURFSENSE_LOCAL_TRANSCRIPTION_BASE_URL, undefined, queue) + } +}) + +test("without a staged audio.cpp the API is told of no speech recognition server", () => { + const ctx = withTranscription() + const api = apiSpec({ ...ctx, audioBinariesDir: join(ctx.binariesDir, "absent") }).env + + assert.equal(api.SURFSENSE_LOCAL_TRANSCRIPTION_BASE_URL, undefined) +}) diff --git a/surfsense_local/electron/src/main/sidecars/python.ts b/surfsense_local/electron/src/main/sidecars/python.ts index b4f9068dbc..38d1c707b4 100644 --- a/surfsense_local/electron/src/main/sidecars/python.ts +++ b/surfsense_local/electron/src/main/sidecars/python.ts @@ -11,6 +11,11 @@ import { binaryPath as audiocppBinary, espeakPaths } from "./audiocpp.ts" import { exe, isWindows } from "./platform.ts" import type { SidecarContext, SidecarSpec } from "./types.ts" +/** Whether audio.cpp's server is staged; without it there are no voices and no speech recognition. */ +function audiocppStaged(ctx: SidecarContext): boolean { + return ctx.audioBinariesDir != null && existsSync(audiocppBinary(ctx)) +} + function pythonEnv(ctx: SidecarContext): Record { return { PYTHONUNBUFFERED: "1", // unbuffered: startup + crash logs show immediately @@ -40,7 +45,7 @@ function pythonEnv(ctx: SidecarContext): Record { ...(ctx.audioModelsDir && ctx.audioUrl && ctx.audioBinariesDir && - existsSync(audiocppBinary(ctx)) && { + audiocppStaged(ctx) && { SURFSENSE_LOCAL_AUDIO_MODELS_DIR: ctx.audioModelsDir, SURFSENSE_LOCAL_AUDIO_BASE_URL: ctx.audioUrl, SURFSENSE_LOCAL_AUDIO_ESPEAK_LIBRARY: espeakPaths(ctx.audioBinariesDir).library, @@ -83,6 +88,11 @@ export function apiSpec(ctx: SidecarContext): SidecarSpec { }), // The API alone checks it: only Electron may print the agent's Word previews. ...(ctx.docxSnapshotKey && { SURFSENSE_LOCAL_DOCX_SNAPSHOT_KEY: ctx.docxSnapshotKey }), + // The API alone calls the speech recognition server; workers go through its gate. + ...(ctx.transcriptionUrl && + audiocppStaged(ctx) && { + SURFSENSE_LOCAL_TRANSCRIPTION_BASE_URL: ctx.transcriptionUrl, + }), }, } } diff --git a/surfsense_local/electron/src/main/sidecars/types.ts b/surfsense_local/electron/src/main/sidecars/types.ts index a1c06af2ad..3ed1b2c1af 100644 --- a/surfsense_local/electron/src/main/sidecars/types.ts +++ b/surfsense_local/electron/src/main/sidecars/types.ts @@ -59,6 +59,9 @@ export interface SidecarContext { audioUrl?: string audioModelsDir?: string audioBinariesDir?: string + /** The same build run again for speech recognition, on its own port; only the API is told the URL. */ + transcriptionPort?: number + transcriptionUrl?: string /** The staged opencode and its ripgrep: `scripts/opencode/stage.mjs` writes it. */ opencodeBinariesDir?: string /** diff --git a/surfsense_local/electron/src/main/sidecars/watch-transcription-models.test.ts b/surfsense_local/electron/src/main/sidecars/watch-transcription-models.test.ts new file mode 100644 index 0000000000..fb0d5523b7 --- /dev/null +++ b/surfsense_local/electron/src/main/sidecars/watch-transcription-models.test.ts @@ -0,0 +1,298 @@ +import assert from "node:assert/strict" +import type { ChildProcess } from "node:child_process" +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import test from "node:test" + +import { SERVER_CONFIG } from "./audiocpp.ts" +import { + AUDIOCPP_TRANSCRIPTION_SIDECAR, + TRANSCRIPTION_CONFIG, +} from "./audiocpp-transcription.ts" +import { exe } from "./platform.ts" +import type { SidecarControl, Sidecars } from "./supervisor.ts" +import type { SidecarContext, SidecarSpec } from "./types.ts" +import { watchTranscriptionModels } from "./watch-transcription-models.ts" + +/** A staged audio.cpp build and an audio folder, no speech model named yet. */ +function staged(): { ctx: SidecarContext; config: string; voices: string } { + const root = mkdtempSync(join(tmpdir(), "watch-transcription-")) + const binaries = join(root, "audiocpp") + const audio = join(root, "audio") + mkdirSync(binaries, { recursive: true }) + mkdirSync(audio, { recursive: true }) + writeFileSync(join(binaries, exe("audiocpp_server")), "") + return { + ctx: { + packaged: true, + backendDir: root, + binariesDir: root, + host: "127.0.0.1", + apiPort: 1, + dataDir: root, + secret: "s", + audioPort: 9998, + transcriptionPort: 9997, + audioModelsDir: audio, + audioBinariesDir: binaries, + }, + config: join(audio, TRANSCRIPTION_CONFIG), + voices: join(audio, SERVER_CONFIG), + } +} + +/** A child as the supervisor keeps it: alive until it exits. */ +type FakeChild = { exitCode: number | null; signalCode: NodeJS.Signals | null } + +/** A control that records what a watcher does, instead of spawning anything. */ +function recording(): { + control: SidecarControl + started: SidecarSpec[] + stopped: string[] + running: Sidecars + /** The last child started, exited as a crash leaves it: still in `running`. */ + crash: () => void +} { + const running: Sidecars = new Map() + const started: SidecarSpec[] = [] + const stopped: string[] = [] + let last: FakeChild | undefined + return { + running, + started, + stopped, + crash: () => { + if (last) last.exitCode = 1 + }, + control: { + running, + start: (spec) => { + started.push(spec) + last = { exitCode: null, signalCode: null } + running.set(spec.name, last as unknown as ChildProcess) + }, + stop: async (name) => { + stopped.push(name) + running.delete(name) + }, + }, + } +} + +/** Resolves once `condition` holds, or fails after `ms`. */ +async function until(condition: () => boolean, ms = 3000): Promise { + const deadline = Date.now() + ms + while (!condition()) { + if (Date.now() > deadline) throw new Error("condition never held") + await new Promise((resolve) => setTimeout(resolve, 5)) + } +} + +/** Wait long enough for a few short-interval polls to have run. */ +const settle = (ms = 60): Promise => + new Promise((resolve) => setTimeout(resolve, ms)) + +/** A watcher on a fast poll and a clock the test moves. */ +function watching() { + const setup = staged() + const rec = recording() + const clock = { now: 0 } + const stop = watchTranscriptionModels({ + ctx: setup.ctx, + control: rec.control, + stopping: () => false, + pollMs: 5, + now: () => clock.now, + }) + return { ...setup, rec, clock, stop } +} + +test("starts the speech recognition server once the API names a speech model", async () => { + const { config, rec, stop } = watching() + try { + writeFileSync(config, '{"models":[1]}') + await until(() => rec.started.length === 1) + assert.equal(rec.started[0].name, AUDIOCPP_TRANSCRIPTION_SIDECAR) + assert.deepEqual(rec.stopped, []) + } finally { + stop() + } +}) + +test("restarts it when the API rewrites its config", async () => { + const { config, rec, stop } = watching() + try { + writeFileSync(config, '{"models":[1]}') + await until(() => rec.started.length === 1) + writeFileSync(config, '{"models":[1,2]}') + await until(() => rec.started.length === 2) + assert.deepEqual(rec.stopped, [AUDIOCPP_TRANSCRIPTION_SIDECAR]) + } finally { + stop() + } +}) + +test("stops it when the last speech model is gone", async () => { + const { config, rec, stop } = watching() + try { + writeFileSync(config, '{"models":[1]}') + await until(() => rec.started.length === 1) + rmSync(config) + await until(() => rec.stopped.length === 1) + await settle() + assert.equal(rec.started.length, 1) + assert.equal(rec.running.has(AUDIOCPP_TRANSCRIPTION_SIDECAR), false) + } finally { + stop() + } +}) + +test("starts nothing while no speech model is named", async () => { + const { rec, stop } = watching() + try { + await settle() + assert.deepEqual(rec.started, []) + assert.deepEqual(rec.stopped, []) + } finally { + stop() + } +}) + +test("a voice install leaves it running", async () => { + // The API rewrites server.json on every voice install and at each start. + const { config, voices, rec, stop } = watching() + try { + writeFileSync(config, '{"models":[1]}') + await until(() => rec.started.length === 1) + writeFileSync(voices, '{"models":[1]}') + await settle() + assert.equal(rec.started.length, 1) + assert.deepEqual(rec.stopped, []) + } finally { + stop() + } +}) + +test("restarts it after a crash, at most once every 10 s", async () => { + const { config, rec, clock, stop } = watching() + try { + writeFileSync(config, '{"models":[1]}') + await until(() => rec.started.length === 1) + clock.now = 4_000 + rec.crash() + await settle() + assert.equal(rec.started.length, 1) + + clock.now = 10_000 + await until(() => rec.started.length === 2) + assert.equal(rec.started[1].name, AUDIOCPP_TRANSCRIPTION_SIDECAR) + } finally { + stop() + } +}) + +test("gives up after 3 crashes in 5 minutes", async () => { + const { config, rec, clock, stop } = watching() + try { + writeFileSync(config, '{"models":[1]}') + await until(() => rec.started.length === 1) + for (const at of [20_000, 40_000]) { + clock.now = at + rec.crash() + await until(() => rec.started.length === (at === 20_000 ? 2 : 3)) + } + clock.now = 60_000 + rec.crash() + await settle() + // Not even once the crashes are long past. + clock.now = 10 * 60_000 + await settle() + assert.equal(rec.started.length, 3) + } finally { + stop() + } +}) + +test("keeps restarting crashes spread wider than 5 minutes", async () => { + const { config, rec, clock, stop } = watching() + try { + writeFileSync(config, '{"models":[1]}') + await until(() => rec.started.length === 1) + for (const [at, starts] of [ + [3 * 60_000, 2], + [6 * 60_000, 3], + [9 * 60_000, 4], + ]) { + clock.now = at + rec.crash() + await until(() => rec.started.length === starts) + } + } finally { + stop() + } +}) + +test("a new speech model brings it back after it gave up", async () => { + const { config, rec, clock, stop } = watching() + try { + writeFileSync(config, '{"models":[1]}') + await until(() => rec.started.length === 1) + for (const [at, starts] of [ + [20_000, 2], + [40_000, 3], + ]) { + clock.now = at + rec.crash() + await until(() => rec.started.length === starts) + } + clock.now = 60_000 + rec.crash() + await settle() + assert.equal(rec.started.length, 3) + + writeFileSync(config, '{"models":[2]}') + await until(() => rec.started.length === 4) + } finally { + stop() + } +}) + +test("restarts nothing while the app is quitting", async () => { + const { ctx, config } = staged() + const rec = recording() + const clock = { now: 0 } + let quitting = false + const stop = watchTranscriptionModels({ + ctx, + control: rec.control, + stopping: () => quitting, + pollMs: 5, + now: () => clock.now, + }) + try { + writeFileSync(config, '{"models":[1]}') + await until(() => rec.started.length === 1) + quitting = true + rec.crash() + clock.now = 60_000 + writeFileSync(config, '{"models":[1,2]}') + await settle() + assert.equal(rec.started.length, 1) + } finally { + stop() + } +}) + +test("watches nothing without an audio folder", () => { + const { ctx } = staged() + const rec = recording() + const stop = watchTranscriptionModels({ + ctx: { ...ctx, audioModelsDir: undefined }, + control: rec.control, + stopping: () => false, + }) + + assert.equal(typeof stop, "function") + stop() +}) diff --git a/surfsense_local/electron/src/main/sidecars/watch-transcription-models.ts b/surfsense_local/electron/src/main/sidecars/watch-transcription-models.ts new file mode 100644 index 0000000000..e206114e9e --- /dev/null +++ b/surfsense_local/electron/src/main/sidecars/watch-transcription-models.ts @@ -0,0 +1,96 @@ +/** + * The speech recognition server runs while the API's transcription.json names + * a model: started when it appears, restarted when it changes, stopped when it + * goes. Unlike the voices' server it comes back after a crash, since a + * recording or a dictation waits on it, but not forever. + */ +import type { ChildProcess } from "node:child_process" +import { join } from "node:path" + +import { sessionLog } from "../session-log/session-log.ts" +import { warnMain } from "../session-log/main-warn.ts" +import { + AUDIOCPP_TRANSCRIPTION_SIDECAR, + TRANSCRIPTION_CONFIG, + audiocppTranscriptionSpec, +} from "./audiocpp-transcription.ts" +import { fileStamp } from "./file-stamp.ts" +import type { WatcherOptions } from "./supervisor.ts" + +const POLL_MS = 5000 +// As opencode's: a server that dies at once is not restarted in a tight loop. +const RESTART_MS = 10_000 +// Past this, it is left stopped and the API reports transcription_unavailable; +// a new config, as when another speech model is chosen, tries again. +const CRASH_LIMIT = 3 +const CRASH_WINDOW_MS = 5 * 60_000 + +export interface TranscriptionWatcherOptions extends WatcherOptions { + /** The clock restarts are spaced and crashes counted on; a test moves it. */ + now?: () => number +} + +/** Follow the API's speech model config and restart the server after a crash; returns the stop. */ +export function watchTranscriptionModels(options: TranscriptionWatcherOptions): () => void { + const { ctx, control, stopping, pollMs = POLL_MS, now = Date.now } = options + if (ctx.audioModelsDir == null) return () => {} + const config = join(ctx.audioModelsDir, TRANSCRIPTION_CONFIG) + let current = fileStamp(config) + let lastStart = now() + let crashes: number[] = [] + let counted: ChildProcess | undefined + let gaveUp = false + + const start = () => { + const spec = audiocppTranscriptionSpec(ctx) + if (!spec) return + control.start(spec) + lastStart = now() + } + + /** Count a crash once per child; true once the server has crashed too often to restart. */ + const tooManyCrashes = (child: ChildProcess): boolean => { + if (child === counted) return gaveUp + counted = child + const at = now() + crashes = [...crashes.filter((t) => at - t < CRASH_WINDOW_MS), at] + if (crashes.length >= CRASH_LIMIT) { + gaveUp = true + warnMain( + sessionLog, + `${AUDIOCPP_TRANSCRIPTION_SIDECAR} crashed ${CRASH_LIMIT} times in 5 minutes; not restarting it` + ) + } + return gaveUp + } + + /** Restart on a config change, or after a crash while the budget allows. */ + const reconcile = async () => { + if (stopping()) return + const stamp = fileStamp(config) + if (stamp !== current) { + current = stamp + crashes = [] + gaveUp = false + if (control.running.has(AUDIOCPP_TRANSCRIPTION_SIDECAR)) { + await control.stop(AUDIOCPP_TRANSCRIPTION_SIDECAR) + } + start() + return + } + + const child = control.running.get(AUDIOCPP_TRANSCRIPTION_SIDECAR) + if (child == null || (child.exitCode === null && child.signalCode === null)) return + if (tooManyCrashes(child)) return + if (now() - lastStart < RESTART_MS) return + start() + } + + const timer = setInterval(() => { + void reconcile().catch(() => { + // Mid-write or mid-restart; the next tick tries again. + }) + }, pollMs) + timer.unref() + return () => clearInterval(timer) +} From 4397a531556a1264a3e7fe54febffd7781ef415b Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Wed, 14 Oct 2026 01:55:29 -0700 Subject: [PATCH 03/17] fix(models): a speech section that fails honestly and never waits on servers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the Speech recognition section found two wrong screens and a glossary row at odds with the app. - A local catalog that fails to load left the section saying "This computer can't run one", which is false. With no server to fall back on, the slot has nothing else to show, so the catalog's error now becomes the section's error, as a failed selection already did. - `ModelSlotSettings` still read the connections query to decide whether a `localOnly` slot is empty, so a failed or slow connections request hid the empty state behind "No speech recognition model chosen". A local-only slot no longer counts servers. - The glossary's new transcript row gave Korean 대본 (a script to perform) and Chinese 转录文本, where the podcast viewer already says 스크립트 and 文字稿 for "Transcript"; the row now matches the strings in use. - The Audio page gains a test that a recogniser, on disk or downloadable, never appears there, as the catalog doc says. --- .agents/skills/translate/glossary.md | 2 +- .../models/your-models/use-speech-models.ts | 3 +- .../models/audio-models-settings.test.tsx | 27 ++++++++++++++ .../settings/models/model-slot-settings.tsx | 2 +- .../models/speech-models-settings.test.tsx | 36 +++++++++++++++++++ 5 files changed, 67 insertions(+), 3 deletions(-) diff --git a/.agents/skills/translate/glossary.md b/.agents/skills/translate/glossary.md index e2fffd1c3b..5e6b106603 100644 --- a/.agents/skills/translate/glossary.md +++ b/.agents/skills/translate/glossary.md @@ -43,7 +43,7 @@ SurfSense, Studio, Markdown, llama.cpp, GGUF, Hugging Face, API, MCP, and every | speech (a model reading text aloud) | 音声合成 | 음성 | 语音 | वॉइस | Sprache | voz | voix | voz | речь | | speech recognition (turning recordings and the voice into text, on this computer) | 音声認識 | 음성 인식 | 语音识别 | स्पीच रिकग्निशन | Spracherkennung | reconocimiento de voz | reconnaissance vocale | reconhecimento de voz | распознавание речи | | voice input (dictating into the chat composer) | 音声入力 | 음성 입력 | 语音输入 | वॉइस इनपुट | Spracheingabe | entrada de voz | saisie vocale | entrada de voz | голосовой ввод | -| transcript (a recording’s text) | 文字起こし | 대본 | 转录文本 | ट्रांसक्रिप्ट | Transkript | transcripción | transcription | transcrição | расшифровка | +| transcript (a recording’s text) | 文字起こし | 스크립트 | 文字稿 | ट्रांसक्रिप्ट | Transkript | transcripción | transcription | transcrição | расшифровка | | sidebar (the workspace’s left column) | サイドバー | 사이드바 | 侧边栏 | साइडबार | Seitenleiste | barra lateral | barre latérale | barra lateral | боковая панель | | workspace | ワークスペース | 워크스페이스 | 工作区 | वर्कस्पेस | Workspace | espacio de trabajo | espace de travail | Workspace | рабочая область | | artifact (a Studio output) | 生成物 | 생성물 | 产物 | आर्टिफ़ैक्ट | Artefakt | artefacto | artefact | artefato | артефакт | diff --git a/surfsense_local/frontend/src/features/models/your-models/use-speech-models.ts b/surfsense_local/frontend/src/features/models/your-models/use-speech-models.ts index 93df77d2d4..6768d6128d 100644 --- a/surfsense_local/frontend/src/features/models/your-models/use-speech-models.ts +++ b/surfsense_local/frontend/src/features/models/your-models/use-speech-models.ts @@ -38,7 +38,8 @@ export function useSpeechModels(): YourModels { // No server serves this slot, so there is none to name. inUse: describeInUse(selection.data, local, undefined), isPending: catalog.isPending || selection.isPending, - error: selection.error, + // With no server to fall back on, a failed catalog leaves nothing to show. + error: selection.error ?? catalog.error, canDownload: models.length > 0, } } diff --git a/surfsense_local/frontend/src/features/settings/models/audio-models-settings.test.tsx b/surfsense_local/frontend/src/features/settings/models/audio-models-settings.test.tsx index 99d3e32e3f..2d61ce06c0 100644 --- a/surfsense_local/frontend/src/features/settings/models/audio-models-settings.test.tsx +++ b/surfsense_local/frontend/src/features/settings/models/audio-models-settings.test.tsx @@ -3,6 +3,10 @@ import { cleanup, screen, waitFor } from "@testing-library/react" import userEvent from "@testing-library/user-event" import { fakeInstallApi } from "@/features/models/local/installs/fake-install-api" +import { + fakeSpeechRow, + SPEECH_INSTALLED, +} from "@/features/models/local/transcription/fake-speech-rows" import { render } from "@/test-utils" import { AudioModelsSettings } from "./audio-models-settings" @@ -169,6 +173,29 @@ describe("audio model settings", () => { expect(facts("8 voices").at(-1)).toBe("English") }) + it("leaves speech recognition models to their own section", async () => { + vi.stubGlobal( + "fetch", + serving([ + audioRow(), + fakeSpeechRow({}, SPEECH_INSTALLED), + fakeSpeechRow( + { id: "nemotron-other", name: "Nemotron Other" }, + { catalog_id: "opaque-other" } + ), + ]) + ) + const user = userEvent.setup() + render( undefined} />) + + // Neither the one on disk nor the one to download is a voice. + expect(await screen.findByText("No audio model yet")).toBeTruthy() + expect(screen.queryByText(/Nemotron/)).toBeNull() + await user.click(screen.getByRole("button", { name: "Add model" })) + expect(await screen.findByText("Kokoro 82M")).toBeTruthy() + expect(screen.queryByText(/Nemotron/)).toBeNull() + }) + it("uses a downloaded audio model through audio.cpp", async () => { const fetchMock = serving([ audioRow({}, { installed_as: "kokoro-82m-q8_0" }), diff --git a/surfsense_local/frontend/src/features/settings/models/model-slot-settings.tsx b/surfsense_local/frontend/src/features/settings/models/model-slot-settings.tsx index 1bd369174a..a0fa242944 100644 --- a/surfsense_local/frontend/src/features/settings/models/model-slot-settings.tsx +++ b/surfsense_local/frontend/src/features/settings/models/model-slot-settings.tsx @@ -147,7 +147,7 @@ export function ModelSlotSettings({ const empty = !models.isPending && - connections.data?.length === 0 && + (localOnly || connections.data?.length === 0) && models.local.length === 0 && !pending && models.inUse === null diff --git a/surfsense_local/frontend/src/features/settings/models/speech-models-settings.test.tsx b/surfsense_local/frontend/src/features/settings/models/speech-models-settings.test.tsx index 65888a56f7..eca87e9632 100644 --- a/surfsense_local/frontend/src/features/settings/models/speech-models-settings.test.tsx +++ b/surfsense_local/frontend/src/features/settings/models/speech-models-settings.test.tsx @@ -109,6 +109,42 @@ describe("speech recognition settings", () => { expect(screen.queryByText(/Kokoro/)).toBeNull() }) + it("says the models could not load, not that this computer cannot run one", async () => { + vi.stubGlobal( + "fetch", + serving([], (path) => + path === "/llm/catalog/local" + ? Response.json({ detail: "catalog failed" }, { status: 500 }) + : null + ) + ) + renderSettings() + + expect( + await screen.findByText("Could not load model settings") + ).toBeTruthy() + expect(screen.queryByText("This computer can’t run one.")).toBeNull() + }) + + it("shows an empty slot without waiting on servers it never offers", async () => { + vi.stubGlobal( + "fetch", + serving([fakeSpeechRow()], (path) => + path === "/llm/connections" + ? Response.json({ detail: "connections failed" }, { status: 500 }) + : null + ) + ) + renderSettings() + + expect( + await screen.findByText("No speech recognition model yet") + ).toBeTruthy() + expect( + screen.getByText("Download one to run on this computer.") + ).toBeTruthy() + }) + it("describes each model by what transcribing takes and what it understands", async () => { vi.stubGlobal("fetch", serving([fakeSpeechRow(), voiceRow])) const user = userEvent.setup() From 75603e3b5e50d86db40bd6b9d24f31f010b8b739 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Wed, 14 Oct 2026 02:17:26 -0700 Subject: [PATCH 04/17] fix(local): never leave a second speech recognition server running The transcription watcher stops the old server and then starts the new one, and that stop can take seconds while the server exits. Two cases started a server that nothing would ever stop: - A poll that came during the stop saw the rewritten config, found no child under the name (the supervisor forgets it before waiting) and started one; the first poll then started another over it, orphaning the first with its port and its model's memory. Polls now skip while one is still running. - A quit that began during the stop stopped only the children it saw, and the poll then started a fresh server after it. A start now checks for the quit again. The proposal's code list also names shared/config.py, which step 2 changed. --- docs/proposals/local-speech-to-text.md | 1 + .../watch-transcription-models.test.ts | 58 +++++++++++++++++++ .../sidecars/watch-transcription-models.ts | 16 ++++- 3 files changed, 72 insertions(+), 3 deletions(-) diff --git a/docs/proposals/local-speech-to-text.md b/docs/proposals/local-speech-to-text.md index 963376e5e9..deabd54a0c 100644 --- a/docs/proposals/local-speech-to-text.md +++ b/docs/proposals/local-speech-to-text.md @@ -11,6 +11,7 @@ code: - surfsense_local/backend/tests/packaging/ - surfsense_local/backend/bundling/ - surfsense_local/backend/pyproject.toml + - surfsense_local/backend/shared/config.py - surfsense_local/electron/src/main/ - surfsense_local/electron/src/preload/ - surfsense_local/electron/scripts/audiocpp/ diff --git a/surfsense_local/electron/src/main/sidecars/watch-transcription-models.test.ts b/surfsense_local/electron/src/main/sidecars/watch-transcription-models.test.ts index fb0d5523b7..73c91f6340 100644 --- a/surfsense_local/electron/src/main/sidecars/watch-transcription-models.test.ts +++ b/surfsense_local/electron/src/main/sidecars/watch-transcription-models.test.ts @@ -53,11 +53,14 @@ function recording(): { running: Sidecars /** The last child started, exited as a crash leaves it: still in `running`. */ crash: () => void + /** Stops wait, as a server slow to exit makes them, until the returned release. */ + holdStops: () => () => void } { const running: Sidecars = new Map() const started: SidecarSpec[] = [] const stopped: string[] = [] let last: FakeChild | undefined + let held: Promise | undefined return { running, started, @@ -65,6 +68,13 @@ function recording(): { crash: () => { if (last) last.exitCode = 1 }, + holdStops: () => { + let release = () => {} + held = new Promise((resolve) => { + release = resolve + }) + return release + }, control: { running, start: (spec) => { @@ -72,9 +82,11 @@ function recording(): { last = { exitCode: null, signalCode: null } running.set(spec.name, last as unknown as ChildProcess) }, + // As the supervisor's: forgotten at once, then waited on until it exits. stop: async (name) => { stopped.push(name) running.delete(name) + await held }, }, } @@ -133,6 +145,26 @@ test("restarts it when the API rewrites its config", async () => { } }) +test("a rewrite while the old server is still stopping starts one server, not two", async () => { + // A second start would replace a running child that nothing then stops. + const { config, rec, stop } = watching() + try { + writeFileSync(config, '{"models":[1]}') + await until(() => rec.started.length === 1) + const release = rec.holdStops() + writeFileSync(config, '{"models":[1,2]}') + await until(() => rec.stopped.length === 1) + writeFileSync(config, '{"models":[1,2,3]}') + await settle() + release() + await until(() => rec.started.length === 3) + await settle() + assert.equal(rec.stopped.length, rec.started.length - 1) + } finally { + stop() + } +}) + test("stops it when the last speech model is gone", async () => { const { config, rec, stop } = watching() try { @@ -284,6 +316,32 @@ test("restarts nothing while the app is quitting", async () => { } }) +test("a restart the quit overtakes starts nothing once the old server stops", async () => { + // The quit stops only the children running when it begins. + const { ctx, config } = staged() + const rec = recording() + let quitting = false + const stop = watchTranscriptionModels({ + ctx, + control: rec.control, + stopping: () => quitting, + pollMs: 5, + }) + try { + writeFileSync(config, '{"models":[1]}') + await until(() => rec.started.length === 1) + const release = rec.holdStops() + writeFileSync(config, '{"models":[1,2]}') + await until(() => rec.stopped.length === 1) + quitting = true + release() + await settle() + assert.equal(rec.started.length, 1) + } finally { + stop() + } +}) + test("watches nothing without an audio folder", () => { const { ctx } = staged() const rec = recording() diff --git a/surfsense_local/electron/src/main/sidecars/watch-transcription-models.ts b/surfsense_local/electron/src/main/sidecars/watch-transcription-models.ts index e206114e9e..43ce38701e 100644 --- a/surfsense_local/electron/src/main/sidecars/watch-transcription-models.ts +++ b/surfsense_local/electron/src/main/sidecars/watch-transcription-models.ts @@ -42,6 +42,8 @@ export function watchTranscriptionModels(options: TranscriptionWatcherOptions): let gaveUp = false const start = () => { + // Checked again after a stop: the quit stops only what runs when it begins. + if (stopping()) return const spec = audiocppTranscriptionSpec(ctx) if (!spec) return control.start(spec) @@ -86,10 +88,18 @@ export function watchTranscriptionModels(options: TranscriptionWatcherOptions): start() } + // A tick during a slow stop would start a second server over the first. + let reconciling = false const timer = setInterval(() => { - void reconcile().catch(() => { - // Mid-write or mid-restart; the next tick tries again. - }) + if (reconciling) return + reconciling = true + void reconcile() + .catch(() => { + // Mid-write or mid-restart; the next tick tries again. + }) + .finally(() => { + reconciling = false + }) }, pollMs) timer.unref() return () => clearInterval(timer) From e6fbf2b482f4cfd550c0cfc09dbc4a1393ed3491 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Wed, 14 Oct 2026 02:20:30 -0700 Subject: [PATCH 05/17] feat(sources): recordings in the sources panel, with how far their transcript is MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 5 of the local speech recognition proposal, frontend half. Audio files can be added as sources, and their rows say how transcription is going, what they wait for, and how search will read them. - `.wav .mp3 .flac .ogg .oga` join the upload mirror of the backend's suffixes, so the panel's and the composer's Upload sources pick them up; `.opus` stays out until a decoder ships, and an Ogg Opus file is refused per file by the server. - `WorkspaceDocument` gains `transcription {language, progress, awaiting}`, and `reuseUnchanged` compares it by value, or every re-read would render every recording's row again. - A documents event may carry a transient `progress`. The panel keeps it on the row without reading the list again when the row already has the event's status, and times the windows for the estimate; any other event still reloads. - A recording's row has a second line (`features/sources/transcription/`): "Transcribing · 23 of 58 min · about 7 min left" over a thin bar while it runs, "Queued · 40% transcribed" once it has made way for other sources, and "Waiting for speech recognition" with Set up, which opens the set-up popover, for one added before any model. That wait is not an error: no alert, no Retry, and the 10-second re-read skips it. - A ready transcript not in English, or in a language not identified, says search finds it best with questions in its own language while the index uses bge-small. No route says which languages an embedder reads, so the frontend knows bge-small by its id. - Recordings get no preview: viewers are keyed by MIME type, and a test now holds that no audio type has one. - Every new string is in all ten catalogs; the glossary gains "recording". The documents doc describes the rows and the progress events. --- .agents/skills/translate/glossary.md | 1 + docs/architecture/documents.md | 3 +- .../src/features/dashboard/dashboard-page.tsx | 8 + .../features/file-viewers/registry.test.ts | 11 + .../frontend/src/features/sources/api.ts | 21 ++ .../features/sources/reuse-unchanged.test.ts | 16 + .../src/features/sources/reuse-unchanged.ts | 26 +- .../features/sources/source-upload.test.tsx | 76 ++++- .../src/features/sources/sources-panel.tsx | 10 + .../transcription/awaiting-speech-line.tsx | 45 +++ .../features/sources/transcription/pace.ts | 32 ++ .../sources/transcription/recording-line.tsx | 53 +++ .../transcription/recording-rows.test.tsx | 305 ++++++++++++++++++ .../transcription/search-caveat-line.tsx | 40 +++ .../transcription-progress-line.tsx | 84 +++++ .../transcription/use-transcription-paces.ts | 41 +++ .../sources/transcription/with-progress.ts | 32 ++ .../features/sources/tree/document-row.tsx | 247 +++++++------- .../src/features/sources/tree/source-tree.tsx | 7 + .../src/features/sources/use-sources.ts | 53 ++- .../src/features/workspaces/read-sse.ts | 11 +- .../workspaces/use-workspace-changes.ts | 13 +- .../workspaces/workspace-changes.test.ts | 23 ++ .../features/workspaces/workspace-changes.ts | 8 +- surfsense_local/frontend/translations/de.json | 8 + surfsense_local/frontend/translations/en.json | 8 + surfsense_local/frontend/translations/es.json | 8 + surfsense_local/frontend/translations/fr.json | 8 + surfsense_local/frontend/translations/hi.json | 8 + surfsense_local/frontend/translations/ja.json | 8 + surfsense_local/frontend/translations/ko.json | 8 + .../frontend/translations/pt-BR.json | 8 + surfsense_local/frontend/translations/ru.json | 8 + .../frontend/translations/zh-CN.json | 8 + 34 files changed, 1100 insertions(+), 146 deletions(-) create mode 100644 surfsense_local/frontend/src/features/sources/transcription/awaiting-speech-line.tsx create mode 100644 surfsense_local/frontend/src/features/sources/transcription/pace.ts create mode 100644 surfsense_local/frontend/src/features/sources/transcription/recording-line.tsx create mode 100644 surfsense_local/frontend/src/features/sources/transcription/recording-rows.test.tsx create mode 100644 surfsense_local/frontend/src/features/sources/transcription/search-caveat-line.tsx create mode 100644 surfsense_local/frontend/src/features/sources/transcription/transcription-progress-line.tsx create mode 100644 surfsense_local/frontend/src/features/sources/transcription/use-transcription-paces.ts create mode 100644 surfsense_local/frontend/src/features/sources/transcription/with-progress.ts diff --git a/.agents/skills/translate/glossary.md b/.agents/skills/translate/glossary.md index 5e6b106603..684dc039eb 100644 --- a/.agents/skills/translate/glossary.md +++ b/.agents/skills/translate/glossary.md @@ -44,6 +44,7 @@ SurfSense, Studio, Markdown, llama.cpp, GGUF, Hugging Face, API, MCP, and every | speech recognition (turning recordings and the voice into text, on this computer) | 音声認識 | 음성 인식 | 语音识别 | स्पीच रिकग्निशन | Spracherkennung | reconocimiento de voz | reconnaissance vocale | reconhecimento de voz | распознавание речи | | voice input (dictating into the chat composer) | 音声入力 | 음성 입력 | 语音输入 | वॉइस इनपुट | Spracheingabe | entrada de voz | saisie vocale | entrada de voz | голосовой ввод | | transcript (a recording’s text) | 文字起こし | 스크립트 | 文字稿 | ट्रांसक्रिप्ट | Transkript | transcripción | transcription | transcrição | расшифровка | +| recording (an audio file added as a source) | 録音 | 녹음 | 录音 | रिकॉर्डिंग | Aufnahme | grabación | enregistrement | gravação | запись | | sidebar (the workspace’s left column) | サイドバー | 사이드바 | 侧边栏 | साइडबार | Seitenleiste | barra lateral | barre latérale | barra lateral | боковая панель | | workspace | ワークスペース | 워크스페이스 | 工作区 | वर्कस्पेस | Workspace | espacio de trabajo | espace de travail | Workspace | рабочая область | | artifact (a Studio output) | 生成物 | 생성물 | 产物 | आर्टिफ़ैक्ट | Artefakt | artefacto | artefact | artefato | артефакт | diff --git a/docs/architecture/documents.md b/docs/architecture/documents.md index 18dff26660..276dc49927 100644 --- a/docs/architecture/documents.md +++ b/docs/architecture/documents.md @@ -64,6 +64,7 @@ The sources panel shows the Library as a tree ([`features/sources/tree/`](../../ - **Rows.** Folders come first, by name, then sources newest first. The list is an ARIA `tree` with one `treeitem` per row: arrow keys move and open, Space ticks, Enter opens, F2 renames and Delete asks to delete. A name filter, opened from the search button in the header, shows matches with the folders holding them opened; Escape or closing it clears it. A name the column cuts off shows whole in a tooltip on hover, or while its row has keyboard focus ([`overflow-tooltip.tsx`](../../surfsense_local/frontend/src/components/ui/overflow-tooltip.tsx)). Rows are memoized, and a re-read of the list keeps the object of every unchanged source ([`reuse-unchanged.ts`](../../surfsense_local/frontend/src/features/sources/reuse-unchanged.ts)), so a poll or an event renders only the rows whose source changed. - **Actions.** New folder (from the header's **+** menu, or inside a folder from its menu), Rename, Move to… and Delete. Move to… is a picker of folders and the keyboard's path; a row dragged onto a folder, or onto a source inside one, moves there too. A folder cannot be moved inside itself. Deleting a folder names how many sources, and filed Studio outputs, go with it, as `GET /folders/{folder_id}/summary` counts them, and deletes them for good. +- **Recordings.** A recording, an `audio/*` source, carries `transcription` on its row, and its row has a second line ([`transcription/`](../../surfsense_local/frontend/src/features/sources/transcription)). While it transcribes, "Transcribing · 23 of 58 min · about 7 min left" over a thin bar: the minutes are `transcription.progress`, and the time left is the speed of the windows the panel has heard of since this run started; until two have arrived there is no estimate. A recording that made way for other sources reads "Queued · 40% transcribed". One waiting for a speech recognition model reads "Waiting for speech recognition" with **Set up**, which opens the speech set-up popover ([catalog](local-models/catalog.md)), and shows no error and no Retry: choosing a model queues it again. A ready transcript that is not in English, or whose language was not identified, says search finds it best with questions in its own language and that a chat reads all of it, while the active index's embedder is bge-small. The frontend knows bge-small by its id, `bge-small-en-v1.5`, since no route says which languages an embedder reads. A recording has no preview: viewers are keyed by MIME type, and none takes `audio/*` yet. - **Ticks.** Each row's tick is tri-state. Ticking a folder ticks everything under it, now and added later; the nearest tick up the tree decides ([`scope-state.ts`](../../surfsense_local/frontend/src/features/sources/tree/scope-state.ts)). Chat and Studio send the ticks as `source_scope`, so every ticked source counts, not only the ones loaded. ## Upload @@ -162,7 +163,7 @@ Ingest keeps the pictures in each source, so the agent can place a source's char - The queue survives restarts. Electron starts the workers beside the API and only the API migrates, so each consumer waits until the database is at the latest revision before taking a job ([`wait_for_schema.py`](../../surfsense_local/backend/worker/wait_for_schema.py)); a job queued before an update would otherwise run against the old schema and fail. A job carries only its task's name, so each consumer imports every task before it starts (`import_tasks()`), and [`tests/integration/test_registration.py`](../../surfsense_local/backend/tests/integration/test_registration.py) fails when that list falls behind the task modules on disk. - Jobs are enqueued by upload, note creation, a note content edit and retry, and by import ([`import.md`](import.md)). The queue takes the highest priority first: interactive work at 100, a bulk copy and an import at 10. The figures-only pass, `extract_figures`, shares the queue ([Figures](#figures)). - A source the app quit in the middle of ingesting is failed with `interrupted when the app closed` when the ingest worker next starts, before it takes a job ([`interrupted_documents.py`](../../surfsense_local/backend/worker/interrupted_documents.py)). Huey drops a job as it starts it, so nothing else would end it, and Retry accepts only a failed or cancelled document. A source still `pending` is still queued and is left alone. -- Each transition the ingest worker makes, to `processing`, `ready` or `failed`, sends a `documents` event keyed by document id, and so does each change the API makes itself: a note written, an upload, a rename or edit, a retry, a cancel, and a delete, whose status is `deleted`. The sources panel reloads its list on each event and whenever a dropped stream is back, so a note a plugin writes shows as it is written; it still refetches every 10 seconds while any row is `pending` or `processing`, in case a notice was lost ([`overview.md`](overview.md#freshness)). +- Each transition the ingest worker makes, to `processing`, `ready` or `failed`, sends a `documents` event keyed by document id, and so does each change the API makes itself: a note written, an upload, a rename or edit, a retry, a cancel, and a delete, whose status is `deleted`. The sources panel reloads its list on each event and whenever a dropped stream is back, except for a transcribed window's event, whose transient `progress` it keeps on the row as it is when the row already has the event's status ([Recordings](#the-sources-tree)), so a note a plugin writes shows as it is written; it still refetches every 10 seconds while any row is `pending` or `processing`, in case a notice was lost ([`overview.md`](overview.md#freshness)). ## Known gaps diff --git a/surfsense_local/frontend/src/features/dashboard/dashboard-page.tsx b/surfsense_local/frontend/src/features/dashboard/dashboard-page.tsx index 1867823fdd..196f8423dd 100644 --- a/surfsense_local/frontend/src/features/dashboard/dashboard-page.tsx +++ b/surfsense_local/frontend/src/features/dashboard/dashboard-page.tsx @@ -103,6 +103,7 @@ function WorkspaceDashboard({ onModelSelected, onOpenLicense, onOpenAudioSettings, + onOpenSpeechSettings, onOpenOfficeSupport, onOpenPlugins, modelsVisited, @@ -118,6 +119,7 @@ function WorkspaceDashboard({ onModelSelected: (selection: ModelSelection) => void onOpenLicense: () => void onOpenAudioSettings: () => void + onOpenSpeechSettings: () => void onOpenOfficeSupport: () => void onOpenPlugins: (start?: PluginsStart) => void modelsVisited: number @@ -195,6 +197,7 @@ function WorkspaceDashboard({ // Stable, like everything else the memoized sources panel takes: this page // re-renders when a run starts, ends or changes state, and for a title, a // thread switch, Settings and the other columns. + const openSpeechSettings = useStableCallback(() => onOpenSpeechSettings()) const toggleSourcePreview = useStableCallback((documentId: number) => { const next = sourcePreviewId === documentId ? null : documentId setSourcePreviewId(next) @@ -252,6 +255,8 @@ function WorkspaceDashboard({ onRename={sources.rename} folderActions={sources.folderActions} notes={sourceNotes} + transcriptionPaces={sources.transcriptionPaces} + onOpenSpeechSettings={openSpeechSettings} /> ), @@ -279,6 +284,8 @@ function WorkspaceDashboard({ sources.rename, sources.folderActions, sourceNotes, + sources.transcriptionPaces, + openSpeechSettings, ] ) const sidebarFooter = useMemo( @@ -785,6 +792,7 @@ export function DashboardPage({ onModelSelected={onModelSelected} onOpenLicense={() => openSettings("license")} onOpenAudioSettings={() => openSettings("audio-models")} + onOpenSpeechSettings={() => openSettings("speech-models")} onOpenOfficeSupport={openOfficeSupport} onOpenPlugins={openPlugins} modelsVisited={modelsVisited} diff --git a/surfsense_local/frontend/src/features/file-viewers/registry.test.ts b/surfsense_local/frontend/src/features/file-viewers/registry.test.ts index 88c9553928..9485d91cf3 100644 --- a/surfsense_local/frontend/src/features/file-viewers/registry.test.ts +++ b/surfsense_local/frontend/src/features/file-viewers/registry.test.ts @@ -8,4 +8,15 @@ describe("source file viewers", () => { expect(getFileViewer("text/plain")).toBeNull() expect(getFileViewer(null)).toBeNull() }) + + it("previews no recording until an audio viewer exists", () => { + for (const mimeType of [ + "audio/wav", + "audio/mpeg", + "audio/flac", + "audio/ogg", + ]) { + expect(getFileViewer(mimeType)).toBeNull() + } + }) }) diff --git a/surfsense_local/frontend/src/features/sources/api.ts b/surfsense_local/frontend/src/features/sources/api.ts index 6a9c644a7b..e2e363d9da 100644 --- a/surfsense_local/frontend/src/features/sources/api.ts +++ b/surfsense_local/frontend/src/features/sources/api.ts @@ -3,6 +3,19 @@ import { apiUrl, requestJson, requestVoid } from "@/lib/api" export type DocumentStatus = "pending" | "processing" | "ready" | "failed" | "cancelled" +/** Seconds of a recording's audio transcribed so far, of all it holds. */ +export type TranscriptionProgress = { done_s: number; total_s: number } + +/** Where a recording's transcript stands. */ +export type DocumentTranscription = { + // ISO 639-1; null until known, or when the model could not name it. + language: string | null + // Null before the first window and once the transcript is ready. + progress: TranscriptionProgress | null + // No speech recognition model was chosen when it was last tried. + awaiting: boolean +} + export type WorkspaceDocument = { id: number title: string @@ -14,6 +27,8 @@ export type WorkspaceDocument = { updated_at: string // Absent from an API without folders; null for a source outside any folder. folder_id?: number | null + // Set for a recording (an audio/* source) only; absent from an older API. + transcription?: DocumentTranscription | null } // UX mirror of backend/modules/documents/storage.py UPLOAD_MIME_BY_SUFFIX. @@ -38,6 +53,12 @@ export const SUPPORTED_SOURCE_EXTENSIONS = [ ".tiff", ".bmp", ".webp", + // Recordings, transcribed on this computer. Opus waits for a decoder. + ".wav", + ".mp3", + ".flac", + ".ogg", + ".oga", ] as const export const SOURCE_FILE_ACCEPT = SUPPORTED_SOURCE_EXTENSIONS.join(",") diff --git a/surfsense_local/frontend/src/features/sources/reuse-unchanged.test.ts b/surfsense_local/frontend/src/features/sources/reuse-unchanged.test.ts index 4833061f2c..9832b1ad13 100644 --- a/surfsense_local/frontend/src/features/sources/reuse-unchanged.test.ts +++ b/surfsense_local/frontend/src/features/sources/reuse-unchanged.test.ts @@ -46,6 +46,22 @@ describe("reuseUnchanged", () => { expect(merged[2]).toBe(current[0]) }) + it("keeps a recording whose transcription reads the same", () => { + const recording = (done_s: number): WorkspaceDocument => ({ + ...source(1, "processing"), + mime_type: "audio/mpeg", + transcription: { + language: null, + progress: { done_s, total_s: 3480 }, + awaiting: false, + }, + }) + const current = [recording(84)] + + expect(reuseUnchanged(current, [recording(84)])).toBe(current) + expect(reuseUnchanged(current, [recording(112)])[0]).not.toBe(current[0]) + }) + it("counts a field only one side has as a change", () => { const current = [source(1)] // An API without folders sends no folder_id. diff --git a/surfsense_local/frontend/src/features/sources/reuse-unchanged.ts b/surfsense_local/frontend/src/features/sources/reuse-unchanged.ts index c85622c4b0..71f3483486 100644 --- a/surfsense_local/frontend/src/features/sources/reuse-unchanged.ts +++ b/surfsense_local/frontend/src/features/sources/reuse-unchanged.ts @@ -1,10 +1,32 @@ -import type { WorkspaceDocument } from "./api" +import type { DocumentTranscription, WorkspaceDocument } from "./api" + +// The one nested field: every re-read parses it into a new object. +function sameTranscription( + a: DocumentTranscription | null | undefined, + b: DocumentTranscription | null | undefined +) { + if (a === b) return true + if (!a || !b) return false + return ( + a.language === b.language && + a.awaiting === b.awaiting && + (a.progress === b.progress || + (a.progress !== null && + b.progress !== null && + a.progress.done_s === b.progress.done_s && + a.progress.total_s === b.progress.total_s)) + ) +} function sameDocument(a: WorkspaceDocument, b: WorkspaceDocument) { const keys = Object.keys(a) as (keyof WorkspaceDocument)[] return ( keys.length === Object.keys(b).length && - keys.every((key) => a[key] === b[key]) + keys.every((key) => + key === "transcription" + ? key in b && sameTranscription(a.transcription, b.transcription) + : a[key] === b[key] + ) ) } diff --git a/surfsense_local/frontend/src/features/sources/source-upload.test.tsx b/surfsense_local/frontend/src/features/sources/source-upload.test.tsx index a9d1adb6c0..63f58cd1b3 100644 --- a/surfsense_local/frontend/src/features/sources/source-upload.test.tsx +++ b/surfsense_local/frontend/src/features/sources/source-upload.test.tsx @@ -207,21 +207,19 @@ describe("source upload", () => { }) expect(readyButton.className).not.toContain("truncate") expect(readyButton.className).toContain("sidebar-row-title-fade") - expect(readyButton.parentElement?.className).toContain("overflow-hidden") - expect(readyButton.parentElement?.className).toContain("h-8") + // The row's first line; a recording adds a second below it. + expect(readyButton.parentElement?.className).toContain("h-7.5") expect(readyButton.parentElement?.className).toContain("gap-1.5") - expect(readyButton.parentElement?.className).toContain("pl-1") - expect(readyButton.parentElement?.className).toContain("rounded-lg") - expect(readyButton.parentElement?.className).toContain("hover:bg-muted") - expect(readyButton.parentElement?.className).toContain( - "dark:hover:bg-muted/50" - ) - expect(readyButton.parentElement?.className).toContain("border-ring") - expect(readyButton.parentElement?.className).not.toContain( - "bg-sidebar-accent" - ) - expect(readyButton.parentElement?.className).not.toContain("text-white") - expect(readyButton.parentElement?.getAttribute("aria-current")).toBe("true") + const readyRow = screen.getByRole("treeitem", { name: ready.title }) + expect(readyRow.className).toContain("overflow-hidden") + expect(readyRow.className).toContain("pl-1") + expect(readyRow.className).toContain("rounded-lg") + expect(readyRow.className).toContain("hover:bg-muted") + expect(readyRow.className).toContain("dark:hover:bg-muted/50") + expect(readyRow.className).toContain("border-ring") + expect(readyRow.className).not.toContain("bg-sidebar-accent") + expect(readyRow.className).not.toContain("text-white") + expect(readyRow.getAttribute("aria-current")).toBe("true") expect(readyCheckbox.getAttribute("aria-checked")).toBe("false") expect(screen.getByRole("button", { name: "Select all" })).toBeTruthy() expect(screen.queryByRole("button", { name: /Delete \(/ })).toBeNull() @@ -920,6 +918,56 @@ describe("source upload", () => { fireEvent.dragLeave(panel, { dataTransfer }) }) + it("takes recordings in the formats speech recognition reads", async () => { + const fetchMock = vi.fn( + async (input: RequestInfo | URL, init?: RequestInit) => { + if ( + String(input) === "/workspaces/1/documents/upload" && + init?.method === "POST" + ) { + return Response.json( + { created: [], duplicates: [], rejected: [] }, + { status: 201 } + ) + } + return Response.json([]) + } + ) + vi.stubGlobal("fetch", fetchMock) + render() + await screen.findByText("No sources yet") + + const accept = screen + .getByLabelText("Upload source files") + .getAttribute("accept") + ?.split(",") + expect(accept).toEqual( + expect.arrayContaining([".wav", ".mp3", ".flac", ".ogg", ".oga"]) + ) + // Opus waits for the decoder that comes later. + expect(accept).not.toContain(".opus") + + const recordings = ["talk.wav", "talk.MP3", "talk.flac", "talk.ogg"].map( + (name) => new File(["audio"], name) + ) + dropOn( + screen.getByRole("region", { name: "Sources" }), + fakeDataTransfer({ + files: [...recordings, new File(["audio"], "voice.opus")], + }) + ) + + await waitFor(() => { + const upload = fetchMock.mock.calls.find( + ([path, init]) => + path === "/workspaces/1/documents/upload" && init?.method === "POST" + ) + expect((upload?.[1]?.body as FormData).getAll("files")).toEqual( + recordings + ) + }) + }) + it("refuses an unsupported dropped file before uploading", async () => { const fetchMock = vi.fn(async () => Response.json([])) vi.stubGlobal("fetch", fetchMock) diff --git a/surfsense_local/frontend/src/features/sources/sources-panel.tsx b/surfsense_local/frontend/src/features/sources/sources-panel.tsx index e36084cbd2..809996c715 100644 --- a/surfsense_local/frontend/src/features/sources/sources-panel.tsx +++ b/surfsense_local/frontend/src/features/sources/sources-panel.tsx @@ -56,6 +56,7 @@ import { type DocumentRowActions, type FolderRowActions, } from "./tree/source-tree" +import type { TranscriptionPace } from "./transcription/pace" import type { DraggedSource } from "./tree/tree-drag" import { TOP_TICK } from "./tree/use-source-scope" import { useDragAutoScroll } from "./use-drag-auto-scroll" @@ -97,6 +98,8 @@ export const SourcesPanel = memo(function SourcesPanel({ onRename, folderActions, notes, + transcriptionPaces, + onOpenSpeechSettings, }: { documents: WorkspaceDocument[] // The sources in their folders; absent, the documents are one flat list. @@ -128,6 +131,10 @@ export const SourcesPanel = memo(function SourcesPanel({ // Absent, the panel offers no folders to make, move or delete. folderActions?: FolderActions notes?: NoteActions + // Each running transcription's speed, by document id. + transcriptionPaces?: ReadonlyMap + // Opens Settings › Speech recognition; absent, a waiting recording only says so. + onOpenSpeechSettings?: () => void }) { const flatIndex = useMemo(() => indexSources([], documents), [documents]) const index = givenIndex ?? flatIndex @@ -277,6 +284,7 @@ export const SourcesPanel = memo(function SourcesPanel({ : undefined, onEditNote: editsNotes ? openNote : undefined, onSelectionChange, + onOpenSpeechSettings, }), [ onOpen, @@ -288,6 +296,7 @@ export const SourcesPanel = memo(function SourcesPanel({ editsNotes, openNote, onSelectionChange, + onOpenSpeechSettings, ] ) const readyDocuments = documents.filter( @@ -428,6 +437,7 @@ export const SourcesPanel = memo(function SourcesPanel({ folderActions={folderRowActions} dropFolder={drop.dropFolder} takesFiles={onDropFiles !== undefined} + transcriptionPaces={transcriptionPaces} /> ) : ( diff --git a/surfsense_local/frontend/src/features/sources/transcription/awaiting-speech-line.tsx b/surfsense_local/frontend/src/features/sources/transcription/awaiting-speech-line.tsx new file mode 100644 index 0000000000..d1fd70e6e4 --- /dev/null +++ b/surfsense_local/frontend/src/features/sources/transcription/awaiting-speech-line.tsx @@ -0,0 +1,45 @@ +import { Button } from "@/components/ui/button" +import { SpeechSetupPopover } from "@/features/speech-setup/speech-setup-popover" +import { intl } from "@/i18n/intl" + +/** + * A recording added before any speech recognition model: a wait, not a + * failure. Choosing a model queues it again, so set-up is the whole fix. + */ +export function AwaitingSpeechLine({ + onOpenSpeechSettings, +}: { + // Absent, the line only says what it waits for. + onOpenSpeechSettings?: () => void +}) { + return ( +
+ + {intl.formatMessage({ + id: "sources_row_awaiting_speech_status", + defaultMessage: "Waiting for speech recognition", + })} + + {onOpenSpeechSettings ? ( + + {intl.formatMessage({ + id: "sources_row_awaiting_speech_button", + defaultMessage: "Set up", + })} + + } + /> + ) : null} +
+ ) +} diff --git a/surfsense_local/frontend/src/features/sources/transcription/pace.ts b/surfsense_local/frontend/src/features/sources/transcription/pace.ts new file mode 100644 index 0000000000..20ce64fee8 --- /dev/null +++ b/surfsense_local/frontend/src/features/sources/transcription/pace.ts @@ -0,0 +1,32 @@ +import type { TranscriptionProgress } from "../api" + +type Sample = { done_s: number; at: number } + +/** Where a running transcription was when first heard from, and when last. */ +export type TranscriptionPace = { first: Sample; latest: Sample } + +/** The pace once a window reports `done_s` at `at` (milliseconds). */ +export function advancePace( + pace: TranscriptionPace | undefined, + done_s: number, + at: number +): TranscriptionPace { + // Going back means a run resumed from its checkpoint: its speed starts over. + if (!pace || done_s < pace.latest.done_s) { + return { first: { done_s, at }, latest: { done_s, at } } + } + return { first: pace.first, latest: { done_s, at } } +} + +/** Seconds until the transcript is done at the speed so far; null before one is known. */ +export function secondsLeft( + pace: TranscriptionPace | undefined, + progress: TranscriptionProgress +): number | null { + if (!pace) return null + const heard = pace.latest.done_s - pace.first.done_s + const took = (pace.latest.at - pace.first.at) / 1000 + if (heard <= 0 || took <= 0) return null + const remaining = Math.max(0, progress.total_s - progress.done_s) + return (remaining * took) / heard +} diff --git a/surfsense_local/frontend/src/features/sources/transcription/recording-line.tsx b/surfsense_local/frontend/src/features/sources/transcription/recording-line.tsx new file mode 100644 index 0000000000..dbd9820c43 --- /dev/null +++ b/surfsense_local/frontend/src/features/sources/transcription/recording-line.tsx @@ -0,0 +1,53 @@ +import type { WorkspaceDocument } from "../api" +import { AwaitingSpeechLine } from "./awaiting-speech-line" +import type { TranscriptionPace } from "./pace" +import { SearchCaveatLine } from "./search-caveat-line" +import { TranscriptionProgressLine } from "./transcription-progress-line" + +function isEnglish(language: string | null) { + return language === "en" || language?.startsWith("en-") === true +} + +/** + * The line under a recording's name: how its transcript is coming along, what + * it waits for, or how search reads it. Nothing for any other source. + */ +export function RecordingLine({ + id, + document, + pace, + onOpenSpeechSettings, +}: { + // What the row's aria-describedby names. + id: string + document: WorkspaceDocument + pace: TranscriptionPace | undefined + onOpenSpeechSettings?: () => void +}) { + const transcription = document.transcription + if (!transcription) return null + const { progress } = transcription + const running = document.status === "processing" + // Pending with windows done: it made way for other sources mid-run. + const yielded = document.status === "pending" && (progress?.done_s ?? 0) > 0 + const line = transcription.awaiting ? ( + + ) : progress && (running || yielded) ? ( + + ) : document.status === "ready" && !isEnglish(transcription.language) ? ( + + ) : null + return ( + // Kept while empty, so the row's aria-describedby always names it. +
+ {line} +
+ ) +} diff --git a/surfsense_local/frontend/src/features/sources/transcription/recording-rows.test.tsx b/surfsense_local/frontend/src/features/sources/transcription/recording-rows.test.tsx new file mode 100644 index 0000000000..811e0c7659 --- /dev/null +++ b/surfsense_local/frontend/src/features/sources/transcription/recording-rows.test.tsx @@ -0,0 +1,305 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" +import { act, cleanup, screen, waitFor, within } from "@testing-library/react" +import userEvent from "@testing-library/user-event" + +import { fakeSpeechRow } from "@/features/models/local/transcription/fake-speech-rows" +import { render } from "@/test-utils" + +import type { DocumentTranscription, WorkspaceDocument } from "../api" +import { SourcesPanel } from "../sources-panel" +import { useSources } from "../use-sources" + +vi.mock("sonner", () => ({ + toast: { error: vi.fn(), info: vi.fn(), success: vi.fn() }, +})) + +const LIST = + "/workspaces/1/documents?document_type=FILE&document_type=NOTE&limit=200&offset=0" + +function recording( + status: WorkspaceDocument["status"], + transcription: Partial, + overrides: Partial = {} +): WorkspaceDocument { + return { + id: 5, + title: "meeting.mp3", + document_type: "FILE", + mime_type: "audio/mpeg", + status, + error_message: null, + created_at: "2026-10-14T00:00:00Z", + updated_at: "2026-10-14T00:00:00Z", + folder_id: null, + transcription: { + language: null, + progress: null, + awaiting: false, + ...transcription, + }, + ...overrides, + } +} + +const BGE_SMALL = { + active: { + name: "bge-small-en-v1.5", + spec: { + id: "bge-small-en-v1.5", + source: "curated", + identified: "measured", + dimension: 384, + }, + }, + building: null, +} + +const MULTILINGUAL = { + active: { + name: "multilingual-e5-small", + spec: { + id: "intfloat/multilingual-e5-small", + source: "huggingface", + identified: "declared", + dimension: 384, + }, + }, + building: null, +} + +/** The workspace's event stream, fed by the test as the worker would feed it. */ +let feed: ReadableStreamDefaultController | null = null +const encoder = new TextEncoder() +const connected = () => feed?.enqueue(encoder.encode(": connected\n\n")) +const transcribed = (done_s: number, total_s = 3480) => + feed?.enqueue( + encoder.encode( + `event: documents\ndata: ${JSON.stringify({ + ids: [5], + status: "processing", + progress: { done_s, total_s }, + })}\n\n` + ) + ) + +/** The API: the listing the test sets, the event stream, and what rows read. */ +function serving( + documents: WorkspaceDocument[], + { embedding = BGE_SMALL }: { embedding?: unknown } = {} +) { + const calls: string[] = [] + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const path = String(input) + calls.push(`${init?.method ?? "GET"} ${path}`) + if (path.endsWith("/events")) { + return new Response( + new ReadableStream({ + start(controller) { + feed = controller + }, + }), + { headers: { "Content-Type": "text/event-stream" } } + ) + } + if (path === LIST) return Response.json(documents) + if (path === "/embedding/index") return Response.json(embedding) + if (path === "/llm/catalog/local") { + return Response.json({ rows: [fakeSpeechRow()], recommended_id: null }) + } + return Response.json({ detail: "not found" }, { status: 404 }) + }) + ) + return { listings: () => calls.filter((call) => call === `GET ${LIST}`) } +} + +function Harness({ + onOpenSpeechSettings = vi.fn(), +}: { + onOpenSpeechSettings?: () => void +}) { + const sources = useSources(1) + return ( + + ) +} + +const row = () => screen.getByRole("treeitem", { name: "meeting.mp3" }) + +beforeEach(() => { + feed = null + vi.stubGlobal( + "ResizeObserver", + class { + observe() {} + unobserve() {} + disconnect() {} + } + ) +}) + +afterEach(() => { + cleanup() + vi.restoreAllMocks() + vi.unstubAllGlobals() +}) + +describe("recordings in the sources panel", () => { + it("shows how much of a recording is transcribed, and how long is left", async () => { + const api = serving([ + recording("processing", { progress: { done_s: 1380, total_s: 3480 } }), + ]) + const now = vi.spyOn(Date, "now").mockReturnValue(1_000_000) + render() + + expect(await screen.findByText("Transcribing · 23 of 58 min")).toBeTruthy() + expect(row().getAttribute("aria-describedby")).toBeTruthy() + + await waitFor(() => expect(feed).not.toBeNull()) + connected() + transcribed(1400) + expect(await screen.findByText("Transcribing · 23 of 58 min")).toBeTruthy() + // 140 s of audio in 10 s: 14 times real time, so 1,940 s take about 2 min. + now.mockReturnValue(1_010_000) + transcribed(1540) + + expect( + await screen.findByText("Transcribing · 25 of 58 min · about 2 min left") + ).toBeTruthy() + // A window's progress is not a change of the row: nothing is read again. + expect(api.listings()).toHaveLength(1) + }) + + it("says less than a minute is left near the end", async () => { + serving([ + recording("processing", { progress: { done_s: 3000, total_s: 3480 } }), + ]) + const now = vi.spyOn(Date, "now").mockReturnValue(1_000_000) + render() + await screen.findByText("Transcribing · 50 of 58 min") + await waitFor(() => expect(feed).not.toBeNull()) + connected() + + transcribed(3100) + await screen.findByText("Transcribing · 51 of 58 min") + now.mockReturnValue(1_010_000) + transcribed(3400) + + expect( + await screen.findByText( + "Transcribing · 56 of 58 min · less than a minute left" + ) + ).toBeTruthy() + }) + + it("shows a recording that made way for other sources as queued", async () => { + serving([ + recording("pending", { progress: { done_s: 1392, total_s: 3480 } }), + ]) + render() + + expect(await screen.findByText("Queued · 40% transcribed")).toBeTruthy() + }) + + it("says a recording waits for speech recognition, not that it failed, and sets it up from the row", async () => { + serving([ + recording( + "failed", + { awaiting: true }, + { error_message: "no speech recognition model has been chosen yet" } + ), + ]) + const onOpenSpeechSettings = vi.fn() + const user = userEvent.setup() + render() + + expect( + await screen.findByText("Waiting for speech recognition") + ).toBeTruthy() + expect( + screen.queryByRole("button", { name: /Ingestion failed/ }) + ).toBeNull() + + await user.click(within(row()).getByRole("button", { name: "Set up" })) + const popover = await screen.findByRole("dialog", { + name: "Set up speech recognition", + }) + expect( + await within(popover).findByRole("button", { + name: "Download and set up", + }) + ).toBeTruthy() + + await user.click( + within(popover).getByRole("button", { name: "More choices" }) + ) + expect(onOpenSpeechSettings).toHaveBeenCalledOnce() + }) + + it("says search finds a recording best in its own language while the index is English-only", async () => { + serving([ + recording("ready", { language: "ja" }), + recording( + "ready", + { language: null }, + { id: 6, title: "interview.wav", mime_type: "audio/wav" } + ), + recording( + "ready", + { language: "en" }, + { id: 7, title: "standup.flac", mime_type: "audio/flac" } + ), + ]) + render() + + expect( + await screen.findByText( + "This recording is in Japanese. Search finds it best with questions in Japanese; add it to a chat to have all of it read." + ) + ).toBeTruthy() + expect( + screen.getByText( + "This recording’s language wasn’t identified. Search finds it best with questions in its language; add it to a chat to have all of it read." + ) + ).toBeTruthy() + // English is what the index reads best. + expect(screen.getAllByText(/Search finds it best/)).toHaveLength(2) + }) + + it("says nothing of search once the index reads every language", async () => { + serving([recording("ready", { language: "ja" })], { + embedding: MULTILINGUAL, + }) + render() + await screen.findByRole("treeitem", { name: "meeting.mp3" }) + await waitFor(() => + expect( + vi + .mocked(fetch) + .mock.calls.some(([input]) => String(input) === "/embedding/index") + ).toBe(true) + ) + // Long enough for the index's answer to render. + await act(() => new Promise((resolve) => setTimeout(resolve, 50))) + + expect(screen.queryByText(/Search finds it best/)).toBeNull() + }) +}) diff --git a/surfsense_local/frontend/src/features/sources/transcription/search-caveat-line.tsx b/surfsense_local/frontend/src/features/sources/transcription/search-caveat-line.tsx new file mode 100644 index 0000000000..5eb0033c28 --- /dev/null +++ b/surfsense_local/frontend/src/features/sources/transcription/search-caveat-line.tsx @@ -0,0 +1,40 @@ +import { useEmbeddingIndex } from "@/features/embedding/use-embedding-index" +import { languageName } from "@/features/models/local/transcription/language-names" +import { intl } from "@/i18n/intl" + +// No endpoint says which languages an embedder reads. The bundled bge-small +// is the English-only one the app ships; any other is not assumed to be. +const ENGLISH_ONLY_EMBEDDERS = new Set(["bge-small-en-v1.5"]) + +/** + * Says search finds a transcript best with questions in its own language, + * while the library's index reads English only. Nothing is switched for it. + */ +export function SearchCaveatLine({ + language, +}: { + // Null when the model could not name the recording's language. + language: string | null +}) { + const index = useEmbeddingIndex() + const embedder = index.data?.active?.spec.id + if (!embedder || !ENGLISH_ONLY_EMBEDDERS.has(embedder)) return null + return ( +

+ {language + ? intl.formatMessage( + { + id: "sources_row_search_caveat_body", + defaultMessage: + "This recording is in {language}. Search finds it best with questions in {language}; add it to a chat to have all of it read.", + }, + { language: languageName(language) } + ) + : intl.formatMessage({ + id: "sources_row_search_caveat_unidentified_body", + defaultMessage: + "This recording’s language wasn’t identified. Search finds it best with questions in its language; add it to a chat to have all of it read.", + })} +

+ ) +} diff --git a/surfsense_local/frontend/src/features/sources/transcription/transcription-progress-line.tsx b/surfsense_local/frontend/src/features/sources/transcription/transcription-progress-line.tsx new file mode 100644 index 0000000000..e2300c10ff --- /dev/null +++ b/surfsense_local/frontend/src/features/sources/transcription/transcription-progress-line.tsx @@ -0,0 +1,84 @@ +import { Progress } from "@/components/ui/progress" +import { intl } from "@/i18n/intl" + +import type { TranscriptionProgress } from "../api" +import { secondsLeft, type TranscriptionPace } from "./pace" + +function transcribingText( + progress: TranscriptionProgress, + pace: TranscriptionPace | undefined +) { + const total = Math.max(1, Math.round(progress.total_s / 60)) + const done = Math.min(Math.floor(progress.done_s / 60), total) + const left = secondsLeft(pace, progress) + if (left === null) { + return intl.formatMessage( + { + id: "sources_row_transcribing_status", + defaultMessage: + "Transcribing · {done, number} of {total, number, ::unit/minute}", + }, + { done, total } + ) + } + if (left < 60) { + return intl.formatMessage( + { + id: "sources_row_transcribing_last_minute_status", + defaultMessage: + "Transcribing · {done, number} of {total, number, ::unit/minute} · less than a minute left", + }, + { done, total } + ) + } + return intl.formatMessage( + { + id: "sources_row_transcribing_left_status", + defaultMessage: + "Transcribing · {done, number} of {total, number, ::unit/minute} · about {left, number, ::unit/minute} left", + }, + { done, total, left: Math.round(left / 60) } + ) +} + +/** + * How much of a recording is transcribed: with a bar and the time left while + * it runs, and as a share while it waits its turn after making way. + */ +export function TranscriptionProgressLine({ + progress, + running, + pace, +}: { + progress: TranscriptionProgress + running: boolean + pace: TranscriptionPace | undefined +}) { + const fraction = + progress.total_s > 0 ? Math.min(1, progress.done_s / progress.total_s) : 0 + if (!running) { + return ( +

+ {intl.formatMessage( + { + id: "sources_row_queued_status", + defaultMessage: + "Queued · {fraction, number, ::percent} transcribed", + }, + { fraction } + )} +

+ ) + } + return ( + <> +

{transcribingText(progress, pace)}

+ {/* The sentence above says the same, so the bar is for the eye only. */} + + + ) +} diff --git a/surfsense_local/frontend/src/features/sources/transcription/use-transcription-paces.ts b/surfsense_local/frontend/src/features/sources/transcription/use-transcription-paces.ts new file mode 100644 index 0000000000..0221c9b53a --- /dev/null +++ b/surfsense_local/frontend/src/features/sources/transcription/use-transcription-paces.ts @@ -0,0 +1,41 @@ +import { useCallback, useState } from "react" + +import type { WorkspaceDocument } from "../api" +import { advancePace, type TranscriptionPace } from "./pace" + +const NONE: ReadonlyMap = new Map() + +/** + * The speed of each running transcription, timed by the windows its events + * report. A re-read list carries no time, so it only ends the paces of + * recordings no longer running: a run resumed after yielding starts afresh. + */ +export function useTranscriptionPaces() { + const [paces, setPaces] = useState(NONE) + + const heard = useCallback((ids: number[], done_s: number) => { + const at = Date.now() + setPaces((current) => { + const next = new Map(current) + for (const id of ids) { + next.set(id, advancePace(current.get(id), done_s, at)) + } + return next + }) + }, []) + + const keepRunning = useCallback((documents: WorkspaceDocument[]) => { + setPaces((current) => { + if (current.size === 0) return current + const running = new Set( + documents + .filter((document) => document.status === "processing") + .map((document) => document.id) + ) + const next = new Map([...current].filter(([id]) => running.has(id))) + return next.size === current.size ? current : next + }) + }, []) + + return { paces, heard, keepRunning } +} diff --git a/surfsense_local/frontend/src/features/sources/transcription/with-progress.ts b/surfsense_local/frontend/src/features/sources/transcription/with-progress.ts new file mode 100644 index 0000000000..6d267fed97 --- /dev/null +++ b/surfsense_local/frontend/src/features/sources/transcription/with-progress.ts @@ -0,0 +1,32 @@ +import type { TranscriptionProgress, WorkspaceDocument } from "../api" + +/** + * Whether a window's progress event describes the rows as the list has them, + * so it needs no re-read: every id known, transcribing, in the event's status. + */ +export function describesListed( + documents: WorkspaceDocument[], + ids: number[], + status: string +): boolean { + return ids.every((id) => { + const document = documents.find((candidate) => candidate.id === id) + return document?.status === status && Boolean(document.transcription) + }) +} + +/** The list with these recordings' progress replaced; every other row kept. */ +export function withProgress( + documents: WorkspaceDocument[], + ids: number[], + progress: TranscriptionProgress +): WorkspaceDocument[] { + return documents.map((document) => + ids.includes(document.id) && document.transcription + ? { + ...document, + transcription: { ...document.transcription, progress }, + } + : document + ) +} diff --git a/surfsense_local/frontend/src/features/sources/tree/document-row.tsx b/surfsense_local/frontend/src/features/sources/tree/document-row.tsx index 7f11f12f9a..df4f4353c1 100644 --- a/surfsense_local/frontend/src/features/sources/tree/document-row.tsx +++ b/surfsense_local/frontend/src/features/sources/tree/document-row.tsx @@ -1,4 +1,4 @@ -import { memo, useCallback, useRef, useState } from "react" +import { memo, useCallback, useId, useRef, useState } from "react" import { Alert02Icon, CancelCircleHalfDotIcon, @@ -8,6 +8,7 @@ import { ExternalLinkIcon, FolderOpenIcon, FolderTransferIcon, + Mic01Icon, PencilIcon, RefreshCwIcon, SquareDashedMousePointerIcon, @@ -37,6 +38,8 @@ import { intl } from "@/i18n/intl" import { cn } from "@/lib/utils" import type { WorkspaceDocument } from "../api" +import type { TranscriptionPace } from "../transcription/pace" +import { RecordingLine } from "../transcription/recording-line" import type { MoveTarget } from "./move-to-dialog" import type { FolderKey } from "./source-index" import type { DocumentRowActions } from "./source-tree" @@ -61,6 +64,7 @@ export const DocumentRow = memo(function DocumentRow({ events, actions, onMoveRequest, + pace, }: TreePlace & { document: WorkspaceDocument // The folder the row sits in, where a drop on it files. @@ -76,13 +80,19 @@ export const DocumentRow = memo(function DocumentRow({ actions: DocumentRowActions // Absent with no folders to move to. onMoveRequest?: (target: MoveTarget) => void + // A running transcription's speed, for the time it has left. + pace?: TranscriptionPace }) { const ready = document.status === "ready" - const failed = document.status === "failed" + // A recording waiting for a speech recognition model has not failed: + // choosing one queues it again. + const awaiting = document.transcription?.awaiting === true + const failed = document.status === "failed" && !awaiting const cancelled = document.status === "cancelled" const retryable = failed || cancelled const ingesting = - document.status === "pending" || document.status === "processing" + (document.status === "pending" || document.status === "processing") && + !awaiting const processing = document.status === "processing" const openable = ready && document.document_type === "FILE" const previewable = getFileViewer(document.mime_type) !== null @@ -93,6 +103,7 @@ export const DocumentRow = memo(function DocumentRow({ // row (not just the retry icon) surfaces the actual error above the row. const [modifierHeld, seedModifier] = useModifierHeld(retryable && rowHovered) const element = useRef(null) + const lineId = useId() const { register } = events // Stable, so React never detaches and attaches the row again on a render. const ref = useCallback( @@ -146,8 +157,9 @@ export const DocumentRow = memo(function DocumentRow({ )} ref={ref} aria-current={highlighted ? "true" : undefined} + aria-describedby={document.transcription ? lineId : undefined} className={cn( - "group group/source relative flex h-8 w-full min-w-0 items-center gap-1.5 overflow-hidden rounded-lg border border-transparent pr-2 pl-1 select-none hover:bg-muted dark:hover:bg-muted/50", + "group group/source relative flex w-full min-w-0 flex-col overflow-hidden rounded-lg border border-transparent pr-2 pl-1 select-none hover:bg-muted dark:hover:bg-muted/50", highlighted && "border-ring", dropdownOpen && "bg-muted dark:bg-muted/50" )} @@ -157,115 +169,130 @@ export const DocumentRow = memo(function DocumentRow({ }} onMouseLeave={() => setRowHovered(false)} > - - {ready ? ( - event.stopPropagation()} - onCheckedChange={(checked) => - onSelectedChange(checked === true) - } - /> - ) : null} - {ingesting ? ( - - ) : null} - {retryable ? ( - - - - - + {/* The row's own height, with a recording's line below it. */} +
+ + {ready ? ( + event.stopPropagation()} + onCheckedChange={(checked) => + onSelectedChange(checked === true) } /> - - {cancelled - ? intl.formatMessage({ - id: "sources_row_retry_cancelled_tooltip", - defaultMessage: "Cancelled. Retry again.", - }) - : intl.formatMessage({ - id: "sources_row_retry_failed_tooltip", - defaultMessage: "Ingestion failed. Retry again.", - })} - - - ) : null} - - - {document.title} - - } + ) : null} + {ingesting ? ( + + ) : null} + {retryable ? ( + + + + + + } + /> + + {cancelled + ? intl.formatMessage({ + id: "sources_row_retry_cancelled_tooltip", + defaultMessage: "Cancelled. Retry again.", + }) + : intl.formatMessage({ + id: "sources_row_retry_failed_tooltip", + defaultMessage: "Ingestion failed. Retry again.", + })} + + + ) : null} + {awaiting ? ( + + ) : null} + + + {document.title} + + } + /> +
+ -
+
void onEditNote?: (documentId: number) => void onSelectionChange: (documentId: number, selected: boolean) => void + // Opens Settings › Speech recognition, from a waiting recording's set-up. + onOpenSpeechSettings?: () => void } /** What a folder row asks of its owner; absent, the tree has no folder actions. */ @@ -59,6 +62,7 @@ export const SourceTree = memo(function SourceTree({ dropFolder, takesFiles, labelledBy, + transcriptionPaces, }: { index: SourceIndex expanded: ReadonlySet @@ -75,6 +79,8 @@ export const SourceTree = memo(function SourceTree({ // Whether rows take files dropped from the desktop. takesFiles: boolean labelledBy: string + // Each running transcription's speed, by document id. + transcriptionPaces?: ReadonlyMap }) { const rows = useMemo( () => visibleRows(index, expanded, filter), @@ -279,6 +285,7 @@ export const SourceTree = memo(function SourceTree({ events={rowEvents} actions={documentActions} onMoveRequest={onMoveRequest} + pace={transcriptionPaces?.get(row.document.id)} /> ) )} diff --git a/surfsense_local/frontend/src/features/sources/use-sources.ts b/surfsense_local/frontend/src/features/sources/use-sources.ts index 16c6456d4b..cfe67dd786 100644 --- a/surfsense_local/frontend/src/features/sources/use-sources.ts +++ b/surfsense_local/frontend/src/features/sources/use-sources.ts @@ -3,6 +3,7 @@ import { toast } from "sonner" import { errorToast } from "@/features/feedback/error-toast" import { useWorkspaceChanges } from "@/features/workspaces/use-workspace-changes" +import type { WorkspaceChange } from "@/features/workspaces/workspace-changes" import { useStableCallback } from "@/hooks/use-stable-callback" import { intl } from "@/i18n/intl" @@ -24,6 +25,8 @@ import { type UploadEntry, } from "./folder-upload/upload-plan" import { reuseUnchanged } from "./reuse-unchanged" +import { useTranscriptionPaces } from "./transcription/use-transcription-paces" +import { describesListed, withProgress } from "./transcription/with-progress" import { indexSources } from "./tree/source-index" import { useFolders } from "./tree/use-folders" import { useSourceScope } from "./tree/use-source-scope" @@ -34,6 +37,14 @@ const MAX_SOURCE_MEGABYTES = MAX_SOURCE_BYTES / 1024 / 1024 // leave it stale, so the list is still re-read now and then until none is. const LOST_NOTICE_POLL_MS = 10_000 +// A recording waiting for a speech recognition model is not running. +function isIngesting(document: WorkspaceDocument) { + return ( + (document.status === "pending" || document.status === "processing") && + document.transcription?.awaiting !== true + ) +} + function isAbort(error: unknown) { return error instanceof DOMException && error.name === "AbortError" } @@ -74,10 +85,12 @@ export function useSources(workspaceId: number) { const uploadController = useRef(null) const pollController = useRef(null) const changeController = useRef(null) - const hasActiveIngestion = documents.some( - (document) => - document.status === "pending" || document.status === "processing" - ) + const hasActiveIngestion = documents.some(isIngesting) + const { + paces: transcriptionPaces, + heard, + keepRunning, + } = useTranscriptionPaces() useEffect(() => { const controller = new AbortController() @@ -86,6 +99,7 @@ export function useSources(workspaceId: number) { .then((next) => { if (listController.current === controller) { setDocuments((current) => reuseUnchanged(current, next)) + keepRunning(next) setError(null) setIsLoading(false) } @@ -102,7 +116,7 @@ export function useSources(workspaceId: number) { pollController.current?.abort() changeController.current?.abort() } - }, [workspaceId]) + }, [workspaceId, keepRunning]) useEffect(() => { if (!hasActiveIngestion) { @@ -121,14 +135,9 @@ export function useSources(workspaceId: number) { return } setDocuments((current) => reuseUnchanged(current, next)) + keepRunning(next) setError(null) - if ( - !next.some( - (document) => - document.status === "pending" || - document.status === "processing" - ) - ) { + if (!next.some(isIngesting)) { return } } @@ -140,7 +149,7 @@ export function useSources(workspaceId: number) { })() return () => controller.abort() - }, [hasActiveIngestion, workspaceId]) + }, [hasActiveIngestion, workspaceId, keepRunning]) // Without the loading state: the list is on screen, and only its rows move. const reloadQuietly = useStableCallback(() => { @@ -151,6 +160,7 @@ export function useSources(workspaceId: number) { .then((next) => { if (changeController.current === controller) { setDocuments((current) => reuseUnchanged(current, next)) + keepRunning(next) setError(null) } }) @@ -158,7 +168,20 @@ export function useSources(workspaceId: number) { // The next change, or the re-read while something ingests, reloads. }) }) - useWorkspaceChanges(workspaceId, "documents", reloadQuietly) + // A transcribed window's progress changes no row the list has: it is kept + // as it comes, and timed for the estimate, without reading the list again. + const documentsChanged = (change?: WorkspaceChange) => { + const progress = change?.progress + if (progress && change.status === "processing") { + heard(change.ids, progress.done_s) + if (describesListed(documents, change.ids, change.status)) { + setDocuments((current) => withProgress(current, change.ids, progress)) + return + } + } + reloadQuietly() + } + useWorkspaceChanges(workspaceId, "documents", documentsChanged) const folders = useFolders(workspaceId, reloadQuietly) const index = useMemo( @@ -176,6 +199,7 @@ export function useSources(workspaceId: number) { const next = await listDocuments(workspaceId, controller.signal) if (listController.current === controller) { setDocuments((current) => reuseUnchanged(current, next)) + keepRunning(next) setError(null) } } catch (cause) { @@ -642,6 +666,7 @@ export function useSources(workspaceId: number) { return { documents, + transcriptionPaces, folders: folders.folders, folderActions: folders.actions, index, diff --git a/surfsense_local/frontend/src/features/workspaces/read-sse.ts b/surfsense_local/frontend/src/features/workspaces/read-sse.ts index cdd41f9352..76799134c6 100644 --- a/surfsense_local/frontend/src/features/workspaces/read-sse.ts +++ b/surfsense_local/frontend/src/features/workspaces/read-sse.ts @@ -1,9 +1,14 @@ +/** How far a recording's transcription is, in seconds of its audio. */ +export type ChangeProgress = { done_s: number; total_s: number } + export type WorkspaceEvent = | { type: "connected" } | { type: "documents" | "artifacts" | "chat-runs" | "folders" ids: number[] status: string + // Sent once per transcribed window and never stored. + progress?: ChangeProgress } const CHANGED = ["documents", "artifacts", "chat-runs", "folders"] as const @@ -22,7 +27,11 @@ function parseFrame(frame: string): WorkspaceEvent | null { if (!kind || !data) return null return { type: kind, - ...(JSON.parse(data) as { ids: number[]; status: string }), + ...(JSON.parse(data) as { + ids: number[] + status: string + progress?: ChangeProgress + }), } } diff --git a/surfsense_local/frontend/src/features/workspaces/use-workspace-changes.ts b/surfsense_local/frontend/src/features/workspaces/use-workspace-changes.ts index 434975acbc..56b0dd174c 100644 --- a/surfsense_local/frontend/src/features/workspaces/use-workspace-changes.ts +++ b/surfsense_local/frontend/src/features/workspaces/use-workspace-changes.ts @@ -2,19 +2,26 @@ import { useEffect, useEffectEvent } from "react" import { subscribeToWorkspaceChanges, + type WorkspaceChange, type WorkspaceChangeKind, } from "./workspace-changes" -/** Calls `onChange` when the workspace reports a change of this kind. */ +/** + * Calls `onChange` when the workspace reports a change of this kind, with the + * change; without one when a dropped stream came back. + */ export function useWorkspaceChanges( workspaceId: number, kind: WorkspaceChangeKind, - onChange: () => void + onChange: (change?: WorkspaceChange) => void ) { const changed = useEffectEvent(onChange) useEffect( - () => subscribeToWorkspaceChanges(workspaceId, kind, () => changed()), + () => + subscribeToWorkspaceChanges(workspaceId, kind, (change) => + changed(change) + ), [workspaceId, kind] ) } diff --git a/surfsense_local/frontend/src/features/workspaces/workspace-changes.test.ts b/surfsense_local/frontend/src/features/workspaces/workspace-changes.test.ts index 10deffe8ba..52b2bdb8f1 100644 --- a/surfsense_local/frontend/src/features/workspaces/workspace-changes.test.ts +++ b/surfsense_local/frontend/src/features/workspaces/workspace-changes.test.ts @@ -22,6 +22,12 @@ function eventStream() { `event: ${kind}\ndata: ${JSON.stringify({ ids: [1], status: "ready" })}\n\n` ) ), + progressed: (progress: { done_s: number; total_s: number }) => + feed.enqueue( + encoder.encode( + `event: documents\ndata: ${JSON.stringify({ ids: [1], status: "processing", progress })}\n\n` + ) + ), drop: () => feed.close(), } } @@ -157,4 +163,21 @@ describe("workspace changes", () => { await vi.advanceTimersByTimeAsync(10_000) expect(fetchMock).toHaveBeenCalledTimes(1) }) + + it("hands on the progress a transcribed window reports", async () => { + const stream = eventStream() + apiStreaming([stream.response]) + + const documents = listen(1, "documents") + stream.connected() + stream.progressed({ done_s: 84, total_s: 3480 }) + + await vi.waitFor(() => + expect(documents).toHaveBeenCalledWith({ + ids: [1], + status: "processing", + progress: { done_s: 84, total_s: 3480 }, + }) + ) + }) }) diff --git a/surfsense_local/frontend/src/features/workspaces/workspace-changes.ts b/surfsense_local/frontend/src/features/workspaces/workspace-changes.ts index fd49359b78..a3594d4da2 100644 --- a/surfsense_local/frontend/src/features/workspaces/workspace-changes.ts +++ b/surfsense_local/frontend/src/features/workspaces/workspace-changes.ts @@ -1,10 +1,15 @@ import { followWorkspaceEvents } from "./api" +import type { ChangeProgress } from "./read-sse" export type WorkspaceChangeKind = "documents" | "artifacts" | "chat-runs" | "folders" /** Which rows changed and to what; absent when a dropped stream came back. */ -export type WorkspaceChange = { ids: number[]; status: string } +export type WorkspaceChange = { + ids: number[] + status: string + progress?: ChangeProgress +} type Listener = (change?: WorkspaceChange) => void @@ -61,6 +66,7 @@ async function follow(workspaceId: number, subscription: Subscription) { notify(subscription.listeners.get(event.type), { ids: event.ids, status: event.status, + ...(event.progress ? { progress: event.progress } : {}), }) } } diff --git a/surfsense_local/frontend/translations/de.json b/surfsense_local/frontend/translations/de.json index ea0418601a..d1f73f56fa 100644 --- a/surfsense_local/frontend/translations/de.json +++ b/surfsense_local/frontend/translations/de.json @@ -1097,6 +1097,8 @@ "sources_rename_toast": "Quelle konnte nicht umbenannt werden", "sources_reveal_original_error": "Quelle konnte nicht gefunden werden", "sources_row_actions_aria": "Aktionen für {title}", + "sources_row_awaiting_speech_button": "Einrichten", + "sources_row_awaiting_speech_status": "Wartet auf Spracherkennung", "sources_row_cancelled_status": "Abgebrochen", "sources_row_failed_status": "Einlesen fehlgeschlagen", "sources_row_menu_cancel_label": "Abbrechen", @@ -1111,11 +1113,17 @@ "sources_row_menu_reveal_label": "Im Ordner anzeigen", "sources_row_menu_select_label": "Auswählen", "sources_row_processing_aria": "{title} wird verarbeitet", + "sources_row_queued_status": "In der Warteschlange · {fraction, number, ::percent} transkribiert", "sources_row_retry_cancelled_aria": "Abgebrochen. {title} erneut versuchen", "sources_row_retry_cancelled_tooltip": "Abgebrochen. Erneut versuchen.", "sources_row_retry_failed_aria": "Einlesen fehlgeschlagen. {title} erneut versuchen", "sources_row_retry_failed_tooltip": "Einlesen fehlgeschlagen. Erneut versuchen.", + "sources_row_search_caveat_body": "Diese Aufnahme ist auf {language}. Die Suche findet sie am besten mit Fragen auf {language}; füge sie einem Chat hinzu, damit sie ganz gelesen wird.", + "sources_row_search_caveat_unidentified_body": "Die Sprache dieser Aufnahme wurde nicht erkannt. Die Suche findet sie am besten mit Fragen in ihrer Sprache; füge sie einem Chat hinzu, damit sie ganz gelesen wird.", "sources_row_select_aria": "{title} auswählen", + "sources_row_transcribing_last_minute_status": "Wird transkribiert · {done, number} von {total, number, ::unit/minute} · weniger als eine Minute übrig", + "sources_row_transcribing_left_status": "Wird transkribiert · {done, number} von {total, number, ::unit/minute} · noch etwa {left, number, ::unit/minute}", + "sources_row_transcribing_status": "Wird transkribiert · {done, number} von {total, number, ::unit/minute}", "sources_tree_filter_empty": "Keine Quelle passt zu „{filter}“", "sources_unexpected_error": "Ein unerwarteter Fehler ist aufgetreten", "sources_upload_added_toast": "{count, plural, one {# Quelle hinzugefügt} other {# Quellen hinzugefügt}}", diff --git a/surfsense_local/frontend/translations/en.json b/surfsense_local/frontend/translations/en.json index dac573f937..73ee56af2b 100644 --- a/surfsense_local/frontend/translations/en.json +++ b/surfsense_local/frontend/translations/en.json @@ -1097,6 +1097,8 @@ "sources_rename_toast": "Couldn’t rename the source", "sources_reveal_original_error": "Couldn’t locate source", "sources_row_actions_aria": "Actions for {title}", + "sources_row_awaiting_speech_button": "Set up", + "sources_row_awaiting_speech_status": "Waiting for speech recognition", "sources_row_cancelled_status": "Cancelled", "sources_row_failed_status": "Ingestion failed", "sources_row_menu_cancel_label": "Cancel", @@ -1111,11 +1113,17 @@ "sources_row_menu_reveal_label": "Show in folder", "sources_row_menu_select_label": "Select", "sources_row_processing_aria": "Processing {title}", + "sources_row_queued_status": "Queued · {fraction, number, ::percent} transcribed", "sources_row_retry_cancelled_aria": "Cancelled. Retry {title}", "sources_row_retry_cancelled_tooltip": "Cancelled. Retry again.", "sources_row_retry_failed_aria": "Ingestion failed. Retry {title}", "sources_row_retry_failed_tooltip": "Ingestion failed. Retry again.", + "sources_row_search_caveat_body": "This recording is in {language}. Search finds it best with questions in {language}; add it to a chat to have all of it read.", + "sources_row_search_caveat_unidentified_body": "This recording’s language wasn’t identified. Search finds it best with questions in its language; add it to a chat to have all of it read.", "sources_row_select_aria": "Select {title}", + "sources_row_transcribing_last_minute_status": "Transcribing · {done, number} of {total, number, ::unit/minute} · less than a minute left", + "sources_row_transcribing_left_status": "Transcribing · {done, number} of {total, number, ::unit/minute} · about {left, number, ::unit/minute} left", + "sources_row_transcribing_status": "Transcribing · {done, number} of {total, number, ::unit/minute}", "sources_tree_filter_empty": "No sources match “{filter}”", "sources_unexpected_error": "An unexpected error occurred", "sources_upload_added_toast": "{count, plural, one {# source added} other {# sources added}}", diff --git a/surfsense_local/frontend/translations/es.json b/surfsense_local/frontend/translations/es.json index d17af6f3f0..4bb1b89a18 100644 --- a/surfsense_local/frontend/translations/es.json +++ b/surfsense_local/frontend/translations/es.json @@ -1097,6 +1097,8 @@ "sources_rename_toast": "No se pudo cambiar el nombre de la fuente", "sources_reveal_original_error": "No se pudo localizar la fuente", "sources_row_actions_aria": "Acciones para {title}", + "sources_row_awaiting_speech_button": "Configurar", + "sources_row_awaiting_speech_status": "Esperando el reconocimiento de voz", "sources_row_cancelled_status": "Cancelado", "sources_row_failed_status": "Falló el procesamiento", "sources_row_menu_cancel_label": "Cancelar", @@ -1111,11 +1113,17 @@ "sources_row_menu_reveal_label": "Mostrar en la carpeta", "sources_row_menu_select_label": "Seleccionar", "sources_row_processing_aria": "Procesando {title}", + "sources_row_queued_status": "En cola · {fraction, number, ::percent} transcrito", "sources_row_retry_cancelled_aria": "Cancelado. Reintentar {title}", "sources_row_retry_cancelled_tooltip": "Cancelado. Vuelve a intentarlo.", "sources_row_retry_failed_aria": "Falló el procesamiento. Reintentar {title}", "sources_row_retry_failed_tooltip": "Falló el procesamiento. Vuelve a intentarlo.", + "sources_row_search_caveat_body": "Esta grabación está en {language}. La búsqueda la encuentra mejor con preguntas en {language}; añádela a un chat para que se lea completa.", + "sources_row_search_caveat_unidentified_body": "No se identificó el idioma de esta grabación. La búsqueda la encuentra mejor con preguntas en su idioma; añádela a un chat para que se lea completa.", "sources_row_select_aria": "Seleccionar {title}", + "sources_row_transcribing_last_minute_status": "Transcribiendo · {done, number} de {total, number, ::unit/minute} · termina en menos de un minuto", + "sources_row_transcribing_left_status": "Transcribiendo · {done, number} de {total, number, ::unit/minute} · termina en aprox. {left, number, ::unit/minute}", + "sources_row_transcribing_status": "Transcribiendo · {done, number} de {total, number, ::unit/minute}", "sources_tree_filter_empty": "Ninguna fuente coincide con «{filter}»", "sources_unexpected_error": "Se produjo un error inesperado", "sources_upload_added_toast": "{count, plural, one {# fuente añadida} many {# fuentes añadidas} other {# fuentes añadidas}}", diff --git a/surfsense_local/frontend/translations/fr.json b/surfsense_local/frontend/translations/fr.json index def6e54d05..2b834d7a7c 100644 --- a/surfsense_local/frontend/translations/fr.json +++ b/surfsense_local/frontend/translations/fr.json @@ -1097,6 +1097,8 @@ "sources_rename_toast": "Impossible de renommer la source", "sources_reveal_original_error": "Impossible de localiser la source", "sources_row_actions_aria": "Actions pour {title}", + "sources_row_awaiting_speech_button": "Configurer", + "sources_row_awaiting_speech_status": "En attente de la reconnaissance vocale", "sources_row_cancelled_status": "Annulé", "sources_row_failed_status": "Échec du traitement", "sources_row_menu_cancel_label": "Annuler", @@ -1111,11 +1113,17 @@ "sources_row_menu_reveal_label": "Afficher dans le dossier", "sources_row_menu_select_label": "Sélectionner", "sources_row_processing_aria": "Traitement de {title}", + "sources_row_queued_status": "En file d’attente · {fraction, number, ::percent} transcrit", "sources_row_retry_cancelled_aria": "Annulé. Réessayer {title}", "sources_row_retry_cancelled_tooltip": "Annulé. Réessayez.", "sources_row_retry_failed_aria": "Échec du traitement. Réessayer {title}", "sources_row_retry_failed_tooltip": "Échec du traitement. Réessayez.", + "sources_row_search_caveat_body": "Cet enregistrement est en {language}. La recherche le trouve mieux avec des questions en {language} ; ajoutez-le à un chat pour qu’il soit lu en entier.", + "sources_row_search_caveat_unidentified_body": "La langue de cet enregistrement n’a pas été identifiée. La recherche le trouve mieux avec des questions dans sa langue ; ajoutez-le à un chat pour qu’il soit lu en entier.", "sources_row_select_aria": "Sélectionner {title}", + "sources_row_transcribing_last_minute_status": "Transcription · {done, number} sur {total, number, ::unit/minute} · moins d’une minute restante", + "sources_row_transcribing_left_status": "Transcription · {done, number} sur {total, number, ::unit/minute} · encore environ {left, number, ::unit/minute}", + "sources_row_transcribing_status": "Transcription · {done, number} sur {total, number, ::unit/minute}", "sources_tree_filter_empty": "Aucune source ne correspond à « {filter} »", "sources_unexpected_error": "Une erreur inattendue s’est produite", "sources_upload_added_toast": "{count, plural, one {# source ajoutée} many {# sources ajoutées} other {# sources ajoutées}}", diff --git a/surfsense_local/frontend/translations/hi.json b/surfsense_local/frontend/translations/hi.json index 348b649419..544f8e066c 100644 --- a/surfsense_local/frontend/translations/hi.json +++ b/surfsense_local/frontend/translations/hi.json @@ -1097,6 +1097,8 @@ "sources_rename_toast": "स्रोत का नाम नहीं बदला जा सका", "sources_reveal_original_error": "स्रोत नहीं मिल सका", "sources_row_actions_aria": "{title} के लिए कार्रवाइयां", + "sources_row_awaiting_speech_button": "सेट अप करें", + "sources_row_awaiting_speech_status": "स्पीच रिकग्निशन की प्रतीक्षा में", "sources_row_cancelled_status": "रद्द किया गया", "sources_row_failed_status": "प्रोसेसिंग विफल रही", "sources_row_menu_cancel_label": "रद्द करें", @@ -1111,11 +1113,17 @@ "sources_row_menu_reveal_label": "फ़ोल्डर में दिखाएं", "sources_row_menu_select_label": "चुनें", "sources_row_processing_aria": "{title} प्रोसेस हो रहा है", + "sources_row_queued_status": "कतार में · {fraction, number, ::percent} ट्रांसक्राइब हुआ", "sources_row_retry_cancelled_aria": "रद्द किया गया। {title} के लिए फिर से कोशिश करें", "sources_row_retry_cancelled_tooltip": "रद्द किया गया। फिर से कोशिश करें।", "sources_row_retry_failed_aria": "प्रोसेसिंग विफल रही। {title} के लिए फिर से कोशिश करें", "sources_row_retry_failed_tooltip": "प्रोसेसिंग विफल रही। फिर से कोशिश करें।", + "sources_row_search_caveat_body": "यह रिकॉर्डिंग {language} में है। खोज इसे {language} में पूछे गए सवालों से सबसे अच्छी तरह ढूंढती है; पूरी रिकॉर्डिंग पढ़वाने के लिए इसे किसी चैट में जोड़ें।", + "sources_row_search_caveat_unidentified_body": "इस रिकॉर्डिंग की भाषा पहचानी नहीं जा सकी। खोज इसे इसकी अपनी भाषा में पूछे गए सवालों से सबसे अच्छी तरह ढूंढती है; पूरी रिकॉर्डिंग पढ़वाने के लिए इसे किसी चैट में जोड़ें।", "sources_row_select_aria": "{title} चुनें", + "sources_row_transcribing_last_minute_status": "ट्रांसक्राइब हो रहा है · {total, number, ::unit/minute} में से {done, number} · एक मिनट से कम बाकी", + "sources_row_transcribing_left_status": "ट्रांसक्राइब हो रहा है · {total, number, ::unit/minute} में से {done, number} · लगभग {left, number, ::unit/minute} बाकी", + "sources_row_transcribing_status": "ट्रांसक्राइब हो रहा है · {total, number, ::unit/minute} में से {done, number}", "sources_tree_filter_empty": "“{filter}” से कोई स्रोत मेल नहीं खाता", "sources_unexpected_error": "एक अनपेक्षित त्रुटि हुई", "sources_upload_added_toast": "{count, plural, one {# स्रोत जोड़ा गया} other {# स्रोत जोड़े गए}}", diff --git a/surfsense_local/frontend/translations/ja.json b/surfsense_local/frontend/translations/ja.json index 330479a0e8..89220693e7 100644 --- a/surfsense_local/frontend/translations/ja.json +++ b/surfsense_local/frontend/translations/ja.json @@ -1097,6 +1097,8 @@ "sources_rename_toast": "ソースの名前を変更できませんでした", "sources_reveal_original_error": "ソースの場所を表示できませんでした", "sources_row_actions_aria": "{title}の操作", + "sources_row_awaiting_speech_button": "設定", + "sources_row_awaiting_speech_status": "音声認識の準備待ち", "sources_row_cancelled_status": "キャンセル済み", "sources_row_failed_status": "取り込みに失敗しました", "sources_row_menu_cancel_label": "キャンセル", @@ -1111,11 +1113,17 @@ "sources_row_menu_reveal_label": "フォルダーに表示", "sources_row_menu_select_label": "選択", "sources_row_processing_aria": "{title}を処理中", + "sources_row_queued_status": "待機中 · {fraction, number, ::percent}文字起こし済み", "sources_row_retry_cancelled_aria": "キャンセルされました。{title}を再試行", "sources_row_retry_cancelled_tooltip": "キャンセルされました。もう一度お試しください。", "sources_row_retry_failed_aria": "取り込みに失敗しました。{title}を再試行", "sources_row_retry_failed_tooltip": "取り込みに失敗しました。もう一度お試しください。", + "sources_row_search_caveat_body": "この録音は{language}です。検索では{language}で質問すると最も見つかりやすくなります。全体を読ませるにはチャットに追加してください。", + "sources_row_search_caveat_unidentified_body": "この録音の言語は特定できませんでした。検索では録音と同じ言語で質問すると最も見つかりやすくなります。全体を読ませるにはチャットに追加してください。", "sources_row_select_aria": "{title}を選択", + "sources_row_transcribing_last_minute_status": "文字起こし中 · {done, number}/{total, number, ::unit/minute} · 残り1分未満", + "sources_row_transcribing_left_status": "文字起こし中 · {done, number}/{total, number, ::unit/minute} · 残り約{left, number, ::unit/minute}", + "sources_row_transcribing_status": "文字起こし中 · {done, number}/{total, number, ::unit/minute}", "sources_tree_filter_empty": "「{filter}」に一致するソースはありません", "sources_unexpected_error": "予期しないエラーが発生しました", "sources_upload_added_toast": "{count, plural, other {#件のソースを追加しました}}", diff --git a/surfsense_local/frontend/translations/ko.json b/surfsense_local/frontend/translations/ko.json index 11a352f73f..392c961896 100644 --- a/surfsense_local/frontend/translations/ko.json +++ b/surfsense_local/frontend/translations/ko.json @@ -1097,6 +1097,8 @@ "sources_rename_toast": "소스 이름을 바꿀 수 없습니다", "sources_reveal_original_error": "소스를 찾지 못했습니다", "sources_row_actions_aria": "{title} 작업", + "sources_row_awaiting_speech_button": "설정", + "sources_row_awaiting_speech_status": "음성 인식 대기 중", "sources_row_cancelled_status": "취소됨", "sources_row_failed_status": "수집 실패", "sources_row_menu_cancel_label": "취소", @@ -1111,11 +1113,17 @@ "sources_row_menu_reveal_label": "폴더에서 표시", "sources_row_menu_select_label": "선택", "sources_row_processing_aria": "{title} 처리 중", + "sources_row_queued_status": "대기 중 · {fraction, number, ::percent} 인식 완료", "sources_row_retry_cancelled_aria": "취소되었습니다. {title} 다시 시도", "sources_row_retry_cancelled_tooltip": "취소되었습니다. 다시 시도하세요.", "sources_row_retry_failed_aria": "수집에 실패했습니다. {title} 다시 시도", "sources_row_retry_failed_tooltip": "수집에 실패했습니다. 다시 시도하세요.", + "sources_row_search_caveat_body": "이 녹음은 {language}입니다. 검색은 같은 언어로 질문할 때 가장 잘 찾습니다. 전체를 읽게 하려면 채팅에 추가하세요.", + "sources_row_search_caveat_unidentified_body": "이 녹음의 언어를 식별하지 못했습니다. 검색은 녹음과 같은 언어로 질문할 때 가장 잘 찾습니다. 전체를 읽게 하려면 채팅에 추가하세요.", "sources_row_select_aria": "{title} 선택", + "sources_row_transcribing_last_minute_status": "음성 인식 중 · {done, number}/{total, number, ::unit/minute} · 1분 미만 남음", + "sources_row_transcribing_left_status": "음성 인식 중 · {done, number}/{total, number, ::unit/minute} · 약 {left, number, ::unit/minute} 남음", + "sources_row_transcribing_status": "음성 인식 중 · {done, number}/{total, number, ::unit/minute}", "sources_tree_filter_empty": "“{filter}”과(와) 일치하는 소스가 없습니다", "sources_unexpected_error": "예기치 않은 오류가 발생했습니다", "sources_upload_added_toast": "{count, plural, other {소스 #개를 추가했습니다}}", diff --git a/surfsense_local/frontend/translations/pt-BR.json b/surfsense_local/frontend/translations/pt-BR.json index 908204ab58..10fe176ef5 100644 --- a/surfsense_local/frontend/translations/pt-BR.json +++ b/surfsense_local/frontend/translations/pt-BR.json @@ -1097,6 +1097,8 @@ "sources_rename_toast": "Não foi possível renomear a fonte", "sources_reveal_original_error": "Não foi possível localizar a fonte", "sources_row_actions_aria": "Ações para {title}", + "sources_row_awaiting_speech_button": "Configurar", + "sources_row_awaiting_speech_status": "Aguardando o reconhecimento de voz", "sources_row_cancelled_status": "Cancelado", "sources_row_failed_status": "Falha no processamento", "sources_row_menu_cancel_label": "Cancelar", @@ -1111,11 +1113,17 @@ "sources_row_menu_reveal_label": "Mostrar na pasta", "sources_row_menu_select_label": "Selecionar", "sources_row_processing_aria": "Processando {title}", + "sources_row_queued_status": "Na fila · {fraction, number, ::percent} transcrito", "sources_row_retry_cancelled_aria": "Cancelado. Tentar {title} novamente", "sources_row_retry_cancelled_tooltip": "Cancelado. Tente novamente.", "sources_row_retry_failed_aria": "Falha no processamento. Tentar {title} novamente", "sources_row_retry_failed_tooltip": "Falha no processamento. Tente novamente.", + "sources_row_search_caveat_body": "Esta gravação está em {language}. A busca a encontra melhor com perguntas em {language}; adicione-a a um chat para que seja lida por inteiro.", + "sources_row_search_caveat_unidentified_body": "O idioma desta gravação não foi identificado. A busca a encontra melhor com perguntas no idioma dela; adicione-a a um chat para que seja lida por inteiro.", "sources_row_select_aria": "Selecionar {title}", + "sources_row_transcribing_last_minute_status": "Transcrevendo · {done, number} de {total, number, ::unit/minute} · termina em menos de um minuto", + "sources_row_transcribing_left_status": "Transcrevendo · {done, number} de {total, number, ::unit/minute} · termina em cerca de {left, number, ::unit/minute}", + "sources_row_transcribing_status": "Transcrevendo · {done, number} de {total, number, ::unit/minute}", "sources_tree_filter_empty": "Nenhuma fonte corresponde a “{filter}”", "sources_unexpected_error": "Ocorreu um erro inesperado", "sources_upload_added_toast": "{count, plural, one {# fonte adicionada} many {# fontes adicionadas} other {# fontes adicionadas}}", diff --git a/surfsense_local/frontend/translations/ru.json b/surfsense_local/frontend/translations/ru.json index 24a1538e86..ae8ae3e14a 100644 --- a/surfsense_local/frontend/translations/ru.json +++ b/surfsense_local/frontend/translations/ru.json @@ -1097,6 +1097,8 @@ "sources_rename_toast": "Не удалось переименовать источник", "sources_reveal_original_error": "Не удалось найти источник", "sources_row_actions_aria": "Действия для {title}", + "sources_row_awaiting_speech_button": "Настроить", + "sources_row_awaiting_speech_status": "Ожидает распознавания речи", "sources_row_cancelled_status": "Отменено", "sources_row_failed_status": "Ошибка обработки", "sources_row_menu_cancel_label": "Отменить", @@ -1111,11 +1113,17 @@ "sources_row_menu_reveal_label": "Показать в папке", "sources_row_menu_select_label": "Выбрать", "sources_row_processing_aria": "Обработка {title}", + "sources_row_queued_status": "В очереди · распознано {fraction, number, ::percent}", "sources_row_retry_cancelled_aria": "Отменено. Повторить {title}", "sources_row_retry_cancelled_tooltip": "Отменено. Попробуйте ещё раз.", "sources_row_retry_failed_aria": "Ошибка обработки. Повторить {title}", "sources_row_retry_failed_tooltip": "Ошибка обработки. Попробуйте ещё раз.", + "sources_row_search_caveat_body": "Язык записи — {language}. Поиск лучше всего находит её по вопросам на этом языке; добавьте её в чат, чтобы она была прочитана целиком.", + "sources_row_search_caveat_unidentified_body": "Язык этой записи не определён. Поиск лучше всего находит её по вопросам на её языке; добавьте её в чат, чтобы она была прочитана целиком.", "sources_row_select_aria": "Выбрать {title}", + "sources_row_transcribing_last_minute_status": "Распознавание · {done, number} из {total, number, ::unit/minute} · осталось меньше минуты", + "sources_row_transcribing_left_status": "Распознавание · {done, number} из {total, number, ::unit/minute} · осталось около {left, number, ::unit/minute}", + "sources_row_transcribing_status": "Распознавание · {done, number} из {total, number, ::unit/minute}", "sources_tree_filter_empty": "Нет источников, подходящих под «{filter}»", "sources_unexpected_error": "Произошла непредвиденная ошибка", "sources_upload_added_toast": "{count, plural, one {Добавлен # источник} few {Добавлено # источника} many {Добавлено # источников} other {Добавлено # источника}}", diff --git a/surfsense_local/frontend/translations/zh-CN.json b/surfsense_local/frontend/translations/zh-CN.json index ea30a0849a..7bc5428a36 100644 --- a/surfsense_local/frontend/translations/zh-CN.json +++ b/surfsense_local/frontend/translations/zh-CN.json @@ -1097,6 +1097,8 @@ "sources_rename_toast": "无法重命名来源", "sources_reveal_original_error": "找不到来源的位置", "sources_row_actions_aria": "{title} 的操作", + "sources_row_awaiting_speech_button": "设置", + "sources_row_awaiting_speech_status": "等待语音识别", "sources_row_cancelled_status": "已取消", "sources_row_failed_status": "处理失败", "sources_row_menu_cancel_label": "取消", @@ -1111,11 +1113,17 @@ "sources_row_menu_reveal_label": "在文件夹中显示", "sources_row_menu_select_label": "选择", "sources_row_processing_aria": "正在处理 {title}", + "sources_row_queued_status": "排队中 · 已转写 {fraction, number, ::percent}", "sources_row_retry_cancelled_aria": "已取消。重试 {title}", "sources_row_retry_cancelled_tooltip": "已取消。请再试一次。", "sources_row_retry_failed_aria": "处理失败。重试 {title}", "sources_row_retry_failed_tooltip": "处理失败。请再试一次。", + "sources_row_search_caveat_body": "这段录音是{language}。用{language}提问时搜索效果最好；把它添加到对话中，即可读取全部内容。", + "sources_row_search_caveat_unidentified_body": "未能识别这段录音的语言。用录音所用的语言提问时搜索效果最好；把它添加到对话中，即可读取全部内容。", "sources_row_select_aria": "选择 {title}", + "sources_row_transcribing_last_minute_status": "正在转写 · {done, number}/{total, number, ::unit/minute} · 剩余不到 1 分钟", + "sources_row_transcribing_left_status": "正在转写 · {done, number}/{total, number, ::unit/minute} · 剩余约 {left, number, ::unit/minute}", + "sources_row_transcribing_status": "正在转写 · {done, number}/{total, number, ::unit/minute}", "sources_tree_filter_empty": "没有与“{filter}”匹配的来源", "sources_unexpected_error": "发生了意外错误", "sources_upload_added_toast": "{count, plural, other {已添加 # 个来源}}", From 057fa19cdc6ddd0d16d6248f1a5e80f68606e309 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Wed, 14 Oct 2026 02:47:23 -0700 Subject: [PATCH 06/17] fix(sources): keep a recording's set-up open, and say when its language is unlisted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the recordings step (step 5 of the local speech recognition proposal, frontend half). - The set-up popover lived inside the "Waiting for speech recognition" line, which unmounted the moment choosing a model queued the recording again. Downloading from a row's Set up closed the popover at 100%, before it could say speech recognition is ready, and dropped focus. The line now stays while its popover is open. - The proposal's Languages section has the row say in one line when the model detected a language it is not listed for. Rows now read "Estonian (detected) · the transcript may be less accurate", comparing with the chosen model's languages from the language route; nothing shows while no speech model is chosen. - A test covers a recording that resumes after making way: the list is read again, and its estimate starts afresh without the queued time. - The recordings test fed every /events stream to the workspace's feed, so a test that opens the popover lost its events to the installs stream. --- docs/architecture/documents.md | 2 +- .../language/use-speech-language.ts | 11 +- .../transcription/awaiting-speech-line.tsx | 6 + .../sources/transcription/recording-line.tsx | 40 +++++-- .../transcription/recording-rows.test.tsx | 112 +++++++++++++++++- .../transcription/unlisted-language-line.tsx | 28 +++++ surfsense_local/frontend/translations/de.json | 1 + surfsense_local/frontend/translations/en.json | 1 + surfsense_local/frontend/translations/es.json | 1 + surfsense_local/frontend/translations/fr.json | 1 + surfsense_local/frontend/translations/hi.json | 1 + surfsense_local/frontend/translations/ja.json | 1 + surfsense_local/frontend/translations/ko.json | 1 + .../frontend/translations/pt-BR.json | 1 + surfsense_local/frontend/translations/ru.json | 1 + .../frontend/translations/zh-CN.json | 1 + 16 files changed, 190 insertions(+), 19 deletions(-) create mode 100644 surfsense_local/frontend/src/features/sources/transcription/unlisted-language-line.tsx diff --git a/docs/architecture/documents.md b/docs/architecture/documents.md index 276dc49927..6808fede04 100644 --- a/docs/architecture/documents.md +++ b/docs/architecture/documents.md @@ -64,7 +64,7 @@ The sources panel shows the Library as a tree ([`features/sources/tree/`](../../ - **Rows.** Folders come first, by name, then sources newest first. The list is an ARIA `tree` with one `treeitem` per row: arrow keys move and open, Space ticks, Enter opens, F2 renames and Delete asks to delete. A name filter, opened from the search button in the header, shows matches with the folders holding them opened; Escape or closing it clears it. A name the column cuts off shows whole in a tooltip on hover, or while its row has keyboard focus ([`overflow-tooltip.tsx`](../../surfsense_local/frontend/src/components/ui/overflow-tooltip.tsx)). Rows are memoized, and a re-read of the list keeps the object of every unchanged source ([`reuse-unchanged.ts`](../../surfsense_local/frontend/src/features/sources/reuse-unchanged.ts)), so a poll or an event renders only the rows whose source changed. - **Actions.** New folder (from the header's **+** menu, or inside a folder from its menu), Rename, Move to… and Delete. Move to… is a picker of folders and the keyboard's path; a row dragged onto a folder, or onto a source inside one, moves there too. A folder cannot be moved inside itself. Deleting a folder names how many sources, and filed Studio outputs, go with it, as `GET /folders/{folder_id}/summary` counts them, and deletes them for good. -- **Recordings.** A recording, an `audio/*` source, carries `transcription` on its row, and its row has a second line ([`transcription/`](../../surfsense_local/frontend/src/features/sources/transcription)). While it transcribes, "Transcribing · 23 of 58 min · about 7 min left" over a thin bar: the minutes are `transcription.progress`, and the time left is the speed of the windows the panel has heard of since this run started; until two have arrived there is no estimate. A recording that made way for other sources reads "Queued · 40% transcribed". One waiting for a speech recognition model reads "Waiting for speech recognition" with **Set up**, which opens the speech set-up popover ([catalog](local-models/catalog.md)), and shows no error and no Retry: choosing a model queues it again. A ready transcript that is not in English, or whose language was not identified, says search finds it best with questions in its own language and that a chat reads all of it, while the active index's embedder is bge-small. The frontend knows bge-small by its id, `bge-small-en-v1.5`, since no route says which languages an embedder reads. A recording has no preview: viewers are keyed by MIME type, and none takes `audio/*` yet. +- **Recordings.** A recording, an `audio/*` source, carries `transcription` on its row, and its row has a second line ([`transcription/`](../../surfsense_local/frontend/src/features/sources/transcription)). While it transcribes, "Transcribing · 23 of 58 min · about 7 min left" over a thin bar: the minutes are `transcription.progress`, and the time left is the speed of the windows the panel has heard of since this run started; until two have arrived there is no estimate. A recording that made way for other sources reads "Queued · 40% transcribed". One waiting for a speech recognition model reads "Waiting for speech recognition" with **Set up**, which opens the speech set-up popover ([catalog](local-models/catalog.md)), and shows no error and no Retry: choosing a model queues it again, and the popover stays open until it is closed, so it can say speech recognition is ready. A ready transcript in a language the chosen speech model detected but is not listed for (`GET /llm/selection/speech_to_text/language`) says "Estonian (detected) · the transcript may be less accurate"; with one curated speech model, there is no other to offer. A ready transcript that is not in English, or whose language was not identified, says search finds it best with questions in its own language and that a chat reads all of it, while the active index's embedder is bge-small. The frontend knows bge-small by its id, `bge-small-en-v1.5`, since no route says which languages an embedder reads. A recording has no preview: viewers are keyed by MIME type, and none takes `audio/*` yet. - **Ticks.** Each row's tick is tri-state. Ticking a folder ticks everything under it, now and added later; the nearest tick up the tree decides ([`scope-state.ts`](../../surfsense_local/frontend/src/features/sources/tree/scope-state.ts)). Chat and Studio send the ticks as `source_scope`, so every ticked source counts, not only the ones loaded. ## Upload diff --git a/surfsense_local/frontend/src/features/models/local/transcription/language/use-speech-language.ts b/surfsense_local/frontend/src/features/models/local/transcription/language/use-speech-language.ts index b8f390320a..a187055228 100644 --- a/surfsense_local/frontend/src/features/models/local/transcription/language/use-speech-language.ts +++ b/surfsense_local/frontend/src/features/models/local/transcription/language/use-speech-language.ts @@ -10,12 +10,17 @@ export const speechLanguageQueryKey = [ "language", ] as const -export function useSpeechLanguage() { - const client = useQueryClient() - const language = useQuery({ +/** The intent and the languages the model is listed for, read only. */ +export function useSpeechLanguageRead() { + return useQuery({ queryKey: speechLanguageQueryKey, queryFn: ({ signal }) => readSpeechLanguage(signal), }) +} + +export function useSpeechLanguage() { + const client = useQueryClient() + const language = useSpeechLanguageRead() const choose = useMutation({ mutationFn: writeSpeechLanguage, onSuccess: (saved) => client.setQueryData(speechLanguageQueryKey, saved), diff --git a/surfsense_local/frontend/src/features/sources/transcription/awaiting-speech-line.tsx b/surfsense_local/frontend/src/features/sources/transcription/awaiting-speech-line.tsx index d1fd70e6e4..9557cd177d 100644 --- a/surfsense_local/frontend/src/features/sources/transcription/awaiting-speech-line.tsx +++ b/surfsense_local/frontend/src/features/sources/transcription/awaiting-speech-line.tsx @@ -7,8 +7,12 @@ import { intl } from "@/i18n/intl" * failure. Choosing a model queues it again, so set-up is the whole fix. */ export function AwaitingSpeechLine({ + setUpOpen, + onSetUpOpenChange, onOpenSpeechSettings, }: { + setUpOpen: boolean + onSetUpOpenChange: (open: boolean) => void // Absent, the line only says what it waits for. onOpenSpeechSettings?: () => void }) { @@ -22,6 +26,8 @@ export function AwaitingSpeechLine({ {onOpenSpeechSettings ? ( void }) { + // Choosing a model queues the recording again mid set-up: the popover + // stays until it is closed, to say speech recognition is ready. + const [setUpOpen, setSetUpOpen] = useState(false) const transcription = document.transcription if (!transcription) return null - const { progress } = transcription + const { progress, language } = transcription const running = document.status === "processing" // Pending with windows done: it made way for other sources mid-run. const yielded = document.status === "pending" && (progress?.done_s ?? 0) > 0 - const line = transcription.awaiting ? ( - - ) : progress && (running || yielded) ? ( - - ) : document.status === "ready" && !isEnglish(transcription.language) ? ( - - ) : null + const line = + transcription.awaiting || setUpOpen ? ( + + ) : progress && (running || yielded) ? ( + + ) : document.status === "ready" ? ( + <> + {language ? : null} + {isEnglish(language) ? null : } + + ) : null return ( // Kept while empty, so the row's aria-describedby always names it.
})}\n\n` ) ) +const changed = (status: WorkspaceDocument["status"]) => + feed?.enqueue( + encoder.encode( + `event: documents\ndata: ${JSON.stringify({ ids: [5], status })}\n\n` + ) + ) /** The API: the listing the test sets, the event stream, and what rows read. */ function serving( @@ -94,10 +103,12 @@ function serving( const path = String(input) calls.push(`${init?.method ?? "GET"} ${path}`) if (path.endsWith("/events")) { + // The set-up popover follows downloads on a stream of its own. + const workspace = path === "/workspaces/1/events" return new Response( new ReadableStream({ start(controller) { - feed = controller + if (workspace) feed = controller }, }), { headers: { "Content-Type": "text/event-stream" } } @@ -108,6 +119,13 @@ function serving( if (path === "/llm/catalog/local") { return Response.json({ rows: [fakeSpeechRow()], recommended_id: null }) } + if (path === "/llm/selection/speech_to_text/language") { + return Response.json({ + language: "auto", + languages: NEMOTRON_LANGUAGES, + detects_language: true, + }) + } return Response.json({ detail: "not found" }, { status: 404 }) }) ) @@ -210,6 +228,45 @@ describe("recordings in the sources panel", () => { ).toBeTruthy() }) + it("reads a recording again when it resumes after making way, and times it afresh", async () => { + const documents = [ + recording("processing", { progress: { done_s: 1380, total_s: 3480 } }), + ] + const api = serving(documents) + const now = vi.spyOn(Date, "now").mockReturnValue(1_000_000) + render() + await screen.findByText("Transcribing · 23 of 58 min") + await waitFor(() => expect(feed).not.toBeNull()) + connected() + transcribed(1440) + await screen.findByText("Transcribing · 24 of 58 min") + now.mockReturnValue(1_010_000) + transcribed(1580) + await screen.findByText("Transcribing · 26 of 58 min · about 2 min left") + + // A PDF takes the queue; ten minutes later the recording resumes. + documents[0] = recording("pending", { + progress: { done_s: 1580, total_s: 3480 }, + }) + changed("pending") + await screen.findByText("Queued · 45% transcribed") + now.mockReturnValue(1_600_000) + documents[0] = recording("processing", { + progress: { done_s: 1680, total_s: 3480 }, + }) + transcribed(1680) + + // The row was queued, so the event's status is new to it: read again. + expect(await screen.findByText("Transcribing · 28 of 58 min")).toBeTruthy() + expect(api.listings()).toHaveLength(3) + now.mockReturnValue(1_610_000) + transcribed(1820) + // 140 s in 10 s again; the queued ten minutes are not in the speed. + expect( + await screen.findByText("Transcribing · 30 of 58 min · about 2 min left") + ).toBeTruthy() + }) + it("shows a recording that made way for other sources as queued", async () => { serving([ recording("pending", { progress: { done_s: 1392, total_s: 3480 } }), @@ -254,6 +311,34 @@ describe("recordings in the sources panel", () => { expect(onOpenSpeechSettings).toHaveBeenCalledOnce() }) + it("keeps the set-up open while choosing a model queues the recording again", async () => { + const documents = [recording("failed", { awaiting: true })] + serving(documents) + const user = userEvent.setup() + render() + await user.click( + within( + await screen.findByRole("treeitem", { name: "meeting.mp3" }) + ).getByRole("button", { name: "Set up" }) + ) + await screen.findByRole("dialog", { name: "Set up speech recognition" }) + await waitFor(() => expect(feed).not.toBeNull()) + connected() + + // The download chose the model, and the API queued the recording again. + documents[0] = recording("pending", {}) + changed("pending") + await screen.findByRole("status", { name: "Processing meeting.mp3" }) + + expect( + screen.getByRole("dialog", { name: "Set up speech recognition" }) + ).toBeTruthy() + await user.keyboard("{Escape}") + await waitFor(() => + expect(screen.queryByText("Waiting for speech recognition")).toBeNull() + ) + }) + it("says search finds a recording best in its own language while the index is English-only", async () => { serving([ recording("ready", { language: "ja" }), @@ -284,6 +369,29 @@ describe("recordings in the sources panel", () => { expect(screen.getAllByText(/Search finds it best/)).toHaveLength(2) }) + it("says a transcript may be less accurate in a language the model is not listed for", async () => { + serving( + [ + recording("ready", { language: "et" }), + recording( + "ready", + { language: "ja" }, + { id: 6, title: "interview.wav", mime_type: "audio/wav" } + ), + ], + { embedding: MULTILINGUAL } + ) + render() + + expect( + await screen.findByText( + "Estonian (detected) · the transcript may be less accurate" + ) + ).toBeTruthy() + // Japanese is one the model is listed for. + expect(screen.getAllByText(/may be less accurate/)).toHaveLength(1) + }) + it("says nothing of search once the index reads every language", async () => { serving([recording("ready", { language: "ja" })], { embedding: MULTILINGUAL, diff --git a/surfsense_local/frontend/src/features/sources/transcription/unlisted-language-line.tsx b/surfsense_local/frontend/src/features/sources/transcription/unlisted-language-line.tsx new file mode 100644 index 0000000000..46fa43a8aa --- /dev/null +++ b/surfsense_local/frontend/src/features/sources/transcription/unlisted-language-line.tsx @@ -0,0 +1,28 @@ +import { languageName } from "@/features/models/local/transcription/language-names" +import { useSpeechLanguageRead } from "@/features/models/local/transcription/language/use-speech-language" +import { intl } from "@/i18n/intl" + +/** + * Says a transcript may read less well when the model detected a language it + * is not listed for. Silent while no speech model is chosen to compare with. + */ +export function UnlistedLanguageLine({ language }: { language: string }) { + const intent = useSpeechLanguageRead() + const listed = intent.data?.languages + const base = language.split("-")[0] + if (!listed || listed.includes(language) || listed.includes(base)) { + return null + } + return ( +

+ {intl.formatMessage( + { + id: "sources_row_unlisted_language_body", + defaultMessage: + "{language} (detected) · the transcript may be less accurate", + }, + { language: languageName(language) } + )} +

+ ) +} diff --git a/surfsense_local/frontend/translations/de.json b/surfsense_local/frontend/translations/de.json index d1f73f56fa..1696881405 100644 --- a/surfsense_local/frontend/translations/de.json +++ b/surfsense_local/frontend/translations/de.json @@ -1124,6 +1124,7 @@ "sources_row_transcribing_last_minute_status": "Wird transkribiert · {done, number} von {total, number, ::unit/minute} · weniger als eine Minute übrig", "sources_row_transcribing_left_status": "Wird transkribiert · {done, number} von {total, number, ::unit/minute} · noch etwa {left, number, ::unit/minute}", "sources_row_transcribing_status": "Wird transkribiert · {done, number} von {total, number, ::unit/minute}", + "sources_row_unlisted_language_body": "Erkannte Sprache: {language} · das Transkript ist möglicherweise ungenauer", "sources_tree_filter_empty": "Keine Quelle passt zu „{filter}“", "sources_unexpected_error": "Ein unerwarteter Fehler ist aufgetreten", "sources_upload_added_toast": "{count, plural, one {# Quelle hinzugefügt} other {# Quellen hinzugefügt}}", diff --git a/surfsense_local/frontend/translations/en.json b/surfsense_local/frontend/translations/en.json index 73ee56af2b..fd5f403351 100644 --- a/surfsense_local/frontend/translations/en.json +++ b/surfsense_local/frontend/translations/en.json @@ -1124,6 +1124,7 @@ "sources_row_transcribing_last_minute_status": "Transcribing · {done, number} of {total, number, ::unit/minute} · less than a minute left", "sources_row_transcribing_left_status": "Transcribing · {done, number} of {total, number, ::unit/minute} · about {left, number, ::unit/minute} left", "sources_row_transcribing_status": "Transcribing · {done, number} of {total, number, ::unit/minute}", + "sources_row_unlisted_language_body": "{language} (detected) · the transcript may be less accurate", "sources_tree_filter_empty": "No sources match “{filter}”", "sources_unexpected_error": "An unexpected error occurred", "sources_upload_added_toast": "{count, plural, one {# source added} other {# sources added}}", diff --git a/surfsense_local/frontend/translations/es.json b/surfsense_local/frontend/translations/es.json index 4bb1b89a18..0737ca0289 100644 --- a/surfsense_local/frontend/translations/es.json +++ b/surfsense_local/frontend/translations/es.json @@ -1124,6 +1124,7 @@ "sources_row_transcribing_last_minute_status": "Transcribiendo · {done, number} de {total, number, ::unit/minute} · termina en menos de un minuto", "sources_row_transcribing_left_status": "Transcribiendo · {done, number} de {total, number, ::unit/minute} · termina en aprox. {left, number, ::unit/minute}", "sources_row_transcribing_status": "Transcribiendo · {done, number} de {total, number, ::unit/minute}", + "sources_row_unlisted_language_body": "Idioma detectado: {language} · la transcripción puede ser menos precisa", "sources_tree_filter_empty": "Ninguna fuente coincide con «{filter}»", "sources_unexpected_error": "Se produjo un error inesperado", "sources_upload_added_toast": "{count, plural, one {# fuente añadida} many {# fuentes añadidas} other {# fuentes añadidas}}", diff --git a/surfsense_local/frontend/translations/fr.json b/surfsense_local/frontend/translations/fr.json index 2b834d7a7c..6220b50b33 100644 --- a/surfsense_local/frontend/translations/fr.json +++ b/surfsense_local/frontend/translations/fr.json @@ -1124,6 +1124,7 @@ "sources_row_transcribing_last_minute_status": "Transcription · {done, number} sur {total, number, ::unit/minute} · moins d’une minute restante", "sources_row_transcribing_left_status": "Transcription · {done, number} sur {total, number, ::unit/minute} · encore environ {left, number, ::unit/minute}", "sources_row_transcribing_status": "Transcription · {done, number} sur {total, number, ::unit/minute}", + "sources_row_unlisted_language_body": "Langue détectée : {language} · la transcription peut être moins précise", "sources_tree_filter_empty": "Aucune source ne correspond à « {filter} »", "sources_unexpected_error": "Une erreur inattendue s’est produite", "sources_upload_added_toast": "{count, plural, one {# source ajoutée} many {# sources ajoutées} other {# sources ajoutées}}", diff --git a/surfsense_local/frontend/translations/hi.json b/surfsense_local/frontend/translations/hi.json index 544f8e066c..365fd440ac 100644 --- a/surfsense_local/frontend/translations/hi.json +++ b/surfsense_local/frontend/translations/hi.json @@ -1124,6 +1124,7 @@ "sources_row_transcribing_last_minute_status": "ट्रांसक्राइब हो रहा है · {total, number, ::unit/minute} में से {done, number} · एक मिनट से कम बाकी", "sources_row_transcribing_left_status": "ट्रांसक्राइब हो रहा है · {total, number, ::unit/minute} में से {done, number} · लगभग {left, number, ::unit/minute} बाकी", "sources_row_transcribing_status": "ट्रांसक्राइब हो रहा है · {total, number, ::unit/minute} में से {done, number}", + "sources_row_unlisted_language_body": "पहचानी गई भाषा: {language} · ट्रांसक्रिप्ट कम सटीक हो सकता है", "sources_tree_filter_empty": "“{filter}” से कोई स्रोत मेल नहीं खाता", "sources_unexpected_error": "एक अनपेक्षित त्रुटि हुई", "sources_upload_added_toast": "{count, plural, one {# स्रोत जोड़ा गया} other {# स्रोत जोड़े गए}}", diff --git a/surfsense_local/frontend/translations/ja.json b/surfsense_local/frontend/translations/ja.json index 89220693e7..a10e404f39 100644 --- a/surfsense_local/frontend/translations/ja.json +++ b/surfsense_local/frontend/translations/ja.json @@ -1124,6 +1124,7 @@ "sources_row_transcribing_last_minute_status": "文字起こし中 · {done, number}/{total, number, ::unit/minute} · 残り1分未満", "sources_row_transcribing_left_status": "文字起こし中 · {done, number}/{total, number, ::unit/minute} · 残り約{left, number, ::unit/minute}", "sources_row_transcribing_status": "文字起こし中 · {done, number}/{total, number, ::unit/minute}", + "sources_row_unlisted_language_body": "検出された言語：{language} · 文字起こしの精度が低い可能性があります", "sources_tree_filter_empty": "「{filter}」に一致するソースはありません", "sources_unexpected_error": "予期しないエラーが発生しました", "sources_upload_added_toast": "{count, plural, other {#件のソースを追加しました}}", diff --git a/surfsense_local/frontend/translations/ko.json b/surfsense_local/frontend/translations/ko.json index 392c961896..cf896609fb 100644 --- a/surfsense_local/frontend/translations/ko.json +++ b/surfsense_local/frontend/translations/ko.json @@ -1124,6 +1124,7 @@ "sources_row_transcribing_last_minute_status": "음성 인식 중 · {done, number}/{total, number, ::unit/minute} · 1분 미만 남음", "sources_row_transcribing_left_status": "음성 인식 중 · {done, number}/{total, number, ::unit/minute} · 약 {left, number, ::unit/minute} 남음", "sources_row_transcribing_status": "음성 인식 중 · {done, number}/{total, number, ::unit/minute}", + "sources_row_unlisted_language_body": "감지된 언어: {language} · 스크립트의 정확도가 낮을 수 있습니다", "sources_tree_filter_empty": "“{filter}”과(와) 일치하는 소스가 없습니다", "sources_unexpected_error": "예기치 않은 오류가 발생했습니다", "sources_upload_added_toast": "{count, plural, other {소스 #개를 추가했습니다}}", diff --git a/surfsense_local/frontend/translations/pt-BR.json b/surfsense_local/frontend/translations/pt-BR.json index 10fe176ef5..22e1b8d965 100644 --- a/surfsense_local/frontend/translations/pt-BR.json +++ b/surfsense_local/frontend/translations/pt-BR.json @@ -1124,6 +1124,7 @@ "sources_row_transcribing_last_minute_status": "Transcrevendo · {done, number} de {total, number, ::unit/minute} · termina em menos de um minuto", "sources_row_transcribing_left_status": "Transcrevendo · {done, number} de {total, number, ::unit/minute} · termina em cerca de {left, number, ::unit/minute}", "sources_row_transcribing_status": "Transcrevendo · {done, number} de {total, number, ::unit/minute}", + "sources_row_unlisted_language_body": "Idioma detectado: {language} · a transcrição pode ser menos precisa", "sources_tree_filter_empty": "Nenhuma fonte corresponde a “{filter}”", "sources_unexpected_error": "Ocorreu um erro inesperado", "sources_upload_added_toast": "{count, plural, one {# fonte adicionada} many {# fontes adicionadas} other {# fontes adicionadas}}", diff --git a/surfsense_local/frontend/translations/ru.json b/surfsense_local/frontend/translations/ru.json index ae8ae3e14a..f4e13c9a0f 100644 --- a/surfsense_local/frontend/translations/ru.json +++ b/surfsense_local/frontend/translations/ru.json @@ -1124,6 +1124,7 @@ "sources_row_transcribing_last_minute_status": "Распознавание · {done, number} из {total, number, ::unit/minute} · осталось меньше минуты", "sources_row_transcribing_left_status": "Распознавание · {done, number} из {total, number, ::unit/minute} · осталось около {left, number, ::unit/minute}", "sources_row_transcribing_status": "Распознавание · {done, number} из {total, number, ::unit/minute}", + "sources_row_unlisted_language_body": "Определён язык: {language} · расшифровка может быть менее точной", "sources_tree_filter_empty": "Нет источников, подходящих под «{filter}»", "sources_unexpected_error": "Произошла непредвиденная ошибка", "sources_upload_added_toast": "{count, plural, one {Добавлен # источник} few {Добавлено # источника} many {Добавлено # источников} other {Добавлено # источника}}", diff --git a/surfsense_local/frontend/translations/zh-CN.json b/surfsense_local/frontend/translations/zh-CN.json index 7bc5428a36..e5b5a4ba19 100644 --- a/surfsense_local/frontend/translations/zh-CN.json +++ b/surfsense_local/frontend/translations/zh-CN.json @@ -1124,6 +1124,7 @@ "sources_row_transcribing_last_minute_status": "正在转写 · {done, number}/{total, number, ::unit/minute} · 剩余不到 1 分钟", "sources_row_transcribing_left_status": "正在转写 · {done, number}/{total, number, ::unit/minute} · 剩余约 {left, number, ::unit/minute}", "sources_row_transcribing_status": "正在转写 · {done, number}/{total, number, ::unit/minute}", + "sources_row_unlisted_language_body": "检测到的语言：{language} · 文字稿可能不够准确", "sources_tree_filter_empty": "没有与“{filter}”匹配的来源", "sources_unexpected_error": "发生了意外错误", "sources_upload_added_toast": "{count, plural, other {已添加 # 个来源}}", From f8e5ba0a891a5d19f9e2ed414c061a43f71eaf59 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Wed, 14 Oct 2026 03:34:13 -0700 Subject: [PATCH 07/17] feat(microphone): grant the mic to the app's own page, and bridge the system's switch Electron approves every permission a session has no handler for, so any frame in the app could open the microphone silently. Speech recognition is about to ask for it, so the main process now installs request and check handlers on every session as it is created, before the app is ready: `media` goes only to the main window's main frame on the app's own page (a file in the SPA's own folder packaged, Vite's origin in development) and only for audio; a camera, a desktop or tab capture and `display-capture` are refused. Every other permission keeps Electron's own answer, so clipboard writes keep working until a strict allowlist has its own audit. The Word snapshot session, which prints untrusted documents, now refuses every permission. The page also needs the operating system's side: `window.surfsense.microphone` reports the switch (getMediaAccessStatus on Windows and macOS; Linux has none), asks macOS before getUserMedia, and opens Windows' or macOS's microphone settings through its own handler, since allowedExternalUrl passes only https. It answers the main window's main frame only. macOS gets the audio-input entitlement, which the hardened runtime requires and without which askForMediaAccess can crash the app, and its own usage sentence in place of Electron's. --- docs/architecture/agent.md | 2 +- docs/architecture/overview.md | 6 +- docs/architecture/packaging.md | 3 + .../electron/build/entitlements.mac.plist | 5 + surfsense_local/electron/electron-builder.yml | 4 + .../electron/scripts/mac-microphone.test.mjs | 24 +++ .../confine-snapshot-session.test.ts | 45 ++++- .../docx-snapshot/confine-snapshot-session.ts | 15 +- surfsense_local/electron/src/main/index.ts | 32 ++- .../electron/src/main/microphone/ipc.test.ts | 144 +++++++++++++ .../electron/src/main/microphone/ipc.ts | 29 +++ .../src/main/microphone/system-access.ts | 53 +++++ .../src/main/permissions/handlers.test.ts | 190 ++++++++++++++++++ .../electron/src/main/permissions/handlers.ts | 60 ++++++ .../electron/src/main/permissions/media.ts | 54 +++++ surfsense_local/electron/src/preload/index.ts | 12 ++ surfsense_local/frontend/src/lib/api.ts | 14 ++ 17 files changed, 682 insertions(+), 10 deletions(-) create mode 100644 surfsense_local/electron/scripts/mac-microphone.test.mjs create mode 100644 surfsense_local/electron/src/main/microphone/ipc.test.ts create mode 100644 surfsense_local/electron/src/main/microphone/ipc.ts create mode 100644 surfsense_local/electron/src/main/microphone/system-access.ts create mode 100644 surfsense_local/electron/src/main/permissions/handlers.test.ts create mode 100644 surfsense_local/electron/src/main/permissions/handlers.ts create mode 100644 surfsense_local/electron/src/main/permissions/media.ts diff --git a/docs/architecture/agent.md b/docs/architecture/agent.md index 6416f88a91..64510bb423 100644 --- a/docs/architecture/agent.md +++ b/docs/architecture/agent.md @@ -201,7 +201,7 @@ The snapshot routes, mounted only beside an opencode ([`previews/router.py`](../ Each but `file` refuses a caller with an `Origin` header, and one without `Authorization: Bearer `, the key Electron made at boot and passed to the API as `SURFSENSE_LOCAL_DOCX_SNAPSHOT_KEY` ([`snapshot_key.py`](../../surfsense_local/backend/modules/agent/previews/snapshot_key.py)), because any process on the machine can reach loopback. The `file` route is fetched by the print window itself, which sends an Origin and holds no key, as it fetches an artifact's file: the request id, a random 128-bit value only the key's holder is handed, lets it in, and only while that request is being printed. -Electron's side ([`docx-snapshot/`](../../surfsense_local/electron/src/main/docx-snapshot/)) takes the next request at once after taking one, prints up to three at a time, gives each print 25 seconds, under the API's 30, and each call to the API 10. A request names its format, `docx` or `pptx`, and its pages as `printToPDF` page ranges; one without them is a Word file's pages 1 to 4, and a format or range it cannot print is reported as the request's failure without printing. A PDF the API refuses is reported as the request's failure. The page lays a Word file out as above, and a deck with @aiden0z/pptx-renderer, the in-app viewer's library: every slide on a page of the deck's own slide size, read from the file, printed with no margin, once each slide's pictures, backgrounds and charts are drawn ([`features/docx-snapshot/`](../../surfsense_local/frontend/src/features/docx-snapshot/)). The document is untrusted, since a source can steer the script that wrote it, so the hidden window runs sandboxed with no preload, in its own in-memory session whose requests may reach only the page's own files and object URLs and the one file being printed, and it never navigates. The page's policy allows only its own script, inline styles, data-URL and object-URL images, data-URL fonts, and connections to loopback; docx-preview renders no altChunk, whose HTML it would put in an unsandboxed frame, and the deck renderer gets no pdf.js, which would start a worker. A page whose script did not start is reported as such, with the window's first console error. +Electron's side ([`docx-snapshot/`](../../surfsense_local/electron/src/main/docx-snapshot/)) takes the next request at once after taking one, prints up to three at a time, gives each print 25 seconds, under the API's 30, and each call to the API 10. A request names its format, `docx` or `pptx`, and its pages as `printToPDF` page ranges; one without them is a Word file's pages 1 to 4, and a format or range it cannot print is reported as the request's failure without printing. A PDF the API refuses is reported as the request's failure. The page lays a Word file out as above, and a deck with @aiden0z/pptx-renderer, the in-app viewer's library: every slide on a page of the deck's own slide size, read from the file, printed with no margin, once each slide's pictures, backgrounds and charts are drawn ([`features/docx-snapshot/`](../../surfsense_local/frontend/src/features/docx-snapshot/)). The document is untrusted, since a source can steer the script that wrote it, so the hidden window runs sandboxed with no preload, in its own in-memory session whose requests may reach only the page's own files and object URLs and the one file being printed, and which refuses it every permission, the microphone and the clipboard among them; it never navigates. The page's policy allows only its own script, inline styles, data-URL and object-URL images, data-URL fonts, and connections to loopback; docx-preview renders no altChunk, whose HTML it would put in an unsandboxed frame, and the deck renderer gets no pdf.js, which would start a worker. A page whose script did not start is reported as such, with the window's first console error. ## A turn diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 988cf9ab76..4f71c3185e 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -98,7 +98,9 @@ The rules that keep the processes out of each other's way: - A Vite and React SPA with Tailwind and shadcn/ui components in `components/ui/`, built on Base UI in shadcn's `base-nova` style; assistant-ui drives the conversation, and still depends on Radix. Packaged, Electron loads `frontend/dist/index.html` from disk, which is why Vite builds with relative asset paths. In dev it loads the Vite server on port 5173. - The API's address comes from the preload: Electron passes it as a command-line argument and [`preload/index.ts`](../../surfsense_local/electron/src/preload/index.ts) exposes it as `window.surfsense.apiUrl`. In a bare browser there is no preload, requests stay root-relative, and the Vite dev server forwards `/health`, `/llm`, `/workspaces`, `/chat` and `/artifacts` to `127.0.0.1:8000`. -- The preload bridge is the renderer's only other channel: opening or revealing an original file natively, opening an external link (an allowlisted one, one the user confirmed in the link popup, or a listed plugin's page by its id), the platform name, updates, theme, the title bar, this session's log and unexpected sidecar-exit notices. The renderer never sees Node or the sidecars directly. A source with a registered viewer (PDF first) is fetched from the API and previewed in place of the left sidebar; other file types keep the native open action ([`documents.md`](documents.md)). +- The preload bridge is the renderer's only other channel: opening or revealing an original file natively, opening an external link (an allowlisted one, one the user confirmed in the link popup, or a listed plugin's page by its id), the platform name, updates, theme, the title bar, this session's log, unexpected sidecar-exit notices, and the operating system's microphone switch for SurfSense. The renderer never sees Node or the sidecars directly. A source with a registered viewer (PDF first) is fetched from the API and previewed in place of the left sidebar; other file types keep the native open action ([`documents.md`](documents.md)). +- Electron approves every permission a session has no handler for, so the main process gives each session request and check handlers as it is created ([`permissions/`](../../surfsense_local/electron/src/main/permissions/)), before the app is ready. `media` is granted only to the main window's main frame on the app's own page, a file in the SPA's own folder when packaged and Vite's origin in development, and only for audio: a camera, a desktop or tab capture and `display-capture` are refused. Every other permission keeps Electron's own answer, so clipboard writes keep working; a strict allowlist would need an audit of every permission the app uses. The Word snapshot window's session refuses every permission ([agent](agent.md#previews)). +- `window.surfsense.microphone` answers the operating system's side ([`microphone/`](../../surfsense_local/electron/src/main/microphone/)), for the main window's main frame only: `status()` is `systemPreferences.getMediaAccessStatus("microphone")` on Windows and macOS, and `granted` on Linux, which has no switch for an AppImage or a deb; `request()` asks macOS with `askForMediaAccess` before the page calls `getUserMedia`, and on Windows reports the switch as it is; `openSettings()` opens `ms-settings:privacy-microphone` or the macOS Privacy › Microphone pane, which `allowedExternalUrl` cannot, since it passes only https, and answers false on Linux. The page never names the address. - API calls go through `request()` in [`lib/api.ts`](../../surfsense_local/frontend/src/lib/api.ts), except the Studio viewers, which fetch an artifact's file bytes directly; an ``, `
- - + + + + )} + {mic("")} + -
) : ( <> - + {mic("-mr-1 mb-0.5")} + )} @@ -302,23 +354,27 @@ export function ChatComposer({ {/* Keeps its line and leaves the mode and the model name what is left, never less than 13rem, so a long translation truncates the name before wrapping. */} -

- {limitNotice ? ( - {limitNotice} - ) : !model || providerAvailable ? ( - intl.formatMessage({ - id: "chat_composer_disclaimer_body", - defaultMessage: - "SurfSense can make mistakes. Check important answers.", - }) - ) : ( - intl.formatMessage({ - id: "chat_composer_provider_offline_body", - defaultMessage: - "Historical chats remain available while the provider is offline.", - }) - )} -

+ {stripShown ? ( + strip("max-w-[calc(100%-13rem)] min-h-7 flex-1") + ) : ( +

+ {limitNotice ? ( + {limitNotice} + ) : !model || providerAvailable ? ( + intl.formatMessage({ + id: "chat_composer_disclaimer_body", + defaultMessage: + "SurfSense can make mistakes. Check important answers.", + }) + ) : ( + intl.formatMessage({ + id: "chat_composer_provider_offline_body", + defaultMessage: + "Historical chats remain available while the provider is offline.", + }) + )} +

+ )}
{model && threadMode !== null ? ( void onCitation?: (chunkId: number) => void + dictation?: Dictation + threadId?: number }) { + const threadId = handlers.threadId ?? THREAD.id const props: ComponentProps = { live: { - threadId: THREAD.id, + threadId, persistedMessages: HISTORY, - adapters: undefined, + adapters: handlers.dictation + ? { dictation: handlers.dictation } + : undefined, isSendDisabled: false, onNew: async () => undefined, onCancel: async () => undefined, }, - thread: THREAD, - view: { status: "active", threadId: THREAD.id }, - model: null, + thread: { ...THREAD, id: threadId }, + view: { status: "active", threadId }, + model: handlers.dictation ? MODEL : null, + dictation: handlers.dictation, isLoading: false, isRunning: false, animateTitle: false, @@ -126,4 +158,38 @@ describe("thread panel", () => { expect(latest).toHaveBeenCalledWith("6") expect(first).not.toHaveBeenCalled() }) + + it("cancels voice input on a thread switch, so nothing lands in the next thread", async () => { + const api = fakeDictationApi() + const late = held(() => Response.json(said("For the first thread."))) + api.answer(late.answer) + vi.mocked(fetch).mockImplementation( + async (input: RequestInfo | URL, init?: RequestInit) => + (await api.handle(String(input), init)) ?? Response.json([]) + ) + const mic = fakeMicrophone() + const dictation = createDictation({ openMicrophone: mic.open }) + const user = userEvent.setup() + const { rerender } = render(panel({ onRetry: () => undefined, dictation })) + + await user.click( + await screen.findByRole("button", { name: "Start voice input" }) + ) + await waitFor(() => expect(mic.state.open).toBe(true)) + act(() => mic.say()) + await waitFor(() => expect(api.phrases).toHaveLength(1)) + rerender(panel({ onRetry: () => undefined, dictation, threadId: 2 })) + + expect(mic.state.open).toBe(false) + late.release() + await waitFor(() => + expect(api.calls).toContain("DELETE /llm/transcription/dictations/d1") + ) + await new Promise((resolve) => setTimeout(resolve, 20)) + expect( + screen.getByRole("textbox", { name: "Message" }) + .value + ).toBe("") + expect(dictation.getView().phase).toBe("idle") + }) }) diff --git a/surfsense_local/frontend/src/features/chat/thread-panel.tsx b/surfsense_local/frontend/src/features/chat/thread-panel.tsx index 795ff309bd..a4c0b88d39 100644 --- a/surfsense_local/frontend/src/features/chat/thread-panel.tsx +++ b/surfsense_local/frontend/src/features/chat/thread-panel.tsx @@ -24,6 +24,7 @@ import { import { Input } from "@/components/ui/input" import { Skeleton } from "@/components/ui/skeleton" import { TypewriterText } from "@/components/typewriter-text" +import type { Dictation } from "@/features/dictation/dictation-adapter" import type { ModelSelection } from "@/features/models/selection/api" import { useStableCallback } from "@/hooks/use-stable-callback" import { intl } from "@/i18n/intl" @@ -114,12 +115,24 @@ function ThreadWelcome({ composer }: { composer: ReactNode }) { ) } -function ComposerDraftLifecycle({ view }: { view: ConversationView }) { +function ComposerDraftLifecycle({ + view, + dictation, +}: { + view: ConversationView + dictation?: Dictation +}) { const aui = useAui() const conversationId = view.status === "active" ? `thread:${view.threadId}` : view.status const returned = view.status === "new" ? view.returned : undefined + // Voice input outlives the composer, so it is cancelled, never stopped: no + // late phrase lands in the thread opened next. + useEffect(() => { + dictation?.cancel() + }, [dictation, conversationId]) + useEffect(() => { if (conversationId === "creating") { return @@ -166,6 +179,8 @@ export function ThreadPanel({ onUploadSources, isUploadingSources, onOpenPlugins, + dictation, + onOpenSpeechSettings, }: { live: LiveThreadSource thread: ChatThread | null @@ -191,6 +206,10 @@ export function ThreadPanel({ isUploadingSources?: boolean // Opens Settings › Plugins, from the composer's add menu. onOpenPlugins?: (start?: PluginsStart) => void + // The runtime's dictation adapter, for the composer's mic. + dictation?: Dictation + // Opens Settings › Speech recognition, from the mic's set-up. + onOpenSpeechSettings?: () => void }) { const titleInputRef = useRef(null) const headingId = useId() @@ -229,6 +248,8 @@ export function ThreadPanel({ } onNewChat={onNewChat} onOpenPlugins={onOpenPlugins} + dictation={dictation} + onOpenSpeechSettings={onOpenSpeechSettings} /> ) const bottomFooter = bottomComposer ? composer("bottom") : undefined @@ -271,7 +292,7 @@ export function ThreadPanel({ // runtime alone, which hands React these same elements, so it skips them. return ( - +
{ + it.each([false, true])( + "offers voice input whether the model reads images or not (reads images: %s)", + async (readsImages) => { + const { result } = renderRuntime({ readsImages }) + + await waitFor(() => + expect(result.current.thread.getState().capabilities.dictation).toBe( + true + ) + ) + expect(result.current.thread.getState().capabilities.attachments).toBe( + readsImages + ) + } + ) + it("keeps a reply arriving while another thread is open", async () => { const { result } = renderRuntime() await openThread(result, 1) diff --git a/surfsense_local/frontend/src/features/chat/use-chat-runtime.ts b/surfsense_local/frontend/src/features/chat/use-chat-runtime.ts index 4d6cf62845..b690cc46c3 100644 --- a/surfsense_local/frontend/src/features/chat/use-chat-runtime.ts +++ b/surfsense_local/frontend/src/features/chat/use-chat-runtime.ts @@ -17,6 +17,7 @@ import { } from "@/features/agent/api" import { isOutdatedThreadRefusal } from "@/features/agent/outdated-thread" import { isUnsupportedModelRefusal } from "@/features/agent/unsupported-model" +import { createDictation } from "@/features/dictation/dictation-adapter" import { errorToast } from "@/features/feedback/error-toast" import type { SourceScope } from "@/features/sources/tree/scope-state" import { subscribeToWorkspaceChanges } from "@/features/workspaces/workspace-changes" @@ -918,9 +919,25 @@ export function useChatRuntime({ } }, [messagesQuery.error]) + // One per page, so a recording outlives the composer's remount on a thread + // switch; that switch cancels it (thread-panel.tsx), and so do leaving and quitting. + const [dictation] = useState(() => createDictation()) + useEffect(() => { + const leave = () => dictation.cancel() + window.addEventListener("pagehide", leave) + return () => { + window.removeEventListener("pagehide", leave) + leave() + } + }, [dictation]) + + // Dictated text is plain text, so the mic is there whatever the model reads. const adapters = useMemo( - () => (readsImages ? { attachments: new ChatImageAdapter() } : undefined), - [readsImages] + () => + readsImages + ? { dictation, attachments: new ChatImageAdapter() } + : { dictation }, + [dictation, readsImages] ) const isSendDisabled = !canSend || isLoadingMessages || isLoadingThreads @@ -953,6 +970,7 @@ export function useChatRuntime({ return { liveThread, + dictation, threads, conversationView, activeThread, diff --git a/surfsense_local/frontend/src/features/dashboard/dashboard-page.tsx b/surfsense_local/frontend/src/features/dashboard/dashboard-page.tsx index 196f8423dd..49b4c35fe8 100644 --- a/surfsense_local/frontend/src/features/dashboard/dashboard-page.tsx +++ b/surfsense_local/frontend/src/features/dashboard/dashboard-page.tsx @@ -497,6 +497,8 @@ function WorkspaceDashboard({ onUploadSources={(files) => void sources.upload(files)} isUploadingSources={sources.isUploading} onOpenPlugins={onOpenPlugins} + dictation={chat.dictation} + onOpenSpeechSettings={openSpeechSettings} onTitleAnimationComplete={chat.finishTitleAnimation} autoNamingThreadId={chat.autoNamingThreadId} onRename={chat.rename} diff --git a/surfsense_local/frontend/src/features/dictation/api.ts b/surfsense_local/frontend/src/features/dictation/api.ts new file mode 100644 index 0000000000..8add364f2a --- /dev/null +++ b/surfsense_local/frontend/src/features/dictation/api.ts @@ -0,0 +1,59 @@ +import { requestJson, requestVoid } from "@/lib/api" + +const ROOT = "/llm/transcription" + +/** Whether voice input can start, and why not; the mic reads it. */ +export type TranscriptionStatus = { + ready: boolean + reason: "needs_speech_model" | "transcription_unavailable" | null + model: string | null + language_intent: string + languages: string[] +} + +export type OpenedDictation = { + id: string + /** `loading` while the model loads for this session's first phrase. */ + warm: "loaded" | "loading" +} + +export type PhraseText = { + text: string + /** ISO 639-1, detected or pinned; null when neither says. */ + language: string | null + no_speech: boolean +} + +export function readTranscriptionStatus( + signal?: AbortSignal +): Promise { + return requestJson(`${ROOT}/status`, { signal }) +} + +export function openDictation(): Promise { + return requestJson(`${ROOT}/dictations`, { method: "POST" }) +} + +/** One phrase: a WAV of 16 kHz mono 16-bit PCM, at most 30 s. */ +export function transcribePhrase(id: string, wav: Blob): Promise { + const body = new FormData() + body.append("audio", wav, "phrase.wav") + return requestJson( + `${ROOT}/dictations/${encodeURIComponent(id)}/phrases`, + { method: "POST", body } + ) +} + +export function keepDictationAlive(id: string): Promise { + return requestVoid(`${ROOT}/dictations/${encodeURIComponent(id)}/keepalive`, { + method: "POST", + }) +} + +export function endDictation(id: string): Promise { + // Outlives the page, so a closing window still releases the gate. + return requestVoid(`${ROOT}/dictations/${encodeURIComponent(id)}`, { + method: "DELETE", + keepalive: true, + }) +} diff --git a/surfsense_local/frontend/src/features/dictation/capture/encode-wav.ts b/surfsense_local/frontend/src/features/dictation/capture/encode-wav.ts new file mode 100644 index 0000000000..3c8eab8b04 --- /dev/null +++ b/surfsense_local/frontend/src/features/dictation/capture/encode-wav.ts @@ -0,0 +1,33 @@ +const HEADER_BYTES = 44 + +/** Mono 16-bit PCM in a WAV, the only input the transcription server reads. */ +export function encodeWav(samples: Float32Array, sampleRate: number): Blob { + const view = new DataView(new ArrayBuffer(HEADER_BYTES + samples.length * 2)) + const text = (offset: number, value: string) => { + for (let i = 0; i < value.length; i++) { + view.setUint8(offset + i, value.charCodeAt(i)) + } + } + text(0, "RIFF") + view.setUint32(4, 36 + samples.length * 2, true) + text(8, "WAVE") + text(12, "fmt ") + view.setUint32(16, 16, true) + view.setUint16(20, 1, true) // PCM + view.setUint16(22, 1, true) // mono + view.setUint32(24, sampleRate, true) + view.setUint32(28, sampleRate * 2, true) + view.setUint16(32, 2, true) + view.setUint16(34, 16, true) + text(36, "data") + view.setUint32(40, samples.length * 2, true) + for (let i = 0; i < samples.length; i++) { + const sample = Math.max(-1, Math.min(1, samples[i])) + view.setInt16( + HEADER_BYTES + i * 2, + sample < 0 ? sample * 0x8000 : sample * 0x7fff, + true + ) + } + return new Blob([view.buffer], { type: "audio/wav" }) +} diff --git a/surfsense_local/frontend/src/features/dictation/capture/fake-microphone.ts b/surfsense_local/frontend/src/features/dictation/capture/fake-microphone.ts new file mode 100644 index 0000000000..97b685d40e --- /dev/null +++ b/surfsense_local/frontend/src/features/dictation/capture/fake-microphone.ts @@ -0,0 +1,62 @@ +import type { MicrophoneFrame, OpenMicrophone } from "./open-microphone" + +const VOICE = 0.1 +const ROOM = 0.001 + +/** + * Stands in for the default microphone: a test says what the speaker does, + * and the frames reach the composer as the worklet would post them. + */ +export function fakeMicrophone({ sampleRate = 16_000 } = {}) { + let handlers: Parameters[0] | null = null + let refusal: unknown = null + const state = { open: false, opens: 0, closes: 0 } + + const open: OpenMicrophone = async (next) => { + if (refusal) throw refusal + handlers = next + state.open = true + state.opens += 1 + return { + close: () => { + if (state.open) state.closes += 1 + state.open = false + }, + } + } + + const frames = (seconds: number, level: number) => { + const size = Math.round(sampleRate * 0.02) + for (let i = 0; i < Math.round(seconds * 50); i++) { + if (!state.open) return + const frame: MicrophoneFrame = { + samples: new Float32Array(size).fill(level), + level, + sampleRate, + } + handlers?.onFrame(frame) + } + } + + return { + open, + state, + /** The next open fails, as getUserMedia would. */ + refuse: (error: unknown) => { + refusal = error + }, + /** A phrase: a breath, speech, and the pause that ends it. */ + say: (seconds = 1.2) => { + frames(0.2, ROOM) + frames(seconds, VOICE) + frames(0.7, ROOM) + }, + /** Speech still going when the speaker stops. */ + speak: (seconds: number) => { + frames(0.2, ROOM) + frames(seconds, VOICE) + }, + quiet: (seconds: number) => frames(seconds, ROOM), + unplug: () => handlers?.onEnded(), + } +} diff --git a/surfsense_local/frontend/src/features/dictation/capture/frame-worklet.js b/surfsense_local/frontend/src/features/dictation/capture/frame-worklet.js new file mode 100644 index 0000000000..c8887c9de9 --- /dev/null +++ b/surfsense_local/frontend/src/features/dictation/capture/frame-worklet.js @@ -0,0 +1,36 @@ +// Runs on the audio thread: hands the page 20 ms frames, their RMS level and +// the context's rate (`sampleRate`, a global of the worklet scope). +// Plain JavaScript, loaded by URL, so it reaches the AudioWorklet as written. +class DictationFrames extends AudioWorkletProcessor { + constructor(options) { + super() + this.size = options.processorOptions.frameSamples + this.frame = new Float32Array(this.size) + this.filled = 0 + } + + process(inputs) { + const channel = inputs[0] && inputs[0][0] + if (!channel) return true + let read = 0 + while (read < channel.length) { + const take = Math.min(channel.length - read, this.size - this.filled) + this.frame.set(channel.subarray(read, read + take), this.filled) + this.filled += take + read += take + if (this.filled === this.size) { + let sum = 0 + for (let i = 0; i < this.size; i++) sum += this.frame[i] * this.frame[i] + const level = Math.sqrt(sum / this.size) + this.port.postMessage({ samples: this.frame, level, sampleRate }, [ + this.frame.buffer, + ]) + this.frame = new Float32Array(this.size) + this.filled = 0 + } + } + return true + } +} + +registerProcessor("dictation-frames", DictationFrames) diff --git a/surfsense_local/frontend/src/features/dictation/capture/noise-floor.ts b/surfsense_local/frontend/src/features/dictation/capture/noise-floor.ts new file mode 100644 index 0000000000..b499625273 --- /dev/null +++ b/surfsense_local/frontend/src/features/dictation/capture/noise-floor.ts @@ -0,0 +1,20 @@ +// 3 s of 20 ms frames. +const WINDOW_FRAMES = 150 +const PERCENTILE = 0.1 + +/** + * The room's level: the 10th percentile of the last 3 s, low enough that the + * gaps between words keep it below the speaker however steadily they talk. + */ +export function createNoiseFloor() { + const levels: number[] = [] + return { + /** Takes one frame's level and answers the floor including it. */ + add(level: number): number { + levels.push(level) + if (levels.length > WINDOW_FRAMES) levels.shift() + const sorted = [...levels].sort((a, b) => a - b) + return sorted[Math.floor(PERCENTILE * (sorted.length - 1))] + }, + } +} diff --git a/surfsense_local/frontend/src/features/dictation/capture/open-microphone.ts b/surfsense_local/frontend/src/features/dictation/capture/open-microphone.ts new file mode 100644 index 0000000000..9c271a9b0b --- /dev/null +++ b/surfsense_local/frontend/src/features/dictation/capture/open-microphone.ts @@ -0,0 +1,93 @@ +import { SPEECH_SAMPLE_RATE } from "./resample" + +export type MicrophoneFrame = { + samples: Float32Array + /** RMS of the samples. */ + level: number + /** 16 kHz, or the device's rate where a platform refuses that. */ + sampleRate: number +} + +export type Microphone = { + /** Stops every track and closes the context, which clears the OS mic indicator. */ + close: () => void +} + +export type OpenMicrophone = (handlers: { + onFrame: (frame: MicrophoneFrame) => void + /** The device went away mid-recording. */ + onEnded: () => void +}) => Promise + +const CONSTRAINTS: MediaStreamConstraints = { + audio: { + channelCount: 1, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, +} + +/** A 16 kHz graph where the platform allows it, else one at the device's rate. */ +function graphFor(stream: MediaStream) { + try { + const context = new AudioContext({ sampleRate: SPEECH_SAMPLE_RATE }) + try { + return { context, source: context.createMediaStreamSource(stream) } + } catch { + void context.close().catch(() => undefined) + } + } catch { + // Refused outright; the device's rate below. + } + const context = new AudioContext() + return { context, source: context.createMediaStreamSource(stream) } +} + +/** + * The default microphone as 20 ms frames. Errors are getUserMedia's own + * DOMExceptions; the system's switch being off reads as NotAllowedError. + */ +export const openMicrophone: OpenMicrophone = async ({ onFrame, onEnded }) => { + // macOS asks the first time; Windows reports its switch; Linux has none. + const system = await window.surfsense?.microphone?.request() + if (system === "denied" || system === "restricted") { + throw new DOMException("Microphone access is off", "NotAllowedError") + } + const stream = await navigator.mediaDevices.getUserMedia(CONSTRAINTS) + let graph: ReturnType | null = null + try { + graph = graphFor(stream) + const { context, source } = graph + // Resolved by Vite like pdf.js's worker, so it loads over file:// too; a + // file, never inlined as a data: URL, which is another origin to a worklet. + await context.audioWorklet.addModule( + new URL("./frame-worklet.js?no-inline", import.meta.url) + ) + const node = new AudioWorkletNode(context, "dictation-frames", { + processorOptions: { frameSamples: Math.round(context.sampleRate * 0.02) }, + }) + node.port.onmessage = (event: MessageEvent) => + onFrame(event.data) + source.connect(node) + // Pulled by the graph only when it reaches the output; it writes silence. + node.connect(context.destination) + void context.resume().catch(() => undefined) + const tracks = stream.getAudioTracks() + tracks.forEach((track) => track.addEventListener("ended", onEnded)) + return { + close: () => { + node.port.onmessage = null + tracks.forEach((track) => track.removeEventListener("ended", onEnded)) + stream.getTracks().forEach((track) => track.stop()) + source.disconnect() + node.disconnect() + void context.close().catch(() => undefined) + }, + } + } catch (error) { + stream.getTracks().forEach((track) => track.stop()) + void graph?.context.close().catch(() => undefined) + throw error + } +} diff --git a/surfsense_local/frontend/src/features/dictation/capture/phrase-cutter.test.ts b/surfsense_local/frontend/src/features/dictation/capture/phrase-cutter.test.ts new file mode 100644 index 0000000000..1b0df30010 --- /dev/null +++ b/surfsense_local/frontend/src/features/dictation/capture/phrase-cutter.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, it } from "vitest" + +import { createPhraseCutter } from "./phrase-cutter" + +// Each frame is 20 ms; its samples all hold the frame's number, so a phrase +// says which frames it kept by its first and last sample. +const SAMPLES_PER_FRAME = 4 +const QUIET = 0.001 +const VOICE = 0.1 + +function feed() { + const cutter = createPhraseCutter() + let next = 0 + const phrases: { at: number; first: number; last: number; frames: number }[] = + [] + const push = (count: number, level: number | ((index: number) => number)) => { + for (let i = 0; i < count; i += 1) { + const index = next + next += 1 + const phrase = cutter.push( + new Float32Array(SAMPLES_PER_FRAME).fill(index), + typeof level === "number" ? level : level(index) + ) + if (phrase) phrases.push({ at: index, ...span(phrase) }) + } + } + return { cutter, push, phrases } +} + +function span(phrase: Float32Array) { + return { + first: phrase[0], + last: phrase[phrase.length - 1], + frames: phrase.length / SAMPLES_PER_FRAME, + } +} + +describe("phrase endpointing", () => { + it("ends a phrase at a 700 ms pause after a second of voice, with 450 ms either side", () => { + const { push, phrases } = feed() + + push(50, QUIET) // frames 0-49 + push(60, VOICE) // 50-109, 1.2 s + push(34, QUIET) // 110-143, 680 ms + expect(phrases).toEqual([]) + push(1, QUIET) // 144: the 700th ms + + // 23 frames (460 ms) before the first voiced frame and after the last. + expect(phrases).toEqual([{ at: 144, first: 27, last: 132, frames: 106 }]) + }) + + it("keeps a word or two open across a pause, so it is not sent alone", () => { + const { push, phrases } = feed() + + push(50, QUIET) + push(20, VOICE) // 50-69, 0.4 s + push(50, QUIET) // 70-119, a full second + expect(phrases).toEqual([]) + push(40, VOICE) // 120-159; 1.2 s of voice in all + push(35, QUIET) // 160-194 + + expect(phrases).toEqual([{ at: 194, first: 27, last: 182, frames: 156 }]) + }) + + it("cuts a phrase that reaches 20 s at its quietest 200 ms", () => { + const { push, phrases } = feed() + // Speech is never flat: one frame in five falls between syllables. + const speech = (index: number) => (index % 5 === 0 ? 0.02 : 0.2) + + push(50, QUIET) + push(750, speech) // 50-799 + push(10, 0.01) // 800-809, a breath too short to end the phrase + push(217, speech) // 810-1026: the phrase, from frame 27, is 20 s long + expect(phrases).toEqual([{ at: 1026, first: 27, last: 804, frames: 778 }]) + + push(73, speech) // 1027-1099 + push(35, QUIET) + expect(phrases[1]).toEqual({ + at: 1134, + first: 805, + last: 1122, + frames: 318, + }) + }) + + it("hands over the phrase in progress when the speaker stops", () => { + const { cutter, push } = feed() + + push(50, QUIET) + push(20, VOICE) // 50-69 + push(5, QUIET) // 70-74 + + const tail = cutter.flush() + expect(tail && span(tail)).toEqual({ first: 27, last: 74, frames: 48 }) + expect(cutter.flush()).toBeNull() + }) + + it("has nothing to hand over when no one spoke", () => { + const { cutter, push } = feed() + + push(100, QUIET) + + expect(cutter.flush()).toBeNull() + }) + + it("hears speech only at three times the room’s noise", () => { + const { push, phrases } = feed() + const NOISE = 0.02 + + push(200, NOISE) // a fan + push(60, 0.05) // 2.5 times the fan: not a voice + push(35, NOISE) + expect(phrases).toEqual([]) + + push(60, 0.08) // four times + push(35, NOISE) + expect(phrases).toHaveLength(1) + }) + + it("takes nothing under its minimum for speech, however still the room", () => { + const { push, phrases, cutter } = feed() + + push(100, 0.0005) + push(60, 0.004) // eight times the room, still a whisper + push(35, 0.0005) + + expect(phrases).toEqual([]) + expect(cutter.flush()).toBeNull() + }) +}) diff --git a/surfsense_local/frontend/src/features/dictation/capture/phrase-cutter.ts b/surfsense_local/frontend/src/features/dictation/capture/phrase-cutter.ts new file mode 100644 index 0000000000..85f9f80a20 --- /dev/null +++ b/surfsense_local/frontend/src/features/dictation/capture/phrase-cutter.ts @@ -0,0 +1,119 @@ +import { createNoiseFloor } from "./noise-floor" + +const FRAME_MS = 20 +const frames = (ms: number) => Math.ceil(ms / FRAME_MS) + +// Handy's pre-roll and hangover: the onset and the last word, which sit under +// the threshold, stay in the phrase. +const PRE_ROLL_FRAMES = frames(450) +const TAIL_FRAMES = frames(450) +// A word or two alone is transcribed worse than with its neighbours. +const MIN_VOICE_FRAMES = frames(1000) +const PAUSE_FRAMES = frames(700) +// Well under the API's 30 s per phrase, and one encoder graph's memory. +const MAX_PHRASE_FRAMES = frames(20_000) +const CUT_FRAMES = frames(200) +// A forced cut looks only in the phrase's second half, so neither piece is short. +const CUT_SEARCH_FROM = MAX_PHRASE_FRAMES / 2 + +const SPEECH_OVER_FLOOR = 3 +// About -46 dBFS (RMS): what a still room and Chromium's noise suppression +// leave stays under it, so a quiet room's floor alone never counts as speech. +export const SPEECH_MIN_LEVEL = 0.005 + +type Frame = { samples: Float32Array; level: number; voiced: boolean } + +function joined(frames: readonly Frame[]): Float32Array { + const length = frames.reduce((sum, frame) => sum + frame.samples.length, 0) + const out = new Float32Array(length) + let offset = 0 + for (const frame of frames) { + out.set(frame.samples, offset) + offset += frame.samples.length + } + return out +} + +/** The index after the quietest run of CUT_FRAMES in the search range. */ +function quietestCut(phrase: readonly Frame[]): number { + let best = CUT_SEARCH_FROM + let bestSum = Infinity + for ( + let start = CUT_SEARCH_FROM; + start + CUT_FRAMES <= phrase.length; + start++ + ) { + let sum = 0 + for (let i = start; i < start + CUT_FRAMES; i++) sum += phrase[i].level + if (sum < bestSum) { + bestSum = sum + best = start + } + } + // Mid-silence: no word straddles the cut, so the pieces need no overlap. + return best + Math.floor(CUT_FRAMES / 2) +} + +/** + * Splits the microphone's 20 ms frames into phrases at the speaker's pauses, + * so each can be transcribed while they go on talking. + */ +export function createPhraseCutter() { + const floor = createNoiseFloor() + let before: Frame[] = [] + let phrase: Frame[] | null = null + + const voicedIn = (frames: readonly Frame[]) => + frames.filter((frame) => frame.voiced).length + + /** The phrase up to its tail; what follows waits as the next one's pre-roll. */ + const close = (frames: Frame[]): Float32Array => { + const lastVoiced = frames.findLastIndex((frame) => frame.voiced) + const end = Math.min(frames.length, lastVoiced + 1 + TAIL_FRAMES) + before = frames.slice(end).slice(-PRE_ROLL_FRAMES) + phrase = null + return joined(frames.slice(0, end)) + } + + return { + /** Takes one frame; answers a phrase when this frame ended one. */ + push(samples: Float32Array, level: number): Float32Array | null { + const threshold = Math.max( + floor.add(level) * SPEECH_OVER_FLOOR, + SPEECH_MIN_LEVEL + ) + const frame = { samples, level, voiced: level > threshold } + + if (phrase === null) { + if (!frame.voiced) { + before.push(frame) + if (before.length > PRE_ROLL_FRAMES) before.shift() + return null + } + phrase = [...before, frame] + before = [] + return null + } + + phrase.push(frame) + const pause = phrase.length - 1 - phrase.findLastIndex((f) => f.voiced) + if (pause >= PAUSE_FRAMES && voicedIn(phrase) >= MIN_VOICE_FRAMES) { + return close(phrase) + } + if (phrase.length >= MAX_PHRASE_FRAMES) { + const cut = quietestCut(phrase) + const rest = phrase.slice(cut) + const done = joined(phrase.slice(0, cut)) + phrase = rest + return done + } + return null + }, + + /** The phrase in progress when the speaker stops; null when there is none. */ + flush(): Float32Array | null { + if (phrase === null || voicedIn(phrase) === 0) return null + return close(phrase) + }, + } +} diff --git a/surfsense_local/frontend/src/features/dictation/capture/resample.ts b/surfsense_local/frontend/src/features/dictation/capture/resample.ts new file mode 100644 index 0000000000..b99a8e3646 --- /dev/null +++ b/surfsense_local/frontend/src/features/dictation/capture/resample.ts @@ -0,0 +1,22 @@ +export const SPEECH_SAMPLE_RATE = 16_000 + +/** + * A phrase captured at the device's rate, at the 16 kHz the model takes. Only + * where a platform refuses a 16 kHz AudioContext; elsewhere Chromium resamples. + */ +export async function resampleForSpeech( + samples: Float32Array, + sampleRate: number +): Promise { + if (sampleRate === SPEECH_SAMPLE_RATE) return samples + const length = Math.ceil((samples.length * SPEECH_SAMPLE_RATE) / sampleRate) + const context = new OfflineAudioContext(1, length, SPEECH_SAMPLE_RATE) + const buffer = context.createBuffer(1, samples.length, sampleRate) + buffer.copyToChannel(new Float32Array(samples), 0) + const source = context.createBufferSource() + source.buffer = buffer + source.connect(context.destination) + source.start() + const rendered = await context.startRendering() + return rendered.getChannelData(0) +} diff --git a/surfsense_local/frontend/src/features/dictation/clamp-to-room.ts b/surfsense_local/frontend/src/features/dictation/clamp-to-room.ts new file mode 100644 index 0000000000..18b86353ff --- /dev/null +++ b/surfsense_local/frontend/src/features/dictation/clamp-to-room.ts @@ -0,0 +1,17 @@ +import { UNSPACED_SCRIPT } from "./join-phrases" + +/** + * The longest start of `text` within `room` characters that ends a word, or + * any character in Chinese, Japanese or Thai, which do not space their words. + */ +export function clampToRoom(text: string, room: number): string { + if (text.length <= room) return text + if (room <= 0) return "" + let cut = room + // Never half of a surrogate pair. + if (/[\uD800-\uDBFF]/.test(text[cut - 1])) cut -= 1 + const edge = text.slice(Math.max(0, cut - 1), cut + 1) + if (UNSPACED_SCRIPT.test(edge)) return text.slice(0, cut) + const space = text.slice(0, cut + 1).search(/\s(?=\S*$)/) + return (space > 0 ? text.slice(0, space) : text.slice(0, cut)).trimEnd() +} diff --git a/surfsense_local/frontend/src/features/dictation/dictation-adapter.test.ts b/surfsense_local/frontend/src/features/dictation/dictation-adapter.test.ts new file mode 100644 index 0000000000..93e912d775 --- /dev/null +++ b/surfsense_local/frontend/src/features/dictation/dictation-adapter.test.ts @@ -0,0 +1,453 @@ +import { afterEach, describe, expect, it, vi } from "vitest" +import { renderHook } from "@testing-library/react" +import { + useExternalStoreRuntime, + type ThreadMessage, +} from "@assistant-ui/react" + +import { QUESTION_MAX_CHARS } from "@/features/chat/question-limit" + +import { fakeMicrophone } from "./capture/fake-microphone" +import { createDictation } from "./dictation-adapter" +import { fakeDictationApi, held, refusal, said } from "./fake-dictation-api" + +function setUp(options: Parameters[0] = {}) { + const api = fakeDictationApi(options) + vi.stubGlobal( + "fetch", + vi.fn( + async (input: RequestInfo | URL, init?: RequestInit) => + (await api.handle(String(input), init)) ?? + Response.json({ detail: "not found" }, { status: 404 }) + ) + ) + const mic = fakeMicrophone() + const dictation = createDictation({ openMicrophone: mic.open }) + const { result } = renderHook(() => + useExternalStoreRuntime({ + messages: [], + onNew: async () => undefined, + adapters: { dictation }, + }) + ) + const composer = result.current.thread.composer + const text = () => composer.getState().text + const listening = () => vi.waitFor(() => expect(mic.state.open).toBe(true)) + return { api, mic, dictation, composer, text, listening } +} + +async function header(wav: Blob) { + const view = new DataView(await wav.arrayBuffer()) + return { + channels: view.getUint16(22, true), + sampleRate: view.getUint32(24, true), + bits: view.getUint16(34, true), + } +} + +afterEach(() => { + vi.useRealTimers() + vi.unstubAllGlobals() +}) + +describe("dictation", () => { + it("is starting until the first real frame, then running", async () => { + const { dictation, mic } = setUp() + const started = vi.fn() + + const session = dictation.listen() + session.onSpeechStart(started) + expect(session.status.type).toBe("starting") + await vi.waitFor(() => expect(mic.state.open).toBe(true)) + expect(session.status.type).toBe("starting") + expect(dictation.getView().phase).toBe("arming") + + mic.quiet(0.02) + + expect(session.status.type).toBe("running") + expect(started).toHaveBeenCalledOnce() + expect(dictation.getView().phase).toBe("listening") + session.cancel() + }) + + it("puts each phrase in the box as it lands, all of it so far", async () => { + const { api, mic, dictation, composer, text, listening } = setUp() + api.answer(said("Hello there."), said("How are you?")) + composer.setText("Note:") + + dictation.start(composer) + await listening() + mic.say() + await vi.waitFor(() => expect(text()).toBe("Note: Hello there.")) + mic.say() + + await vi.waitFor(() => + expect(text()).toBe("Note: Hello there. How are you?") + ) + expect(composer.getState().dictation).toBeDefined() + dictation.cancel() + }) + + it("sends each phrase as a 16 kHz mono 16-bit WAV", async () => { + const { api, mic, dictation, composer, listening } = setUp() + api.answer(said("Hello.")) + + dictation.start(composer) + await listening() + mic.say() + + await vi.waitFor(() => expect(api.phrases).toHaveLength(1)) + expect(await header(api.phrases[0])).toEqual({ + channels: 1, + sampleRate: 16_000, + bits: 16, + }) + dictation.cancel() + }) + + it("on stop, waits for the last phrase and hands over one final result before it ends", async () => { + const { api, mic, dictation, listening } = setUp() + const last = held(() => Response.json(said("Fine, thanks."))) + api.answer(said("How are you?"), last.answer) + const heard: string[] = [] + + const session = dictation.listen() + session.onSpeech(({ transcript, isFinal }) => + heard.push( + `${isFinal ? "final" : "interim"}:${transcript}:${session.status.type}` + ) + ) + await listening() + mic.say() + await vi.waitFor(() => expect(heard).toHaveLength(1)) + mic.speak(0.6) + + const stopped = session.stop() + expect(mic.state.open).toBe(false) + expect(dictation.getView().phase).toBe("finishing") + last.release() + await stopped + + expect(heard).toEqual([ + "interim:How are you?:running", + "interim:How are you? Fine, thanks.:running", + "final:How are you? Fine, thanks.:running", + ]) + expect(session.status).toEqual({ type: "ended", reason: "stopped" }) + expect(dictation.getView().phase).toBe("idle") + await vi.waitFor(() => + expect(api.calls).toContain("DELETE /llm/transcription/dictations/d1") + ) + }) + + it("lands the last phrase in the box when stopped through the runtime", async () => { + const { api, mic, dictation, composer, text, listening } = setUp() + api.answer(said("One."), said("Two.")) + + dictation.start(composer) + await listening() + mic.say() + await vi.waitFor(() => expect(text()).toBe("One.")) + mic.speak(0.8) + dictation.stop() + + await vi.waitFor(() => + expect(composer.getState().dictation).toBeUndefined() + ) + expect(text()).toBe("One. Two.") + }) + + it("inserts nothing when nothing was said", async () => { + const { mic, dictation, composer, text, listening } = setUp() + composer.setText("Draft") + + dictation.start(composer) + await listening() + mic.quiet(1) + dictation.stop() + + await vi.waitFor(() => + expect(composer.getState().dictation).toBeUndefined() + ) + expect(text()).toBe("Draft") + }) + + it("joins Chinese and Japanese phrases with no space, and others with one", async () => { + const { api, mic, dictation, composer, text, listening } = setUp() + api.answer( + said("你好。", "zh"), + said("今天天气很好。", "zh"), + said("ありがとう。", "ja"), + said("OK, thanks.", "en") + ) + + dictation.start(composer) + await listening() + for (let i = 0; i < 4; i++) mic.say() + + await vi.waitFor(() => + expect(text()).toBe("你好。今天天气很好。ありがとう。 OK, thanks.") + ) + dictation.cancel() + }) + + it("stops at the message limit, cut at the last whole word", async () => { + const { api, mic, dictation, composer, text, listening } = setUp() + // Room for 20 characters after the separating space. + const before = "x".repeat(QUESTION_MAX_CHARS - 21) + composer.setText(before) + api.answer(said("Seven words fit here and then"), said("never sent")) + + dictation.start(composer) + await listening() + mic.say() + + await vi.waitFor(() => + expect(dictation.getView()).toEqual({ + phase: "problem", + problem: { kind: "at_limit" }, + canRetry: false, + }) + ) + expect(text()).toBe(`${before} Seven words fit here`) + expect(text().length).toBeLessThanOrEqual(QUESTION_MAX_CHARS) + expect(mic.state.open).toBe(false) + mic.say() + expect(api.phrases).toHaveLength(1) + }) + + it("cuts Japanese at the limit by the character", async () => { + const { api, mic, dictation, composer, text, listening } = setUp() + const before = "x".repeat(QUESTION_MAX_CHARS - 6) + composer.setText(before) + api.answer(said("今日はとても良い天気です。", "ja")) + + dictation.start(composer) + await listening() + mic.say() + + await vi.waitFor(() => expect(text()).toBe(`${before} 今日はとて`)) + }) + + it("on cancel, drops what was not transcribed and lands nothing later", async () => { + const { api, mic, dictation, composer, text, listening } = setUp() + const late = held(() => Response.json(said("Too late."))) + api.answer(said("Kept?"), late.answer) + composer.setText("Before") + + dictation.start(composer) + await listening() + mic.say() + await vi.waitFor(() => expect(text()).toBe("Before Kept?")) + mic.say() + await vi.waitFor(() => expect(api.phrases).toHaveLength(2)) + + dictation.cancel(composer) + late.release() + + expect(mic.state.open).toBe(false) + expect(text()).toBe("Before") + expect(dictation.getView().phase).toBe("idle") + await vi.waitFor(() => + expect(api.calls).toContain("DELETE /llm/transcription/dictations/d1") + ) + await new Promise((resolve) => setTimeout(resolve, 10)) + expect(text()).toBe("Before") + }) + + it("keeps a phrase that failed twice, and Retry lands it", async () => { + vi.useFakeTimers({ toFake: ["setTimeout", "setInterval", "Date"] }) + const { api, mic, dictation, composer, text } = setUp() + api.answer( + said("First part."), + () => refusal(503, "transcription_unavailable"), + () => refusal(503, "transcription_unavailable") + ) + + dictation.start(composer) + await vi.waitFor(() => expect(mic.state.open).toBe(true)) + mic.say() + await vi.waitFor(() => expect(text()).toBe("First part.")) + mic.say() + await vi.advanceTimersByTimeAsync(2_000) + + await vi.waitFor(() => + expect(dictation.getView()).toEqual({ + phase: "problem", + problem: { kind: "phrase_failed" }, + canRetry: true, + }) + ) + expect(mic.state.open).toBe(false) + expect(text()).toBe("First part.") + + api.answer(said("Second part.")) + dictation.retry(composer) + + await vi.waitFor(() => expect(text()).toBe("First part. Second part.")) + await vi.waitFor(() => expect(dictation.getView().phase).toBe("idle")) + expect(mic.state.opens).toBe(1) + }) + + it("says why when the microphone is refused, and lets the API go", async () => { + const { api, mic, dictation, composer, text } = setUp() + mic.refuse(new DOMException("Permission denied", "NotAllowedError")) + composer.setText("Draft") + + dictation.start(composer) + + await vi.waitFor(() => + expect(dictation.getView()).toEqual({ + phase: "problem", + problem: { kind: "microphone_off" }, + canRetry: false, + }) + ) + await vi.waitFor(() => + expect(api.calls).toContain("DELETE /llm/transcription/dictations/d1") + ) + expect(text()).toBe("Draft") + }) + + it.each([ + ["NotFoundError", "no_microphone"], + ["NotReadableError", "microphone_busy"], + ])("names a %s as %s", async (name, kind) => { + const { mic, dictation, composer } = setUp() + mic.refuse(new DOMException("", name)) + + dictation.start(composer) + + await vi.waitFor(() => + expect(dictation.getView()).toMatchObject({ problem: { kind } }) + ) + }) + + it("is refused at once when memory is short, before anyone speaks", async () => { + const { mic, dictation, composer } = setUp({ + open: () => + refusal(409, "transcription_needs_memory", { + needed_bytes: 2_900_000_000, + available_bytes: 1_200_000_000, + }), + }) + + dictation.start(composer) + + await vi.waitFor(() => + expect(dictation.getView()).toEqual({ + phase: "problem", + problem: { + kind: "transcription_needs_memory", + neededBytes: 2_900_000_000, + availableBytes: 1_200_000_000, + message: "Refused: transcription_needs_memory", + }, + canRetry: false, + }) + ) + await vi.waitFor(() => + expect(mic.state).toMatchObject({ opens: 1, open: false }) + ) + }) + + it("keeps what was said when the microphone is unplugged", async () => { + const { api, mic, dictation, composer, text, listening } = setUp() + api.answer(said("Before the cable."), said("Last words.")) + + dictation.start(composer) + await listening() + mic.say() + await vi.waitFor(() => expect(text()).toBe("Before the cable.")) + mic.speak(0.6) + mic.unplug() + + await vi.waitFor(() => + expect(dictation.getView()).toMatchObject({ + problem: { kind: "microphone_disconnected" }, + }) + ) + expect(text()).toBe("Before the cable. Last words.") + }) + + it("stops, keeping the text, when the API no longer knows the session", async () => { + const { api, mic, dictation, composer, text, listening } = setUp() + api.answer(said("Saved."), () => refusal(404, "dictation_ended")) + + dictation.start(composer) + await listening() + mic.say() + mic.say() + + await vi.waitFor(() => + expect(dictation.getView()).toMatchObject({ + problem: { kind: "dictation_ended" }, + }) + ) + expect(text()).toBe("Saved.") + expect(mic.state.open).toBe(false) + }) + + it("stops when the speech model was removed mid-recording", async () => { + const { api, mic, dictation, composer, listening } = setUp() + api.answer(() => refusal(409, "needs_speech_model")) + + dictation.start(composer) + await listening() + mic.say() + + await vi.waitFor(() => + expect(dictation.getView()).toMatchObject({ + problem: { kind: "model_removed" }, + }) + ) + }) + + it("warns at 4:30 and stops at 5:00, keeping the text", async () => { + vi.useFakeTimers({ toFake: ["setTimeout", "setInterval", "Date"] }) + const { api, mic, dictation, composer, text } = setUp() + api.answer(said("A long thought.")) + + dictation.start(composer) + await vi.waitFor(() => expect(mic.state.open).toBe(true)) + mic.say() + await vi.waitFor(() => expect(text()).toBe("A long thought.")) + + await vi.advanceTimersByTimeAsync(269_000) + expect(dictation.getView()).toMatchObject({ warning: false }) + await vi.advanceTimersByTimeAsync(1_000) + expect(dictation.getView()).toMatchObject({ + phase: "listening", + warning: true, + }) + await vi.advanceTimersByTimeAsync(30_000) + + await vi.waitFor(() => + expect(dictation.getView()).toMatchObject({ + problem: { kind: "time_limit" }, + }) + ) + expect(mic.state.open).toBe(false) + expect(text()).toBe("A long thought.") + }) + + it("says the model is loading only once a phrase has waited 2 s for it", async () => { + vi.useFakeTimers({ toFake: ["setTimeout", "setInterval", "Date"] }) + const { api, mic, dictation, composer, text } = setUp({ warm: "loading" }) + const first = held(() => Response.json(said("Loaded."))) + api.answer(first.answer) + + dictation.start(composer) + await vi.waitFor(() => expect(mic.state.open).toBe(true)) + mic.say() + await vi.advanceTimersByTimeAsync(1_900) + expect(dictation.getView()).toMatchObject({ loading: false }) + await vi.advanceTimersByTimeAsync(100) + expect(dictation.getView()).toMatchObject({ loading: true }) + + first.release() + await vi.waitFor(() => expect(text()).toBe("Loaded.")) + expect(dictation.getView()).toMatchObject({ loading: false }) + dictation.cancel() + }) +}) diff --git a/surfsense_local/frontend/src/features/dictation/dictation-adapter.ts b/surfsense_local/frontend/src/features/dictation/dictation-adapter.ts new file mode 100644 index 0000000000..c8b5b06629 --- /dev/null +++ b/surfsense_local/frontend/src/features/dictation/dictation-adapter.ts @@ -0,0 +1,149 @@ +import type { DictationAdapter } from "@assistant-ui/react" + +import { QUESTION_MAX_CHARS } from "@/features/chat/question-limit" + +import { + openMicrophone as browserMicrophone, + type OpenMicrophone, +} from "./capture/open-microphone" +import type { DictationProblem } from "./dictation-problem" +import { Recording, type RecordingView } from "./recording" + +/** What the mic and the strip show. */ +export type DictationView = + | { phase: "idle" } + | RecordingView + | { phase: "problem"; problem: DictationProblem; canRetry: boolean } + +/** The composer methods dictation drives; assistant-ui's composer has them. */ +export type DictationComposer = { + getState: () => { text: string } + setText: (text: string) => void + startDictation: () => void +} + +export type Dictation = DictationAdapter & { + subscribe: (listener: () => void) => () => void + getView: () => DictationView + /** The microphone's latest RMS level, every 20 ms while listening. */ + subscribeLevel: (listener: () => void) => () => void + getLevel: () => number + /** The mic: listens, counting the limit from the text already there. */ + start: (composer: DictationComposer) => void + /** Click, Enter, Send and a push-to-talk release: the text is kept. */ + stop: () => void + /** Escape and ✕ pass the composer, which gets its text from before the mic. */ + cancel: (composer?: Pick) => void + /** Transcribes the audio a failed phrase left, with no microphone. */ + retry: (composer: DictationComposer) => void + dismiss: () => void +} + +const IDLE: DictationView = { phase: "idle" } + +/** Room for dictated text: assistant-ui adds a space after text that lacks one. */ +function roomAfter(text: string): number { + const separator = text && !text.endsWith(" ") ? 1 : 0 + return QUESTION_MAX_CHARS - text.length - separator +} + +/** + * Voice input for the chat composer, as assistant-ui's DictationAdapter. One + * per chat runtime: it outlives a thread switch, which cancels it. + */ +export function createDictation({ + openMicrophone = browserMicrophone, +}: { openMicrophone?: OpenMicrophone } = {}): Dictation { + let view: DictationView = IDLE + let level = 0 + const viewListeners = new Set<() => void>() + const levelListeners = new Set<() => void>() + let recording: Recording | null = null + let textBefore = "" + let kept: Blob[] = [] + let retrying = false + + const show = (next: DictationView) => { + view = next + for (const listener of [...viewListeners]) listener() + } + + const listen = (): DictationAdapter.Session => { + recording?.cancel() + const toRetry = retrying ? kept : undefined + retrying = false + kept = [] + const current: Recording = new Recording({ + room: roomAfter(textBefore), + openMicrophone, + kept: toRetry, + onView: (next) => { + if (recording === current) show(next) + }, + onLevel: (next) => { + if (recording !== current) return + level = next + for (const listener of [...levelListeners]) listener() + }, + onEnd: ({ problem, kept: left }) => { + if (recording !== current) return + recording = null + level = 0 + kept = problem?.kind === "phrase_failed" ? left : [] + show( + problem + ? { phase: "problem", problem, canRetry: kept.length > 0 } + : IDLE + ) + }, + }) + recording = current + current.begin() + return current + } + + return { + listen, + // The textarea goes read-only instead, keeping its focus and its keys. + disableInputDuringDictation: false, + subscribe: (listener) => { + viewListeners.add(listener) + return () => void viewListeners.delete(listener) + }, + getView: () => view, + subscribeLevel: (listener) => { + levelListeners.add(listener) + return () => void levelListeners.delete(listener) + }, + getLevel: () => level, + start: (composer) => { + if (recording) return + textBefore = composer.getState().text + composer.startDictation() + }, + stop: () => { + void recording?.stop() + }, + cancel: (composer) => { + kept = [] + const current = recording + if (current) { + current.cancel() + composer?.setText(textBefore) + } else if (view.phase !== "idle") { + show(IDLE) + } + }, + retry: (composer) => { + if (recording || kept.length === 0) return + textBefore = composer.getState().text + retrying = true + composer.startDictation() + }, + dismiss: () => { + if (recording) return + kept = [] + show(IDLE) + }, + } +} diff --git a/surfsense_local/frontend/src/features/dictation/dictation-composer.test.tsx b/surfsense_local/frontend/src/features/dictation/dictation-composer.test.tsx new file mode 100644 index 0000000000..65fdbedbb9 --- /dev/null +++ b/surfsense_local/frontend/src/features/dictation/dictation-composer.test.tsx @@ -0,0 +1,368 @@ +import { afterEach, describe, expect, it, vi } from "vitest" +import { act, cleanup, screen, waitFor } from "@testing-library/react" +import userEvent from "@testing-library/user-event" +import { + AssistantRuntimeProvider, + useExternalStoreRuntime, + type ThreadMessage, +} from "@assistant-ui/react" + +import { ChatComposer } from "@/features/chat/chat-composer" +import { QUESTION_MAX_CHARS } from "@/features/chat/question-limit" +import { + fakeSpeechRow, + SPEECH_IN_USE, +} from "@/features/models/local/transcription/fake-speech-rows" +import type { ModelSelection } from "@/features/models/selection/api" +import { render } from "@/test-utils" + +import { fakeMicrophone } from "./capture/fake-microphone" +import { createDictation, type Dictation } from "./dictation-adapter" +import { fakeDictationApi, held, refusal, said } from "./fake-dictation-api" + +const MODEL: ModelSelection = { + model_type: "text_gen", + provider: "llamacpp", + connection_id: null, + name: "Qwen3-1.7B-Q4_K_M", + updated_at: "2026-09-05T00:00:00Z", +} + +function Harness({ + dictation, + model = MODEL, + onNew = async () => undefined, + onOpenSpeechSettings, +}: { + dictation: Dictation + model?: ModelSelection | null + onNew?: () => Promise + onOpenSpeechSettings?: () => void +}) { + const runtime = useExternalStoreRuntime({ + messages: [], + onNew, + adapters: { dictation }, + }) + return ( + + undefined} + onModelSelected={() => undefined} + readsImages={false} + dictation={dictation} + onOpenSpeechSettings={onOpenSpeechSettings} + /> + + ) +} + +function setUp( + options: Parameters[0] = {}, + other: (path: string) => Response | null = () => null +) { + const api = fakeDictationApi(options) + vi.stubGlobal( + "fetch", + vi.fn( + async (input: RequestInfo | URL, init?: RequestInit) => + (await api.handle(String(input), init)) ?? + other(String(input)) ?? + Response.json([]) + ) + ) + const mic = fakeMicrophone() + const dictation = createDictation({ openMicrophone: mic.open }) + const user = userEvent.setup() + const box = () => + screen.getByRole("textbox", { name: "Message" }) + const listening = () => waitFor(() => expect(mic.state.open).toBe(true)) + return { api, mic, dictation, user, box, listening } +} + +const startButton = () => + screen.findByRole("button", { name: "Start voice input" }) + +afterEach(() => { + cleanup() + vi.unstubAllGlobals() +}) + +describe("voice input in the composer", () => { + it("listens on a click and puts each phrase in the box as it lands", async () => { + const { api, mic, dictation, user, box, listening } = setUp() + api.answer(said("Hello there."), said("How are you?")) + render() + + await user.click(await startButton()) + expect(screen.getByRole("status").textContent).toBe( + "Starting the microphone…" + ) + await listening() + act(() => mic.say()) + + await waitFor(() => expect(box().value).toBe("Hello there.")) + expect(box().readOnly).toBe(true) + expect(screen.getByRole("status").textContent).toBe("Listening") + expect(screen.getByText("0:00")).toBeTruthy() + expect( + screen + .getByRole("button", { name: "Stop voice input" }) + .getAttribute("aria-pressed") + ).toBe("true") + act(() => mic.say()) + await waitFor(() => expect(box().value).toBe("Hello there. How are you?")) + }) + + it("on Enter, stops, waits for the last phrase and inserts it, sending nothing", async () => { + const { api, mic, dictation, user, box, listening } = setUp() + const last = held(() => Response.json(said("Second."))) + api.answer(said("First."), last.answer) + const onNew = vi.fn(async () => undefined) + render() + + await user.click(await startButton()) + await listening() + act(() => mic.say()) + await waitFor(() => expect(box().value).toBe("First.")) + act(() => mic.speak(0.8)) + await user.type(box(), "{Enter}") + + expect(mic.state.open).toBe(false) + expect(await screen.findByText("Transcribing…")).toBeTruthy() + last.release() + await waitFor(() => expect(box().value).toBe("First. Second.")) + await waitFor(() => expect(box().readOnly).toBe(false)) + expect(onNew).not.toHaveBeenCalled() + }) + + it("on Send, stops and inserts instead of sending", async () => { + const { api, mic, dictation, user, box, listening } = setUp() + api.answer(said("Draft question."), said("More.")) + const onNew = vi.fn(async () => undefined) + render() + + await user.click(await startButton()) + await listening() + act(() => mic.say()) + await waitFor(() => expect(box().value).toBe("Draft question.")) + act(() => mic.speak(0.8)) + await user.click(screen.getByRole("button", { name: "Send message" })) + + await waitFor(() => expect(box().value).toBe("Draft question. More.")) + await waitFor(() => expect(box().readOnly).toBe(false)) + expect(onNew).not.toHaveBeenCalled() + }) + + it("on Escape, drops the recording and puts back the text from before the mic", async () => { + const { api, mic, dictation, user, box, listening } = setUp() + api.answer(said("Never mind.")) + render() + await user.type(box(), "Before") + + await user.click(await startButton()) + await listening() + act(() => mic.say()) + await waitFor(() => expect(box().value).toBe("Before Never mind.")) + await user.type(box(), "{Escape}") + + expect(box().value).toBe("Before") + expect(mic.state.open).toBe(false) + expect(box().readOnly).toBe(false) + expect(await startButton()).toBeTruthy() + }) + + it("stops on a second click and keeps the text", async () => { + const { api, mic, dictation, user, box, listening } = setUp() + api.answer(said("Keep this.")) + render() + + await user.click(await startButton()) + await listening() + act(() => mic.speak(1.2)) + await user.click(screen.getByRole("button", { name: "Stop voice input" })) + + await waitFor(() => expect(box().value).toBe("Keep this.")) + expect(mic.state.open).toBe(false) + expect(await startButton()).toBeTruthy() + }) + + it("listens while the mic is held, and stops on release", async () => { + const { api, mic, dictation, user, box, listening } = setUp() + api.answer(said("Push to talk.")) + render() + const button = await startButton() + + await user.pointer({ keys: "[MouseLeft>]", target: button }) + await listening() + act(() => mic.speak(1.2)) + await new Promise((resolve) => setTimeout(resolve, 320)) + await user.pointer({ keys: "[/MouseLeft]", target: button }) + + expect(mic.state.open).toBe(false) + await waitFor(() => expect(box().value).toBe("Push to talk.")) + }) + + it("toggles with Enter on the focused mic", async () => { + const { api, mic, dictation, user, box, listening } = setUp() + api.answer(said("By keyboard.")) + render() + + ;(await startButton()).focus() + await user.keyboard("{Enter}") + await listening() + act(() => mic.speak(1.2)) + await user.keyboard("{Enter}") + + await waitFor(() => expect(box().value).toBe("By keyboard.")) + expect(mic.state.open).toBe(false) + }) + + it("sets up speech recognition instead when no model is chosen", async () => { + const { dictation, user, mic } = setUp({ + status: { ready: false, reason: "needs_speech_model", model: null }, + }) + render() + + await user.click( + await screen.findByRole("button", { name: "Set up voice input" }) + ) + + expect( + await screen.findByRole("dialog", { name: "Set up speech recognition" }) + ).toBeTruthy() + expect(mic.state.opens).toBe(0) + }) + + it("starts talking from the set-up once a speech model is in use", async () => { + const { api, dictation, user, mic, box, listening } = setUp( + { status: { ready: false, reason: "needs_speech_model", model: null } }, + (path) => + path === "/llm/catalog/local" + ? Response.json({ + rows: [fakeSpeechRow({}, SPEECH_IN_USE)], + recommended_id: null, + }) + : null + ) + api.answer(said("Set up and talking.")) + render() + + await user.click( + await screen.findByRole("button", { name: "Set up voice input" }) + ) + await user.click( + await screen.findByRole("button", { name: "Start talking" }) + ) + await listening() + act(() => mic.say()) + + await waitFor(() => expect(box().value).toBe("Set up and talking.")) + expect( + screen.getByRole("button", { name: "Stop voice input" }) + ).toBeTruthy() + }) + + it("is off whenever the message box is, and at the message limit", async () => { + const { dictation, user, box } = setUp() + const { rerender } = render() + expect((await startButton()).hasAttribute("disabled")).toBe(true) + + rerender() + expect((await startButton()).hasAttribute("disabled")).toBe(false) + await user.click(box()) + await user.paste("x".repeat(QUESTION_MAX_CHARS)) + + expect((await startButton()).hasAttribute("disabled")).toBe(true) + }) + + it("says the microphone is off, and opens the system’s settings", async () => { + const openSettings = vi.fn(async () => true) + const { mic, dictation, user } = setUp() + vi.stubGlobal("surfsense", { + apiUrl: "", + platform: "win32", + microphone: { + status: async () => "denied", + request: async () => "denied", + openSettings, + }, + }) + mic.refuse(new DOMException("Permission denied", "NotAllowedError")) + render() + + await user.click(await startButton()) + + expect((await screen.findByRole("alert")).textContent).toBe( + "Microphone access is off for SurfSense." + ) + await user.click(screen.getByRole("button", { name: "Open settings" })) + expect(openSettings).toHaveBeenCalledOnce() + }) + + it("keeps a phrase it could not transcribe, and Retry puts it in the box", async () => { + const { api, mic, dictation, user, box, listening } = setUp() + api.answer(said("Got this."), () => + Response.json({ detail: [{ msg: "bad audio" }] }, { status: 422 }) + ) + render() + + await user.click(await startButton()) + await listening() + act(() => mic.say()) + act(() => mic.say()) + + expect((await screen.findByRole("alert")).textContent).toBe( + "Couldn’t turn part of your speech into text." + ) + expect(box().value).toBe("Got this.") + api.answer(said("And this.")) + await user.click(screen.getByRole("button", { name: "Retry" })) + + await waitFor(() => expect(box().value).toBe("Got this. And this.")) + await waitFor(() => expect(screen.queryByRole("alert")).toBeNull()) + }) + + it("names both sizes when memory is short", async () => { + const { dictation, user } = setUp({ + open: () => + refusal(409, "transcription_needs_memory", { + needed_bytes: 2_900_000_000, + available_bytes: 1_200_000_000, + }), + }) + render() + + await user.click(await startButton()) + + expect((await screen.findByRole("alert")).textContent).toBe( + "Transcribing needs about 2.9 GB free; this computer has 1.2 GB." + ) + await user.click(screen.getByRole("button", { name: "Dismiss" })) + expect(screen.queryByRole("alert")).toBeNull() + }) + + it("says why it stopped at the message limit", async () => { + const { api, mic, dictation, user, box, listening } = setUp() + api.answer(said("one two three four five")) + render() + await user.click(box()) + await user.paste("x".repeat(QUESTION_MAX_CHARS - 10)) + + await user.click(await startButton()) + await listening() + act(() => mic.say()) + + await waitFor(() => + expect(screen.getByRole("status").textContent).toBe( + "Voice input stopped at the 4,096-character message limit." + ) + ) + expect(box().value).toBe(`${"x".repeat(QUESTION_MAX_CHARS - 10)} one two`) + }) +}) diff --git a/surfsense_local/frontend/src/features/dictation/dictation-problem.ts b/surfsense_local/frontend/src/features/dictation/dictation-problem.ts new file mode 100644 index 0000000000..f6e9375961 --- /dev/null +++ b/surfsense_local/frontend/src/features/dictation/dictation-problem.ts @@ -0,0 +1,56 @@ +import { ApiError } from "@/lib/api" + +/** Why a recording ended other than by the user, or could not start. */ +export type DictationProblem = + | { + kind: + | "microphone_off" + | "no_microphone" + | "microphone_busy" + | "microphone_failed" + | "microphone_disconnected" + | "phrase_failed" + | "at_limit" + | "time_limit" + | "dictation_ended" + | "model_removed" + | "needs_speech_model" + | "transcription_unavailable" + } + | { + kind: "transcription_needs_memory" + neededBytes: number + availableBytes: number + /** The API's English sentence, for when the sizes are missing. */ + message: string + } + +/** getUserMedia's own error names, and the system switch read as NotAllowedError. */ +export function microphoneProblem(error: unknown): DictationProblem { + const name = error instanceof DOMException ? error.name : "" + if (name === "NotAllowedError" || name === "SecurityError") { + return { kind: "microphone_off" } + } + if (name === "NotFoundError" || name === "OverconstrainedError") { + return { kind: "no_microphone" } + } + if (name === "NotReadableError" || name === "AbortError") { + return { kind: "microphone_busy" } + } + return { kind: "microphone_failed" } +} + +/** A refusal to open a session, by its code; anything else is a server not answering. */ +export function openingProblem(error: unknown): DictationProblem { + if (!(error instanceof ApiError)) return { kind: "transcription_unavailable" } + if (error.code === "needs_speech_model") return { kind: "needs_speech_model" } + if (error.code === "transcription_needs_memory") { + return { + kind: "transcription_needs_memory", + neededBytes: Number(error.detail.needed_bytes), + availableBytes: Number(error.detail.available_bytes), + message: error.message, + } + } + return { kind: "transcription_unavailable" } +} diff --git a/surfsense_local/frontend/src/features/dictation/dictation-session.test.ts b/surfsense_local/frontend/src/features/dictation/dictation-session.test.ts new file mode 100644 index 0000000000..2b80be218d --- /dev/null +++ b/surfsense_local/frontend/src/features/dictation/dictation-session.test.ts @@ -0,0 +1,152 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" + +import { ApiError } from "@/lib/api" + +import { openDictationSession } from "./dictation-session" +import { fakeDictationApi, held, refusal, said } from "./fake-dictation-api" + +const wav = (label: string) => new Blob([label], { type: "audio/wav" }) + +function serve(api: ReturnType) { + vi.stubGlobal( + "fetch", + vi.fn( + async (input: RequestInfo | URL, init?: RequestInit) => + (await api.handle(String(input), init)) ?? + Response.json({ detail: "not found" }, { status: 404 }) + ) + ) +} + +beforeEach(() => { + vi.useFakeTimers() +}) + +afterEach(() => { + vi.useRealTimers() + vi.unstubAllGlobals() +}) + +describe("the dictation session", () => { + it("opens a session and says whether the model is already loaded", async () => { + const api = fakeDictationApi({ warm: "loading" }) + serve(api) + + const session = await openDictationSession({ onEnded: () => undefined }) + + expect(session.warm).toBe("loading") + expect(api.calls).toEqual(["POST /llm/transcription/dictations"]) + session.end() + }) + + it("sends one phrase at a time, in the order they were spoken", async () => { + const api = fakeDictationApi() + const first = held(() => Response.json(said("Hello there."))) + api.answer(first.answer, said("How are you?")) + serve(api) + const session = await openDictationSession({ onEnded: () => undefined }) + + const one = session.transcribe(wav("one")) + const two = session.transcribe(wav("two")) + await vi.advanceTimersByTimeAsync(0) + expect(api.phrases).toHaveLength(1) + + first.release() + expect((await one).text).toBe("Hello there.") + expect((await two).text).toBe("How are you?") + expect(await api.phrases[1].text()).toBe("two") + session.end() + }) + + it("asks once more, 2 s later, while the server restarts", async () => { + const api = fakeDictationApi() + api.answer( + () => refusal(503, "transcription_unavailable"), + said("Still here.") + ) + serve(api) + const session = await openDictationSession({ onEnded: () => undefined }) + + const phrase = session.transcribe(wav("one")) + await vi.advanceTimersByTimeAsync(1_999) + expect(api.phrases).toHaveLength(1) + await vi.advanceTimersByTimeAsync(1) + + expect((await phrase).text).toBe("Still here.") + expect(api.phrases).toHaveLength(2) + session.end() + }) + + it("gives a phrase up when the second try fails too, and goes on with the next", async () => { + const api = fakeDictationApi() + api.answer( + () => refusal(503, "transcription_unavailable"), + () => refusal(503, "transcription_unavailable"), + said("Next one.") + ) + serve(api) + const session = await openDictationSession({ onEnded: () => undefined }) + + const failed = session.transcribe(wav("one")) + const next = session.transcribe(wav("two")) + const outcome = failed.catch((error: unknown) => error) + await vi.advanceTimersByTimeAsync(2_000) + + const error = await outcome + expect(error).toBeInstanceOf(ApiError) + expect((error as ApiError).code).toBe("transcription_unavailable") + expect((await next).text).toBe("Next one.") + session.end() + }) + + it("does not ask again for a phrase the server refused for itself", async () => { + const api = fakeDictationApi() + api.answer(() => refusal(409, "needs_speech_model")) + serve(api) + const session = await openDictationSession({ onEnded: () => undefined }) + + const outcome = session + .transcribe(wav("one")) + .catch((error: unknown) => error) + await vi.advanceTimersByTimeAsync(5_000) + + expect(((await outcome) as ApiError).code).toBe("needs_speech_model") + expect(api.phrases).toHaveLength(1) + session.end() + }) + + it("keeps itself alive every 10 s until it ends, then says so once", async () => { + const api = fakeDictationApi() + serve(api) + const session = await openDictationSession({ onEnded: () => undefined }) + + await vi.advanceTimersByTimeAsync(25_000) + session.end() + session.end() + await vi.advanceTimersByTimeAsync(30_000) + + expect(api.calls).toEqual([ + "POST /llm/transcription/dictations", + "POST /llm/transcription/dictations/d1/keepalive", + "POST /llm/transcription/dictations/d1/keepalive", + "DELETE /llm/transcription/dictations/d1", + ]) + }) + + it("tells its owner when the API no longer knows it", async () => { + const api = fakeDictationApi() + serve(api) + vi.mocked(fetch).mockImplementation( + async (input: RequestInfo | URL, init?: RequestInit) => + String(input).endsWith("/keepalive") + ? refusal(404, "dictation_ended") + : ((await api.handle(String(input), init)) as Response) + ) + const onEnded = vi.fn() + await openDictationSession({ onEnded }) + + await vi.advanceTimersByTimeAsync(10_000) + + expect(onEnded).toHaveBeenCalledOnce() + }) +}) diff --git a/surfsense_local/frontend/src/features/dictation/dictation-session.ts b/surfsense_local/frontend/src/features/dictation/dictation-session.ts new file mode 100644 index 0000000000..8a7606c212 --- /dev/null +++ b/surfsense_local/frontend/src/features/dictation/dictation-session.ts @@ -0,0 +1,88 @@ +import { ApiError } from "@/lib/api" + +import { + endDictation, + keepDictationAlive, + openDictation, + transcribePhrase, + type PhraseText, +} from "./api" + +// The API ends a session after 20 s without a phrase or a keepalive. +const KEEPALIVE_MS = 10_000 +const RETRY_AFTER_MS = 2_000 + +export type DictationSession = { + warm: "loaded" | "loading" + /** Queued behind the phrases before it: one is in flight at a time. */ + transcribe: (wav: Blob) => Promise + /** Stops the keepalive and releases the API's session; safe to repeat. */ + end: () => void +} + +/** The API no longer knows the session, as after it restarted. */ +export function isDictationEnded(error: unknown): boolean { + return ( + error instanceof ApiError && + error.status === 404 && + error.code === "dictation_ended" + ) +} + +// A refused connection or a server mid-restart; not a refusal of the phrase. +function worthAnotherTry(error: unknown): boolean { + return ( + error instanceof TypeError || + (error instanceof ApiError && error.status === 503) + ) +} + +const pause = (ms: number) => + new Promise((resolve) => setTimeout(resolve, ms)) + +/** One recording's session with the API, from the mic press to its last phrase. */ +export async function openDictationSession({ + onEnded, +}: { + /** The API dropped the session, which a keepalive found out. */ + onEnded: () => void +}): Promise { + const { id, warm } = await openDictation() + let ended = false + let queue: Promise = Promise.resolve() + + const end = () => { + if (ended) return + ended = true + clearInterval(keepalive) + void endDictation(id).catch(() => undefined) + } + + const keepalive = setInterval(() => { + keepDictationAlive(id).catch((error: unknown) => { + if (!isDictationEnded(error) || ended) return + end() + onEnded() + }) + }, KEEPALIVE_MS) + + const send = async (wav: Blob) => { + try { + return await transcribePhrase(id, wav) + } catch (error) { + if (!worthAnotherTry(error)) throw error + await pause(RETRY_AFTER_MS) + return transcribePhrase(id, wav) + } + } + + return { + warm, + transcribe: (wav) => { + const result = queue.then(() => send(wav)) + queue = result.catch(() => undefined) + return result + }, + end, + } +} diff --git a/surfsense_local/frontend/src/features/dictation/fake-dictation-api.ts b/surfsense_local/frontend/src/features/dictation/fake-dictation-api.ts new file mode 100644 index 0000000000..f40c98d790 --- /dev/null +++ b/surfsense_local/frontend/src/features/dictation/fake-dictation-api.ts @@ -0,0 +1,104 @@ +import type { PhraseText, TranscriptionStatus } from "./api" + +const READY: TranscriptionStatus = { + ready: true, + reason: null, + model: "nemotron-3.5-asr-streaming-0.6b-q8_0", + language_intent: "auto", + languages: ["en", "ja", "zh"], +} + +/** What one phrase upload answers: a result, or a response made when asked. */ +export type PhraseAnswer = PhraseText | (() => Response | Promise) + +/** The API's refusal, in the app's `{detail: {message, code, …}}` shape. */ +export function refusal( + status: number, + code: string, + extra: Record = {} +): Response { + return Response.json( + { detail: { message: `Refused: ${code}`, code, ...extra } }, + { status } + ) +} + +/** + * Stands in for the transcription routes, so a test can say what each phrase + * comes back as, and see what the composer sent. Shared by every dictation test. + */ +export function fakeDictationApi({ + status = READY, + warm = "loaded", + open, +}: { + status?: Partial + warm?: "loaded" | "loading" + /** In place of a session: a refusal, say. */ + open?: () => Response +} = {}) { + const calls: string[] = [] + const phrases: Blob[] = [] + const answers: PhraseAnswer[] = [] + let current: TranscriptionStatus = { ...READY, ...status } + let opened = 0 + + const handle = ( + path: string, + init?: RequestInit + ): Response | Promise | null => { + if (!path.startsWith("/llm/transcription/")) return null + const method = init?.method ?? "GET" + calls.push(`${method} ${path}`) + if (path === "/llm/transcription/status") return Response.json(current) + if (path === "/llm/transcription/dictations" && method === "POST") { + if (open) return open() + opened += 1 + return Response.json({ id: `d${opened}`, warm }, { status: 201 }) + } + if (method === "POST" && path.endsWith("/phrases")) { + phrases.push((init?.body as FormData).get("audio") as Blob) + const answer = answers.shift() ?? { + text: "", + language: null, + no_speech: true, + } + return typeof answer === "function" ? answer() : Response.json(answer) + } + if ( + (method === "POST" && path.endsWith("/keepalive")) || + method === "DELETE" + ) { + return new Response(null, { status: 204 }) + } + return Response.json({ detail: "not found" }, { status: 404 }) + } + + return { + handle, + calls, + /** Every phrase the composer sent, as the WAV it uploaded. */ + phrases, + /** What the next phrases come back as, in order. */ + answer: (...next: PhraseAnswer[]) => { + answers.push(...next) + }, + setStatus: (next: Partial) => { + current = { ...current, ...next } + }, + } +} + +/** A phrase's text, as Nemotron detected its language. */ +export function said(text: string, language: string | null = "en"): PhraseText { + return { text, language, no_speech: false } +} + +/** A response the test lets go of when it chooses. */ +export function held(response: () => Response) { + let release = () => {} + const gate = new Promise((resolve) => { + release = resolve + }) + return { answer: () => gate.then(response), release: () => release() } +} diff --git a/surfsense_local/frontend/src/features/dictation/join-phrases.ts b/surfsense_local/frontend/src/features/dictation/join-phrases.ts new file mode 100644 index 0000000000..9154923895 --- /dev/null +++ b/surfsense_local/frontend/src/features/dictation/join-phrases.ts @@ -0,0 +1,36 @@ +export type TranscribedPhrase = { text: string; language: string | null } + +// Written without spaces between words, so their phrases join without one. +const UNSPACED_LANGUAGES = new Set(["zh", "ja", "yue", "th"]) +// For a phrase whose language went unnamed: Han, kana and Thai. +export const UNSPACED_SCRIPT = + /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Thai}]/u + +function unspaced(phrase: TranscribedPhrase, edge: string): boolean { + if (phrase.language !== null) { + return UNSPACED_LANGUAGES.has(phrase.language.split("-")[0].toLowerCase()) + } + return UNSPACED_SCRIPT.test(edge) +} + +/** + * Every phrase so far as one text: a space between phrases, none between two + * in zh, ja, yue or th. assistant-ui would put a space between its finals. + */ +export function joinPhrases(phrases: readonly TranscribedPhrase[]): string { + let joined = "" + let previous: TranscribedPhrase | null = null + for (const phrase of phrases) { + const text = phrase.text.trim() + if (!text) continue + if (previous !== null) { + const glued = + unspaced(previous, [...previous.text].at(-1) ?? "") && + unspaced(phrase, [...text][0]) + if (!glued) joined += " " + } + joined += text + previous = { ...phrase, text } + } + return joined +} diff --git a/surfsense_local/frontend/src/features/dictation/mic/download-ring.tsx b/surfsense_local/frontend/src/features/dictation/mic/download-ring.tsx new file mode 100644 index 0000000000..e46f6306bc --- /dev/null +++ b/surfsense_local/frontend/src/features/dictation/mic/download-ring.tsx @@ -0,0 +1,32 @@ +import { installView } from "@/features/models/local/chat/install-view" +import { isRunning, jobsFor } from "@/features/models/local/installs/job-state" +import { useInstalls } from "@/features/models/local/installs/use-installs" + +const RADIUS = 16 +const CIRCUMFERENCE = 2 * Math.PI * RADIUS + +/** How far a speech model's download got, around the mic while it runs. */ +export function DownloadRing() { + const download = jobsFor(useInstalls().filter(isRunning), "speech_to_text")[0] + const percent = download ? installView(download.event).percent : null + if (percent === null) return null + return ( + + ) +} diff --git a/surfsense_local/frontend/src/features/dictation/mic/mic-button.tsx b/surfsense_local/frontend/src/features/dictation/mic/mic-button.tsx new file mode 100644 index 0000000000..8f05d42174 --- /dev/null +++ b/surfsense_local/frontend/src/features/dictation/mic/mic-button.tsx @@ -0,0 +1,159 @@ +import { useRef, useState, type PointerEvent } from "react" +import { useAui } from "@assistant-ui/react" + +import { Button } from "@/components/ui/button" +import { Mic01Icon } from "@/components/ui/icons" +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@/components/ui/tooltip" +import { SpeechSetupPopover } from "@/features/speech-setup/speech-setup-popover" +import { intl } from "@/i18n/intl" +import { cn } from "@/lib/utils" + +import type { Dictation } from "../dictation-adapter" +import { isRecording, useDictationView } from "../use-dictation-view" +import { useNeedsSpeechModel } from "../use-transcription-status" +import { DownloadRing } from "./download-ring" + +// Handy's hold-or-toggle: a press held this long is push to talk. +const HOLD_MS = 300 + +/** + * Voice input, beside Send. A click toggles it, a hold talks until release, + * and Space or Enter on the focused mic toggles it; with no speech model yet + * it sets one up instead. + */ +export function MicButton({ + dictation, + disabled, + onOpenSpeechSettings, + className, +}: { + dictation: Dictation + /** Whenever the message box is, and once it is at its limit. */ + disabled: boolean + onOpenSpeechSettings?: () => void + className?: string +}) { + const aui = useAui() + const view = useDictationView(dictation) + const needsModel = useNeedsSpeechModel() + const [setUpOpen, setSetUpOpen] = useState(false) + // The pointer press under way: whether it started the recording, and when. + const press = useRef<{ startedIt: boolean; at: number } | null>(null) + const active = isRecording(view) + const start = () => dictation.start(aui.composer()) + const tooltip = intl.formatMessage({ + id: "dictation_mic_idle_tooltip", + defaultMessage: "Voice input stays on this computer", + }) + + // A recording is always stopped from the mic; an open set-up stays until it + // is closed, even once its model is chosen. + if ((needsModel && !active) || setUpOpen) { + return ( + + undefined)} + trigger={ + + + + + } + /> + } + /> + {tooltip} + + ) + } + + const pressed = (event: PointerEvent) => { + if (event.button !== 0) return + if (view.phase === "finishing") return + event.currentTarget.setPointerCapture?.(event.pointerId) + press.current = { startedIt: !active, at: performance.now() } + if (!active) start() + } + + const released = () => { + const held = press.current + press.current = null + if (!held) return + // A tap that started it keeps listening; a hold, or a tap while it listens, stops. + if (!held.startedIt || performance.now() - held.at >= HOLD_MS) { + dictation.stop() + } + } + + return ( + + event.preventDefault()} + onPointerDown={pressed} + onPointerUp={released} + onPointerCancel={released} + onClick={(event) => { + // Pointers act on down and up; this is Space or Enter on the mic. + if (event.detail !== 0) return + if (active) dictation.stop() + else start() + }} + onKeyDown={(event) => { + if (event.key === "Escape" && active) { + event.preventDefault() + dictation.cancel(aui.composer()) + } + }} + > + + + } + /> + {tooltip} + + ) +} diff --git a/surfsense_local/frontend/src/features/dictation/recording.ts b/surfsense_local/frontend/src/features/dictation/recording.ts new file mode 100644 index 0000000000..07d338d6f7 --- /dev/null +++ b/surfsense_local/frontend/src/features/dictation/recording.ts @@ -0,0 +1,359 @@ +import type { DictationAdapter, Unsubscribe } from "@assistant-ui/react" + +import { ApiError } from "@/lib/api" + +import { encodeWav } from "./capture/encode-wav" +import type { + Microphone, + MicrophoneFrame, + OpenMicrophone, +} from "./capture/open-microphone" +import { createPhraseCutter } from "./capture/phrase-cutter" +import { resampleForSpeech, SPEECH_SAMPLE_RATE } from "./capture/resample" +import { clampToRoom } from "./clamp-to-room" +import { + microphoneProblem, + openingProblem, + type DictationProblem, +} from "./dictation-problem" +import { + isDictationEnded, + openDictationSession, + type DictationSession, +} from "./dictation-session" +import type { PhraseText } from "./api" +import { joinPhrases, type TranscribedPhrase } from "./join-phrases" + +const WARN_AT_MS = 270_000 +const STOP_AT_MS = 300_000 +// Handy's: a warm load takes under 2 s, so only a slower one is worth a word. +const LOADING_NOTICE_MS = 2_000 + +/** What a recording shows while it lasts. */ +export type RecordingView = + | { phase: "arming" } + | { phase: "listening"; since: number; warning: boolean; loading: boolean } + | { phase: "finishing"; loading: boolean } + +export type RecordingOutcome = { + problem: DictationProblem | null + /** Audio not yet transcribed when a phrase failed, for Retry. */ + kept: Blob[] +} + +type RecordingOptions = { + /** Characters left under the message limit for the dictated text. */ + room: number + openMicrophone: OpenMicrophone + /** Audio a failed phrase left: transcribed again, with no microphone. */ + kept?: Blob[] + onView: (view: RecordingView) => void + onLevel: (level: number) => void + /** Once, however it ends. */ + onEnd: (outcome: RecordingOutcome) => void +} + +/** + * One recording, from the mic press to its final text: assistant-ui's + * dictation session. Phrases go up one at a time as the speaker pauses; each + * answer re-emits the whole text so far, and stop emits it once as final. + */ +export class Recording implements DictationAdapter.Session { + status: DictationAdapter.Status = { type: "starting" } + + private readonly options: RecordingOptions + + private readonly speech = new Set<(result: DictationAdapter.Result) => void>() + private readonly starts = new Set<() => void>() + private readonly ends = new Set<(result: DictationAdapter.Result) => void>() + private readonly cutter = createPhraseCutter() + private session: Promise = Promise.resolve(null) + private opened: DictationSession | null = null + private mic: Microphone | null = null + private chain: Promise = Promise.resolve() + private phrases: TranscribedPhrase[] = [] + private text = "" + private pending = 0 + private rate = SPEECH_SAMPLE_RATE + private since: number | null = null + private warning = false + private modelLoading = false + private loadingNoticed = false + private loadingTimer: ReturnType | undefined + private timers: ReturnType[] = [] + // False once no more phrases can be sent; `kept` then holds them, if any. + private sending = true + private kept: Blob[] | null = null + private problem: DictationProblem | null = null + private wrapping = false + private over = false + private settle = () => {} + private readonly done = new Promise((resolve) => { + this.settle = resolve + }) + + constructor(options: RecordingOptions) { + this.options = options + } + + onSpeech(callback: (result: DictationAdapter.Result) => void): Unsubscribe { + this.speech.add(callback) + return () => void this.speech.delete(callback) + } + + onSpeechStart(callback: () => void): Unsubscribe { + this.starts.add(callback) + return () => void this.starts.delete(callback) + } + + onSpeechEnd( + callback: (result: DictationAdapter.Result) => void + ): Unsubscribe { + this.ends.add(callback) + return () => void this.ends.delete(callback) + } + + /** Opens the microphone and the API's session side by side. */ + begin(): void { + if (this.options.room <= 0) { + this.sending = false + void this.wrapUp({ kind: "at_limit" }) + return + } + this.session = openDictationSession({ + onEnded: () => this.stopSending({ kind: "dictation_ended" }), + }).then( + (session) => { + if (this.over) { + session.end() + return null + } + this.opened = session + if (session.warm === "loading") this.noteLoading() + return session + }, + (error: unknown) => { + this.refusedToOpen(openingProblem(error)) + return null + } + ) + const retrying = this.options.kept + if (retrying) { + for (const wav of retrying) this.enqueue(async () => wav) + void this.wrapUp(null) + return + } + this.report() + this.options + .openMicrophone({ + onFrame: (frame) => this.hear(frame), + onEnded: () => void this.wrapUp({ kind: "microphone_disconnected" }), + }) + .then( + (mic) => { + if (this.wrapping) mic.close() + else this.mic = mic + }, + (error: unknown) => void this.wrapUp(microphoneProblem(error)) + ) + } + + /** Flushes the tail, waits for every phrase, emits the final, then ends. */ + stop(): Promise { + return this.wrapUp(null) + } + + /** Drops what is not transcribed yet; nothing lands after this. */ + cancel(): void { + if (this.over) return + this.over = true + this.release() + this.status = { type: "ended", reason: "cancelled" } + for (const callback of [...this.ends]) callback({ transcript: "" }) + this.options.onEnd({ problem: null, kept: [] }) + this.settle() + } + + private hear(frame: MicrophoneFrame) { + if (this.wrapping || this.over) return + this.rate = frame.sampleRate + if (this.since === null) { + // The first real frame: the device is delivering, so the timer starts. + this.since = Date.now() + this.status = { type: "running" } + for (const callback of [...this.starts]) callback() + this.timers.push( + setTimeout(() => { + this.warning = true + this.report() + }, WARN_AT_MS), + setTimeout(() => void this.wrapUp({ kind: "time_limit" }), STOP_AT_MS) + ) + this.report() + } + this.options.onLevel(frame.level) + const phrase = this.cutter.push(frame.samples, frame.level) + if (phrase) this.enqueuePhrase(phrase) + } + + private enqueuePhrase(samples: Float32Array) { + const rate = this.rate + this.enqueue(async () => + encodeWav(await resampleForSpeech(samples, rate), SPEECH_SAMPLE_RATE) + ) + } + + private enqueue(wavOf: () => Promise) { + this.pending += 1 + this.report() + this.chain = this.chain.then(() => this.transcribe(wavOf)) + } + + private async transcribe(wavOf: () => Promise) { + try { + if (this.over) return + const wav = await wavOf() + if (!this.sending) { + this.kept?.push(wav) + return + } + const session = await this.session + if (session === null || this.over) { + this.kept?.push(wav) + return + } + try { + const answer = await session.transcribe(wav) + if (!this.over) this.accept(answer) + } catch (error) { + if (!this.over) this.refused(error, wav) + } + } catch { + // Audio that could not be encoded is dropped, never the recording. + } finally { + this.pending -= 1 + this.report() + } + } + + private accept(answer: PhraseText) { + this.modelLoading = false + if (!answer.no_speech && answer.text.trim()) { + this.phrases.push({ text: answer.text, language: answer.language }) + } + const joined = joinPhrases(this.phrases) + const text = clampToRoom(joined, this.options.room) + if (text !== this.text) { + this.text = text + for (const callback of [...this.speech]) { + callback({ transcript: text, isFinal: false }) + } + } + if (text.length < joined.length) this.stopSending({ kind: "at_limit" }) + } + + private refused(error: unknown, wav: Blob) { + this.modelLoading = false + if (isDictationEnded(error)) { + this.stopSending({ kind: "dictation_ended" }) + } else if ( + error instanceof ApiError && + error.code === "needs_speech_model" + ) { + this.stopSending({ kind: "model_removed" }) + } else { + // This phrase and every one after it wait for Retry. + this.kept = [wav] + this.stopSending({ kind: "phrase_failed" }) + } + } + + private refusedToOpen(problem: DictationProblem) { + if (this.options.kept) { + // A retry keeps its audio whatever stopped it, so Retry stays offered. + this.kept = [] + this.stopSending({ kind: "phrase_failed" }) + } else { + this.stopSending(problem) + } + } + + private stopSending(problem: DictationProblem) { + this.sending = false + void this.wrapUp(problem) + } + + private wrapUp(problem: DictationProblem | null): Promise { + if (this.over) return this.done + if (problem && !this.problem) this.problem = problem + if (this.wrapping) return this.done + this.wrapping = true + this.closeMic() + this.timers.forEach(clearTimeout) + if (this.sending || this.kept) { + const tail = this.cutter.flush() + if (tail) this.enqueuePhrase(tail) + } + this.report() + // Queued behind every phrase, never awaited from inside one. + this.chain = this.chain.then(() => this.finish()) + return this.done + } + + private finish() { + if (this.over) return + this.over = true + this.release() + // The runtime drops anything emitted once the session has ended. + for (const callback of [...this.speech]) { + callback({ transcript: this.text, isFinal: true }) + } + const notice = + this.problem === null || + this.problem.kind === "at_limit" || + this.problem.kind === "time_limit" + this.status = { type: "ended", reason: notice ? "stopped" : "error" } + for (const callback of [...this.ends]) { + callback({ transcript: this.text, isFinal: true }) + } + this.options.onEnd({ problem: this.problem, kept: this.kept ?? [] }) + this.settle() + } + + private noteLoading() { + this.modelLoading = true + this.loadingTimer = setTimeout(() => { + this.loadingNoticed = true + this.report() + }, LOADING_NOTICE_MS) + } + + private closeMic() { + this.mic?.close() + this.mic = null + } + + private release() { + this.closeMic() + this.timers.forEach(clearTimeout) + clearTimeout(this.loadingTimer) + this.opened?.end() + } + + private report() { + if (this.over) return + const loading = this.modelLoading && this.loadingNoticed && this.pending > 0 + if (this.wrapping || this.options.kept) { + this.options.onView({ phase: "finishing", loading }) + } else if (this.since === null) { + this.options.onView({ phase: "arming" }) + } else { + this.options.onView({ + phase: "listening", + since: this.since, + warning: this.warning, + loading, + }) + } + } +} diff --git a/surfsense_local/frontend/src/features/dictation/strip/dictation-strip.tsx b/surfsense_local/frontend/src/features/dictation/strip/dictation-strip.tsx new file mode 100644 index 0000000000..7939bd9804 --- /dev/null +++ b/surfsense_local/frontend/src/features/dictation/strip/dictation-strip.tsx @@ -0,0 +1,143 @@ +import { useAui } from "@assistant-ui/react" + +import { Button } from "@/components/ui/button" +import { XIcon } from "@/components/ui/icons" +import { Spinner } from "@/components/ui/spinner" +import { intl } from "@/i18n/intl" +import { cn } from "@/lib/utils" + +import type { Dictation, DictationView } from "../dictation-adapter" +import { useDictationView } from "../use-dictation-view" +import { LevelBars } from "./level-bars" +import { ProblemLine } from "./problem-line" +import { RecordingTimer } from "./recording-timer" + +type Status = { text: string; shown: boolean } + +function statusOf(view: DictationView): Status | null { + if (view.phase === "arming") { + return { + text: intl.formatMessage({ + id: "dictation_strip_arming_status", + defaultMessage: "Starting the microphone…", + }), + shown: false, + } + } + if (view.phase !== "listening" && view.phase !== "finishing") return null + if (view.loading) { + return { + text: intl.formatMessage({ + id: "dictation_strip_loading_status", + defaultMessage: "Loading the speech model…", + }), + shown: true, + } + } + if (view.phase === "finishing") { + return { + text: intl.formatMessage({ + id: "dictation_strip_transcribing_status", + defaultMessage: "Transcribing…", + }), + shown: true, + } + } + if (view.warning) { + return { + text: intl.formatMessage( + { + id: "dictation_strip_time_warning_status", + defaultMessage: + "Voice input stops in {seconds, plural, one {# second} other {# seconds}}.", + }, + { seconds: 30 } + ), + shown: true, + } + } + return { + text: intl.formatMessage({ + id: "dictation_strip_listening_status", + defaultMessage: "Listening", + }), + shown: false, + } +} + +/** + * Inside the composer while voice input runs, in place of the source count + * and model (center) or the disclaimer (bottom): never above the composer, + * where the model notice and the scroll button sit. + */ +export function DictationStrip({ + dictation, + onOpenSpeechSettings, + className, +}: { + dictation: Dictation + onOpenSpeechSettings?: () => void + className?: string +}) { + const aui = useAui() + const view = useDictationView(dictation) + + if (view.phase === "idle") return null + if (view.phase === "problem") { + return ( + + ) + } + + const status = statusOf(view) + return ( +
+ {view.phase === "finishing" ? ( +
+ ) +} diff --git a/surfsense_local/frontend/src/features/dictation/strip/level-bars.tsx b/surfsense_local/frontend/src/features/dictation/strip/level-bars.tsx new file mode 100644 index 0000000000..b6f358604c --- /dev/null +++ b/surfsense_local/frontend/src/features/dictation/strip/level-bars.tsx @@ -0,0 +1,62 @@ +import { useEffect, useState } from "react" + +import { cn } from "@/lib/utils" + +import type { Dictation } from "../dictation-adapter" + +const BARS = 9 +// RMS from the room's hush to loud speech; Handy's 0.7 power keeps soft speech visible. +const QUIET_DB = -60 +const LOUD_DB = -20 +const CURVE = 0.7 +const ARMING_HEIGHTS = [4, 6, 8, 10, 12, 10, 8, 6, 4] + +function barHeight(level: number): number { + const db = 20 * Math.log10(Math.max(level, 1e-6)) + const share = Math.min(1, Math.max(0, (db - QUIET_DB) / (LOUD_DB - QUIET_DB))) + return 3 + 13 * share ** CURVE +} + +/** + * The last 9 frames' levels, newest on the right. Muted and pulsing while the + * device starts; hidden under reduced motion, where the dot beside it stays still. + */ +export function LevelBars({ + dictation, + arming, +}: { + dictation: Dictation + arming: boolean +}) { + const [levels, setLevels] = useState(() => + new Array(BARS).fill(0) + ) + + useEffect( + () => + dictation.subscribeLevel(() => + setLevels((previous) => [...previous.slice(1), dictation.getLevel()]) + ), + [dictation] + ) + + return ( +