diff --git a/docs/design.md b/docs/design.md index e21270c85d..947c69ead6 100644 --- a/docs/design.md +++ b/docs/design.md @@ -70,7 +70,7 @@ the sidebar. Apply it as `backgroundColor`; setting `opacity` on the control als Sizes: `xs` for ultra-tight inline triggers. `sm` for any button sitting in a row. `md` is the page default. `lg` is reserved for large standalone CTAs. -Sizes are a shared contract across control kinds, defined once in `control-geometry.ts`: `xs` = 28px tall with `fontSize.sm` labels, `sm` = 32px with `fontSize.base`, `md`/`lg` = 44px with `fontSize.base`. `` (`packages/app/src/components/ui/segmented-control.tsx`) takes the same `xs`/`sm`/`md` sizes — a segmented control next to a ` diff --git a/packages/app/src/components/settings/row-density.browser.test.tsx b/packages/app/src/components/settings/row-density.browser.test.tsx new file mode 100644 index 0000000000..d2742e3172 --- /dev/null +++ b/packages/app/src/components/settings/row-density.browser.test.tsx @@ -0,0 +1,77 @@ +import React, { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, describe, expect, it } from "vitest"; +import { Button } from "@/components/ui/button"; +import { SettingsCard, SettingsCollapsibleRow, SettingsRow } from "@/components/settings"; + +let root: Root | null = null; +let container: HTMLDivElement | null = null; + +afterEach(() => { + act(() => root?.unmount()); + container?.remove(); + root = null; + container = null; +}); + +function mountRows() { + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); + act(() => + root?.render( + + + + + + + + + + + , + ), + ); +} + +function element(testID: string): HTMLElement { + const found = document.querySelector(`[data-testid="${testID}"]`); + if (!(found instanceof HTMLElement)) throw new Error(`Nothing rendered as ${testID}`); + return found; +} + +function heightOf(testID: string): number { + return element(testID).getBoundingClientRect().height; +} + +function titleFontSize(rowTestID: string, label: string): number { + const title = [...element(rowTestID).querySelectorAll("div")].find( + (node) => node.textContent === label && node.children.length === 0, + ); + if (!title) throw new Error(`No title ${label}`); + return Number.parseFloat(getComputedStyle(title).fontSize); +} + +describe("settings row density", () => { + it("lines up nested rows with and without a button, tighter than card rows", () => { + mountRows(); + + expect(heightOf("nested-with-button")).toBe(heightOf("nested-without-button")); + expect(heightOf("nested-with-button")).toBeLessThan(heightOf("card-row")); + expect(heightOf("group-toggle")).toBeGreaterThan(heightOf("nested-without-button")); + }); + + it("sizes nested text and controls one step below card rows", () => { + mountRows(); + + expect(heightOf("nested-button")).toBeLessThan(heightOf("card-button")); + expect(titleFontSize("nested-with-button", "Nested with a button")).toBeLessThan( + titleFontSize("card-row", "Card row"), + ); + }); +}); diff --git a/packages/app/src/components/settings/row-density.tsx b/packages/app/src/components/settings/row-density.tsx new file mode 100644 index 0000000000..d4bcdce33e --- /dev/null +++ b/packages/app/src/components/settings/row-density.tsx @@ -0,0 +1,57 @@ +import { createContext, useContext, useMemo, type ReactNode } from "react"; +import type { StyleProp, TextStyle, ViewStyle } from "react-native"; +import { StyleSheet } from "react-native-unistyles"; +import { CONTROL_HEIGHTS } from "@/components/ui/control-geometry"; +import type { ContainedControlSize } from "@/components/ui/control-size"; +import { settingsStyles } from "@/styles/settings"; + +// Rows revealed under a collapsible row belong to it, so they sit one step smaller than the rows +// of the card: tighter padding, smaller text, smaller controls. The container decides; a row +// never knows on its own that it is nested. +const NestedRowsContext = createContext(false); + +export function NestedSettingsRows({ children }: { children: ReactNode }) { + return {children}; +} + +export interface SettingsRowLayout { + row: StyleProp; + title: StyleProp; + hint: StyleProp; + /** The size of the controls in the row's trailing slot. */ + controlSize: ContainedControlSize; +} + +/** How a row lays out where it is rendered: in a card, or nested under a collapsible row. */ +export function useSettingsRowLayout(): SettingsRowLayout { + const nested = useContext(NestedRowsContext); + return useMemo( + () => + nested + ? { + row: [settingsStyles.row, styles.nestedRow], + title: [settingsStyles.rowTitle, styles.nestedText], + hint: [settingsStyles.rowHint, styles.nestedText], + controlSize: "xs", + } + : { + row: settingsStyles.row, + title: settingsStyles.rowTitle, + hint: settingsStyles.rowHint, + controlSize: "sm", + }, + [nested], + ); +} + +const styles = StyleSheet.create((theme) => ({ + // Every nested row is as tall as one holding an `xs` control, so rows with and without a + // trailing button line up. + nestedRow: { + paddingVertical: theme.spacing[2], + minHeight: CONTROL_HEIGHTS.tight + theme.spacing[2] * 2, + }, + nestedText: { + fontSize: theme.fontSize.sm, + }, +})); diff --git a/packages/app/src/components/ui/button.tsx b/packages/app/src/components/ui/button.tsx index 707ce60648..abcaf240d6 100644 --- a/packages/app/src/components/ui/button.tsx +++ b/packages/app/src/components/ui/button.tsx @@ -1,4 +1,5 @@ import { LoadingSpinner } from "@/components/ui/loading-spinner"; +import { useContainedControlSize } from "@/components/ui/control-size"; import { default as React, useCallback, @@ -162,7 +163,7 @@ const styles = StyleSheet.create((theme) => { export function Button({ children, variant = "secondary", - size = "md", + size: sizeProp, leftIcon, trailing, style, @@ -185,6 +186,8 @@ export function Button({ } >) { const [hovered, setHovered] = useState(false); + const containedSize = useContainedControlSize(); + const size = sizeProp ?? containedSize ?? "md"; const isDisabled = disabled || loading; let variantStyle: ViewStyle; diff --git a/packages/app/src/components/ui/control-geometry.test.ts b/packages/app/src/components/ui/control-geometry.test.ts index 0b0675596f..3544d9f940 100644 --- a/packages/app/src/components/ui/control-geometry.test.ts +++ b/packages/app/src/components/ui/control-geometry.test.ts @@ -81,6 +81,8 @@ describe("control geometry", () => { it("keeps field text sizing tied to control size", () => { const geometry = createControlGeometry(theme); + expect(geometry.fieldTextXs.fontSize).toBe(12); + expect(geometry.fieldTextXs.lineHeight).toBe(17); expect(geometry.fieldTextSm.fontSize).toBe(14); expect(geometry.fieldTextSm.lineHeight).toBe(20); expect(geometry.fieldTextMd.fontSize).toBe(14); @@ -95,6 +97,13 @@ describe("control geometry", () => { const geometry = createControlGeometry(theme); const borderWidth = theme.borderWidth[1]; + expect(geometry.fieldControlXs.minHeight).toBe(28); + expect( + geometry.fieldTextXs.lineHeight + + geometry.fieldControlXs.paddingVertical * 2 + + borderWidth * 2, + ).toBe(geometry.fieldControlXs.minHeight); + expect(geometry.fieldControlSm.minHeight).toBe(32); expect(geometry.fieldControlSm.paddingVertical).toBe(5); expect( diff --git a/packages/app/src/components/ui/control-geometry.ts b/packages/app/src/components/ui/control-geometry.ts index d56200887e..0a41f80039 100644 --- a/packages/app/src/components/ui/control-geometry.ts +++ b/packages/app/src/components/ui/control-geometry.ts @@ -2,7 +2,7 @@ import type { StyleProp, ViewStyle } from "react-native"; import { ICON_SIZE, type Theme } from "@/styles/theme"; export type ButtonControlSize = "xs" | "sm" | "md" | "lg"; -export type FieldControlSize = "sm" | "md"; +export type FieldControlSize = "xs" | "sm" | "md"; export type SegmentedControlSize = "xs" | "sm" | "md"; export type ControlInteractionPhase = "rest" | "hover" | "active"; @@ -188,8 +188,19 @@ export function resolveControlInteractionStyles( export function createControlGeometry(theme: Theme) { const controlBorderWidth = theme.borderWidth[1]; + const fieldTextXsLineHeight = fieldLineHeight(theme.fontSize.sm); const fieldTextSmLineHeight = fieldLineHeight(theme.fontSize.base); const fieldTextMdLineHeight = fieldLineHeight(theme.fontSize.base); + const fieldControlXs = { + minHeight: CONTROL_HEIGHTS.tight, + paddingHorizontal: theme.spacing[2], + paddingVertical: fieldVerticalPadding( + CONTROL_HEIGHTS.tight, + fieldTextXsLineHeight, + controlBorderWidth, + ), + borderRadius: theme.borderRadius.md, + }; const fieldControlSm = { minHeight: CONTROL_HEIGHTS.compact, paddingHorizontal: theme.spacing[3], @@ -210,6 +221,10 @@ export function createControlGeometry(theme: Theme) { ), borderRadius: theme.borderRadius.lg, }; + const fieldTextXs = { + fontSize: theme.fontSize.sm, + lineHeight: fieldTextXsLineHeight, + }; const fieldTextSm = { fontSize: theme.fontSize.base, lineHeight: fieldTextSmLineHeight, @@ -252,6 +267,10 @@ export function createControlGeometry(theme: Theme) { fontSize: theme.fontSize.sm, }, alert, + formTextInputXs: { + ...fieldControlXs, + ...fieldTextXs, + }, formTextInputSm: { ...fieldControlSm, ...fieldTextSm, @@ -264,8 +283,10 @@ export function createControlGeometry(theme: Theme) { ...fieldControlMd, ...fieldTextMd, }, + fieldControlXs, fieldControlSm, fieldControlMd, + fieldTextXs, fieldTextSm, fieldTextMd, controlRest: { diff --git a/packages/app/src/components/ui/control-size.tsx b/packages/app/src/components/ui/control-size.tsx new file mode 100644 index 0000000000..c6bcbbea78 --- /dev/null +++ b/packages/app/src/components/ui/control-size.tsx @@ -0,0 +1,23 @@ +import { createContext, useContext, type ReactNode } from "react"; + +/** The control sizes a container can ask of the controls placed in it. */ +export type ContainedControlSize = "xs" | "sm"; + +// A row decides how big its controls are, so a button or field placed in it needs no size of +// its own. A size passed to the control still wins. +const ControlSizeContext = createContext(null); + +export function ControlSizeProvider({ + size, + children, +}: { + size: ContainedControlSize; + children: ReactNode; +}) { + return {children}; +} + +/** The size the surrounding container asks for, or null outside one. */ +export function useContainedControlSize(): ContainedControlSize | null { + return useContext(ControlSizeContext); +} diff --git a/packages/app/src/components/ui/dropdown-trigger.tsx b/packages/app/src/components/ui/dropdown-trigger.tsx index 2fc1eb26ac..a276e40652 100644 --- a/packages/app/src/components/ui/dropdown-trigger.tsx +++ b/packages/app/src/components/ui/dropdown-trigger.tsx @@ -9,6 +9,7 @@ import { } from "@/components/ui/control-geometry"; import type { Theme } from "@/styles/theme"; import { DropdownMenuTrigger, type DropdownMenuTriggerProps } from "@/components/ui/dropdown-menu"; +import { useContainedControlSize } from "@/components/ui/control-size"; const ThemedChevronDown = withUnistyles(ChevronDown); @@ -32,10 +33,12 @@ interface DropdownTriggerProps extends Omit; +const FIELD_TEXT = { xs: "inputXs", sm: "inputSm", md: "inputMd" } as const satisfies Record< + FieldControlSize, + string +>; interface FieldProps { label: string; @@ -159,11 +169,13 @@ function assignTextInputRef( } export const FormTextInput = forwardRef( - function FormTextInput({ size = "md", style, onFocus, onBlur, editable, ...props }, ref) { + function FormTextInput({ size: sizeProp, style, onFocus, onBlur, editable, ...props }, ref) { const [focused, setFocused] = useState(false); + const containedSize = useContainedControlSize(); + const size = sizeProp ?? containedSize ?? "md"; const isDisabled = editable === false; - const chromeSizeStyle = size === "sm" ? formInputStyles.chromeSm : formInputStyles.chromeMd; - const inputSizeStyle = size === "sm" ? formInputStyles.inputSm : formInputStyles.inputMd; + const chromeSizeStyle = formInputStyles[FIELD_CHROME[size]]; + const inputSizeStyle = formInputStyles[FIELD_TEXT[size]]; const splitStyle = useMemo(() => splitFormTextInputStyle(style), [style]); const setInputRef = useCallback( (node: EditingTextInputHandle | null) => { @@ -262,6 +274,9 @@ const formInputStyles = StyleSheet.create((theme) => { chrome: { backgroundColor: theme.colors.surface2, }, + chromeXs: { + ...geometry.fieldControlXs, + }, chromeSm: { ...geometry.fieldControlSm, }, @@ -289,6 +304,9 @@ const formInputStyles = StyleSheet.create((theme) => { outlineColor: "transparent", outlineWidth: 0, }, + inputXs: { + ...geometry.fieldTextXs, + }, inputSm: { ...geometry.fieldTextSm, }, diff --git a/packages/app/src/i18n/resources/ar.ts b/packages/app/src/i18n/resources/ar.ts index 4adf945b61..627096b2a0 100644 --- a/packages/app/src/i18n/resources/ar.ts +++ b/packages/app/src/i18n/resources/ar.ts @@ -2106,6 +2106,7 @@ export const ar: TranslationResources = { connections: "اتصالات", agents: "Agents", metadata: "البيانات الوصفية", + voice: "الصوت", workspaces: "Workspaces", providers: "مقدمي الخدمات", usage: "الاستخدام", @@ -2128,6 +2129,75 @@ export const ar: TranslationResources = { docs: "الوثائق", saveError: "تعذر تحديث إنشاء البيانات الوصفية", }, + voice: { + offline: "اتصل بهذا المضيف لضبط الصوت.", + updateHost: "حدّث المضيف لضبط الصوت.", + saveError: "تعذّر تحديث إعدادات الصوت", + provider: "المزوّد", + providers: { + local: "محلي", + openai: "OpenAI", + }, + sttModel: "نموذج تحويل الكلام إلى نص", + ttsModel: "نموذج الصوت", + voice: "الصوت", + notDownloadedOption: "{{label}} (غير منزّل)", + save: "حفظ", + retry: "إعادة المحاولة", + dictation: { + title: "الإملاء", + enabled: "الإملاء", + enabledHint: "تحدّث في مربع الرسالة بدلًا من الكتابة", + }, + voiceMode: { + title: "وضع الصوت", + info: "يحوّل وضع الصوت كلامك إلى نص ويقرأ ردود الوكيل بصوت عالٍ.", + enabled: "وضع الصوت", + enabledHint: "تحدّث مع وكلائك دون استخدام يديك", + sttProvider: "تحويل الكلام إلى نص", + ttsProvider: "تحويل النص إلى كلام", + }, + openai: { + apiKey: "مفتاح API", + apiKeyHint: "يُحفظ على المضيف ولا يُعرض مرة أخرى.", + keyPlaceholder: "sk-...", + replaceKeyPlaceholder: "مفتاح جديد", + removeKey: "إزالة", + removeKeyTitle: "إزالة مفتاح OpenAI؟", + removeKeyMessage: "ستتوقف الميزات المضبوطة على OpenAI عن العمل حتى تضيف مفتاحًا.", + baseUrl: "عنوان URL الأساسي", + baseUrlHint: "للخوادم المتوافقة مع OpenAI. اتركه فارغًا لاستخدام OpenAI.", + missingKeyWarning: "تم اختيار OpenAI لكن لم يُضبط مفتاح API.", + }, + models: { + loading: "جارٍ تحميل النماذج...", + loadError: "تعذّر تحميل النماذج المحلية", + downloaded: "منزّل", + inUse: "قيد الاستخدام", + recommended: "موصى به", + download: "تنزيل", + downloading: "جارٍ التنزيل...", + downloadingPercent: "جارٍ التنزيل {{percent}}%", + extracting: "جارٍ الاستخراج...", + delete: "حذف", + deleteAction: "حذف {{label}}", + deleteTitle: "حذف {{label}}؟", + deleteMessage: "سيؤدي هذا إلى تحرير {{size}}. يمكنك تنزيله مرة أخرى لاحقًا.", + languages: "لغات {{label}}", + moreLanguages: "{{languages}} و{{count}} أخرى", + }, + providerSettings: { + title: "المزوّدون", + info: "النماذج وبيانات الاعتماد لكل مزوّد. تعمل النماذج المحلية على هذا المضيف وتعرض حجمها ولغاتها قبل التنزيل.", + localStt: "تحويل الكلام إلى نص محليًا", + localTts: "تحويل النص إلى كلام محليًا", + openai: "OpenAI", + downloaded_one: "تم تنزيل {{count}} نموذج", + downloaded_other: "تم تنزيل {{count}} نماذج", + configured: "تم الإعداد", + notConfigured: "غير مُعد", + }, + }, general: { title: "عام", sending: "الإرسال", diff --git a/packages/app/src/i18n/resources/en.ts b/packages/app/src/i18n/resources/en.ts index 9bb0460036..ac97e1c2f5 100644 --- a/packages/app/src/i18n/resources/en.ts +++ b/packages/app/src/i18n/resources/en.ts @@ -2169,6 +2169,7 @@ export const en = { connections: "Connections", agents: "Agents", metadata: "Metadata", + voice: "Voice", workspaces: "Workspaces", providers: "Providers", usage: "Usage", @@ -2253,6 +2254,75 @@ export const en = { docs: "Docs", saveError: "Unable to update metadata generation", }, + voice: { + offline: "Connect to this host to configure voice.", + updateHost: "Update the host to configure voice.", + saveError: "Unable to update voice settings", + provider: "Provider", + providers: { + local: "Local", + openai: "OpenAI", + }, + sttModel: "Speech-to-text model", + ttsModel: "Voice model", + voice: "Voice", + notDownloadedOption: "{{label}} (not downloaded)", + save: "Save", + retry: "Retry", + dictation: { + title: "Dictation", + enabled: "Dictation", + enabledHint: "Speak into the message box instead of typing", + }, + voiceMode: { + title: "Voice mode", + info: "Voice mode transcribes what you say and reads agent replies aloud.", + enabled: "Voice mode", + enabledHint: "Talk with your agents hands-free", + sttProvider: "Speech-to-text", + ttsProvider: "Text-to-speech", + }, + openai: { + apiKey: "API key", + apiKeyHint: "Saved on the host and never shown again.", + keyPlaceholder: "sk-...", + replaceKeyPlaceholder: "New key", + removeKey: "Remove", + removeKeyTitle: "Remove the OpenAI API key?", + removeKeyMessage: "Features set to OpenAI stop working until you add a key.", + baseUrl: "Base URL", + baseUrlHint: "For OpenAI-compatible servers. Leave empty to use OpenAI.", + missingKeyWarning: "OpenAI is selected but no API key is set.", + }, + models: { + loading: "Loading models...", + loadError: "Couldn't load local models", + downloaded: "Downloaded", + inUse: "In use", + recommended: "Recommended", + download: "Download", + downloading: "Downloading...", + downloadingPercent: "Downloading {{percent}}%", + extracting: "Extracting...", + delete: "Delete", + deleteAction: "Delete {{label}}", + deleteTitle: "Delete {{label}}?", + deleteMessage: "This frees {{size}}. You can download it again later.", + languages: "{{label}} languages", + moreLanguages: "{{languages}} and {{count}} more", + }, + providerSettings: { + title: "Providers", + info: "Models and credentials for each provider. Local models run on this host and list their size and languages before you download.", + localStt: "Local speech-to-text", + localTts: "Local text-to-speech", + openai: "OpenAI", + downloaded_one: "{{count}} model downloaded", + downloaded_other: "{{count}} models downloaded", + configured: "Configured", + notConfigured: "Not configured", + }, + }, general: { title: "General", sending: "Sending", diff --git a/packages/app/src/i18n/resources/es.ts b/packages/app/src/i18n/resources/es.ts index 9ed1e85572..e3e824b296 100644 --- a/packages/app/src/i18n/resources/es.ts +++ b/packages/app/src/i18n/resources/es.ts @@ -2154,6 +2154,7 @@ export const es: TranslationResources = { connections: "Conexiones", agents: "Agents", metadata: "Metadatos", + voice: "Voz", workspaces: "Workspaces", providers: "Proveedores", usage: "Uso", @@ -2176,6 +2177,77 @@ export const es: TranslationResources = { docs: "Documentación", saveError: "No se pudo actualizar la generación de metadatos", }, + voice: { + offline: "Conéctate a este host para configurar la voz.", + updateHost: "Actualiza el host para configurar la voz.", + saveError: "No se pudo actualizar la configuración de voz", + provider: "Proveedor", + providers: { + local: "Local", + openai: "OpenAI", + }, + sttModel: "Modelo de voz a texto", + ttsModel: "Modelo de voz", + voice: "Voz", + notDownloadedOption: "{{label}} (sin descargar)", + save: "Guardar", + retry: "Reintentar", + dictation: { + title: "Dictado", + enabled: "Dictado", + enabledHint: "Habla en el cuadro de mensaje en lugar de escribir", + }, + voiceMode: { + title: "Modo de voz", + info: "El modo de voz transcribe lo que dices y lee en voz alta las respuestas del agente.", + enabled: "Modo de voz", + enabledHint: "Habla con tus agentes sin usar las manos", + sttProvider: "Voz a texto", + ttsProvider: "Texto a voz", + }, + openai: { + apiKey: "Clave de API", + apiKeyHint: "Se guarda en el host y no se vuelve a mostrar.", + keyPlaceholder: "sk-...", + replaceKeyPlaceholder: "Nueva clave", + removeKey: "Quitar", + removeKeyTitle: "¿Quitar la clave de API de OpenAI?", + removeKeyMessage: + "Las funciones configuradas con OpenAI dejarán de funcionar hasta que añadas una clave.", + baseUrl: "URL base", + baseUrlHint: "Para servidores compatibles con OpenAI. Déjala vacía para usar OpenAI.", + missingKeyWarning: + "OpenAI está seleccionado, pero no hay ninguna clave de API configurada.", + }, + models: { + loading: "Cargando modelos...", + loadError: "No se pudieron cargar los modelos locales", + downloaded: "Descargado", + inUse: "En uso", + recommended: "Recomendado", + download: "Descargar", + downloading: "Descargando...", + downloadingPercent: "Descargando {{percent}} %", + extracting: "Extrayendo...", + delete: "Eliminar", + deleteAction: "Eliminar {{label}}", + deleteTitle: "¿Eliminar {{label}}?", + deleteMessage: "Esto libera {{size}}. Puedes volver a descargarlo más tarde.", + languages: "Idiomas de {{label}}", + moreLanguages: "{{languages}} y {{count}} más", + }, + providerSettings: { + title: "Proveedores", + info: "Modelos y credenciales de cada proveedor. Los modelos locales se ejecutan en este host e indican su tamaño e idiomas antes de descargarlos.", + localStt: "Voz a texto local", + localTts: "Texto a voz local", + openai: "OpenAI", + downloaded_one: "{{count}} modelo descargado", + downloaded_other: "{{count}} modelos descargados", + configured: "Configurado", + notConfigured: "Sin configurar", + }, + }, general: { title: "General", sending: "Envío", diff --git a/packages/app/src/i18n/resources/fr.ts b/packages/app/src/i18n/resources/fr.ts index 6563b1100e..5f1465ddbb 100644 --- a/packages/app/src/i18n/resources/fr.ts +++ b/packages/app/src/i18n/resources/fr.ts @@ -2153,6 +2153,7 @@ export const fr: TranslationResources = { connections: "Connexions", agents: "Agents", metadata: "Métadonnées", + voice: "Voix", workspaces: "Espaces de travail", providers: "Fournisseurs", usage: "Utilisation", @@ -2175,6 +2176,76 @@ export const fr: TranslationResources = { docs: "Documentation", saveError: "Impossible de mettre à jour la génération de métadonnées", }, + voice: { + offline: "Connectez-vous à cet hôte pour configurer la voix.", + updateHost: "Mettez à jour l'hôte pour configurer la voix.", + saveError: "Impossible de mettre à jour les réglages vocaux", + provider: "Fournisseur", + providers: { + local: "Local", + openai: "OpenAI", + }, + sttModel: "Modèle de transcription", + ttsModel: "Modèle de voix", + voice: "Voix", + notDownloadedOption: "{{label}} (non téléchargé)", + save: "Enregistrer", + retry: "Réessayer", + dictation: { + title: "Dictée", + enabled: "Dictée", + enabledHint: "Parlez dans la zone de message au lieu de taper", + }, + voiceMode: { + title: "Mode vocal", + info: "Le mode vocal transcrit ce que vous dites et lit à voix haute les réponses de l'agent.", + enabled: "Mode vocal", + enabledHint: "Parlez à vos agents sans les mains", + sttProvider: "Transcription", + ttsProvider: "Synthèse vocale", + }, + openai: { + apiKey: "Clé d'API", + apiKeyHint: "Enregistrée sur l'hôte, jamais réaffichée.", + keyPlaceholder: "sk-...", + replaceKeyPlaceholder: "Nouvelle clé", + removeKey: "Supprimer", + removeKeyTitle: "Supprimer la clé d'API OpenAI ?", + removeKeyMessage: + "Les fonctions réglées sur OpenAI cesseront de fonctionner jusqu'à l'ajout d'une clé.", + baseUrl: "URL de base", + baseUrlHint: "Pour les serveurs compatibles OpenAI. Laissez vide pour utiliser OpenAI.", + missingKeyWarning: "OpenAI est sélectionné, mais aucune clé d'API n'est définie.", + }, + models: { + loading: "Chargement des modèles...", + loadError: "Impossible de charger les modèles locaux", + downloaded: "Téléchargé", + inUse: "Utilisé", + recommended: "Recommandé", + download: "Télécharger", + downloading: "Téléchargement...", + downloadingPercent: "Téléchargement {{percent}} %", + extracting: "Extraction...", + delete: "Supprimer", + deleteAction: "Supprimer {{label}}", + deleteTitle: "Supprimer {{label}} ?", + deleteMessage: "Cela libère {{size}}. Vous pourrez le retélécharger plus tard.", + languages: "Langues de {{label}}", + moreLanguages: "{{languages}} et {{count}} autres", + }, + providerSettings: { + title: "Fournisseurs", + info: "Modèles et identifiants de chaque fournisseur. Les modèles locaux s'exécutent sur cet hôte et indiquent leur taille et leurs langues avant le téléchargement.", + localStt: "Reconnaissance vocale locale", + localTts: "Synthèse vocale locale", + openai: "OpenAI", + downloaded_one: "{{count}} modèle téléchargé", + downloaded_other: "{{count}} modèles téléchargés", + configured: "Configuré", + notConfigured: "Non configuré", + }, + }, general: { title: "Général", sending: "Envoi", diff --git a/packages/app/src/i18n/resources/ja.ts b/packages/app/src/i18n/resources/ja.ts index eb22dd663a..53f77465ac 100644 --- a/packages/app/src/i18n/resources/ja.ts +++ b/packages/app/src/i18n/resources/ja.ts @@ -2127,6 +2127,7 @@ export const ja: TranslationResources = { connections: "接続", agents: "エージェント", metadata: "メタデータ", + voice: "音声", workspaces: "ワークスペース", providers: "プロバイダー", usage: "使用状況", @@ -2149,6 +2150,75 @@ export const ja: TranslationResources = { docs: "ドキュメント", saveError: "メタデータ生成を更新できません", }, + voice: { + offline: "音声を設定するには、このホストに接続してください。", + updateHost: "音声を設定するには、ホストを更新してください。", + saveError: "音声設定を更新できませんでした", + provider: "プロバイダー", + providers: { + local: "ローカル", + openai: "OpenAI", + }, + sttModel: "音声認識モデル", + ttsModel: "音声モデル", + voice: "ボイス", + notDownloadedOption: "{{label}}（未ダウンロード）", + save: "保存", + retry: "再試行", + dictation: { + title: "音声入力", + enabled: "音声入力", + enabledHint: "入力する代わりにメッセージ欄に話しかけます", + }, + voiceMode: { + title: "ボイスモード", + info: "ボイスモードは話した内容を文字起こしし、エージェントの返答を読み上げます。", + enabled: "ボイスモード", + enabledHint: "ハンズフリーでエージェントと会話します", + sttProvider: "音声認識", + ttsProvider: "音声合成", + }, + openai: { + apiKey: "API キー", + apiKeyHint: "ホストに保存され、再表示されません。", + keyPlaceholder: "sk-...", + replaceKeyPlaceholder: "新しいキー", + removeKey: "削除", + removeKeyTitle: "OpenAI API キーを削除しますか？", + removeKeyMessage: "キーを追加するまで、OpenAI に設定された機能は動作しません。", + baseUrl: "ベース URL", + baseUrlHint: "OpenAI 互換サーバー用です。OpenAI を使う場合は空欄のままにします。", + missingKeyWarning: "OpenAI が選択されていますが、API キーが設定されていません。", + }, + models: { + loading: "モデルを読み込み中...", + loadError: "ローカルモデルを読み込めませんでした", + downloaded: "ダウンロード済み", + inUse: "使用中", + recommended: "おすすめ", + download: "ダウンロード", + downloading: "ダウンロード中...", + downloadingPercent: "ダウンロード中 {{percent}}%", + extracting: "展開中...", + delete: "削除", + deleteAction: "{{label}} を削除", + deleteTitle: "{{label}} を削除しますか？", + deleteMessage: "{{size}} が解放されます。後で再ダウンロードできます。", + languages: "{{label}} の対応言語", + moreLanguages: "{{languages}} ほか {{count}} 言語", + }, + providerSettings: { + title: "プロバイダー", + info: "各プロバイダーのモデルと認証情報です。ローカルモデルはこのホストで動作し、ダウンロード前にサイズと対応言語を確認できます。", + localStt: "ローカル音声認識", + localTts: "ローカル音声合成", + openai: "OpenAI", + downloaded_one: "{{count}} 個のモデルをダウンロード済み", + downloaded_other: "{{count}} 個のモデルをダウンロード済み", + configured: "設定済み", + notConfigured: "未設定", + }, + }, general: { title: "一般", sending: "送信", diff --git a/packages/app/src/i18n/resources/ko.ts b/packages/app/src/i18n/resources/ko.ts index afdb8974e9..4841f5f6c0 100644 --- a/packages/app/src/i18n/resources/ko.ts +++ b/packages/app/src/i18n/resources/ko.ts @@ -2116,6 +2116,7 @@ export const ko: TranslationResources = { connections: "연결", agents: "에이전트", metadata: "메타데이터", + voice: "음성", workspaces: "워크스페이스", providers: "프로바이더", usage: "사용량", @@ -2138,6 +2139,75 @@ export const ko: TranslationResources = { docs: "문서", saveError: "메타데이터 생성을 업데이트할 수 없습니다", }, + voice: { + offline: "음성을 설정하려면 이 호스트에 연결하세요.", + updateHost: "음성을 설정하려면 호스트를 업데이트하세요.", + saveError: "음성 설정을 업데이트할 수 없습니다", + provider: "제공자", + providers: { + local: "로컬", + openai: "OpenAI", + }, + sttModel: "음성 인식 모델", + ttsModel: "음성 모델", + voice: "목소리", + notDownloadedOption: "{{label}} (다운로드 안 됨)", + save: "저장", + retry: "다시 시도", + dictation: { + title: "받아쓰기", + enabled: "받아쓰기", + enabledHint: "입력하는 대신 메시지 상자에 말하세요", + }, + voiceMode: { + title: "음성 모드", + info: "음성 모드는 말한 내용을 텍스트로 옮기고 에이전트의 답변을 소리 내어 읽어 줍니다.", + enabled: "음성 모드", + enabledHint: "손을 쓰지 않고 에이전트와 대화하세요", + sttProvider: "음성 인식", + ttsProvider: "음성 합성", + }, + openai: { + apiKey: "API 키", + apiKeyHint: "호스트에 저장되며 다시 표시되지 않습니다.", + keyPlaceholder: "sk-...", + replaceKeyPlaceholder: "새 키", + removeKey: "제거", + removeKeyTitle: "OpenAI API 키를 제거할까요?", + removeKeyMessage: "키를 추가할 때까지 OpenAI로 설정된 기능이 작동하지 않습니다.", + baseUrl: "기본 URL", + baseUrlHint: "OpenAI 호환 서버용입니다. OpenAI를 사용하려면 비워 두세요.", + missingKeyWarning: "OpenAI가 선택되었지만 API 키가 설정되지 않았습니다.", + }, + models: { + loading: "모델을 불러오는 중...", + loadError: "로컬 모델을 불러올 수 없습니다", + downloaded: "다운로드됨", + inUse: "사용 중", + recommended: "추천", + download: "다운로드", + downloading: "다운로드 중...", + downloadingPercent: "다운로드 중 {{percent}}%", + extracting: "압축 해제 중...", + delete: "삭제", + deleteAction: "{{label}} 삭제", + deleteTitle: "{{label}}을(를) 삭제할까요?", + deleteMessage: "{{size}}가 확보됩니다. 나중에 다시 다운로드할 수 있습니다.", + languages: "{{label}} 지원 언어", + moreLanguages: "{{languages}} 외 {{count}}개", + }, + providerSettings: { + title: "제공자", + info: "제공자별 모델과 자격 증명입니다. 로컬 모델은 이 호스트에서 실행되며 다운로드 전에 크기와 지원 언어를 보여줍니다.", + localStt: "로컬 음성 인식", + localTts: "로컬 음성 합성", + openai: "OpenAI", + downloaded_one: "모델 {{count}}개 다운로드됨", + downloaded_other: "모델 {{count}}개 다운로드됨", + configured: "설정됨", + notConfigured: "설정 안 됨", + }, + }, general: { title: "일반", sending: "전송", diff --git a/packages/app/src/i18n/resources/pt-BR.ts b/packages/app/src/i18n/resources/pt-BR.ts index f89fb97bf0..5e519d81ec 100644 --- a/packages/app/src/i18n/resources/pt-BR.ts +++ b/packages/app/src/i18n/resources/pt-BR.ts @@ -2139,6 +2139,7 @@ export const ptBR: TranslationResources = { connections: "Conexões", agents: "Agentes", metadata: "Metadados", + voice: "Voz", workspaces: "Workspaces", providers: "Provedores", usage: "Uso", @@ -2161,6 +2162,76 @@ export const ptBR: TranslationResources = { docs: "Documentação", saveError: "Não foi possível atualizar a geração de metadados", }, + voice: { + offline: "Conecte-se a este host para configurar a voz.", + updateHost: "Atualize o host para configurar a voz.", + saveError: "Não foi possível atualizar as configurações de voz", + provider: "Provedor", + providers: { + local: "Local", + openai: "OpenAI", + }, + sttModel: "Modelo de fala para texto", + ttsModel: "Modelo de voz", + voice: "Voz", + notDownloadedOption: "{{label}} (não baixado)", + save: "Salvar", + retry: "Tentar novamente", + dictation: { + title: "Ditado", + enabled: "Ditado", + enabledHint: "Fale na caixa de mensagem em vez de digitar", + }, + voiceMode: { + title: "Modo de voz", + info: "O modo de voz transcreve o que você diz e lê em voz alta as respostas do agente.", + enabled: "Modo de voz", + enabledHint: "Converse com seus agentes sem usar as mãos", + sttProvider: "Fala para texto", + ttsProvider: "Texto para fala", + }, + openai: { + apiKey: "Chave de API", + apiKeyHint: "Salva no host e nunca mostrada novamente.", + keyPlaceholder: "sk-...", + replaceKeyPlaceholder: "Nova chave", + removeKey: "Remover", + removeKeyTitle: "Remover a chave de API da OpenAI?", + removeKeyMessage: + "Os recursos configurados com a OpenAI param de funcionar até você adicionar uma chave.", + baseUrl: "URL base", + baseUrlHint: "Para servidores compatíveis com a OpenAI. Deixe vazio para usar a OpenAI.", + missingKeyWarning: "A OpenAI está selecionada, mas nenhuma chave de API foi definida.", + }, + models: { + loading: "Carregando modelos...", + loadError: "Não foi possível carregar os modelos locais", + downloaded: "Baixado", + inUse: "Em uso", + recommended: "Recomendado", + download: "Baixar", + downloading: "Baixando...", + downloadingPercent: "Baixando {{percent}}%", + extracting: "Extraindo...", + delete: "Excluir", + deleteAction: "Excluir {{label}}", + deleteTitle: "Excluir {{label}}?", + deleteMessage: "Isso libera {{size}}. Você pode baixá-lo de novo depois.", + languages: "Idiomas do {{label}}", + moreLanguages: "{{languages}} e mais {{count}}", + }, + providerSettings: { + title: "Provedores", + info: "Modelos e credenciais de cada provedor. Os modelos locais rodam neste host e mostram tamanho e idiomas antes do download.", + localStt: "Fala para texto local", + localTts: "Texto para fala local", + openai: "OpenAI", + downloaded_one: "{{count}} modelo baixado", + downloaded_other: "{{count}} modelos baixados", + configured: "Configurado", + notConfigured: "Não configurado", + }, + }, general: { title: "Geral", sending: "Envio", diff --git a/packages/app/src/i18n/resources/ru.ts b/packages/app/src/i18n/resources/ru.ts index a1a70822b4..81108c36e0 100644 --- a/packages/app/src/i18n/resources/ru.ts +++ b/packages/app/src/i18n/resources/ru.ts @@ -2139,6 +2139,7 @@ export const ru: TranslationResources = { connections: "Подключения", agents: "Агенты", metadata: "Метаданные", + voice: "Голос", workspaces: "Рабочие пространства", providers: "Провайдеры", usage: "Использование", @@ -2161,6 +2162,77 @@ export const ru: TranslationResources = { docs: "Документация", saveError: "Не удалось обновить настройки генерации метаданных", }, + voice: { + offline: "Подключитесь к этому хосту, чтобы настроить голос.", + updateHost: "Обновите хост, чтобы настроить голос.", + saveError: "Не удалось обновить голосовые настройки", + provider: "Провайдер", + providers: { + local: "Локально", + openai: "OpenAI", + }, + sttModel: "Модель распознавания речи", + ttsModel: "Модель голоса", + voice: "Голос", + notDownloadedOption: "{{label}} (не скачана)", + save: "Сохранить", + retry: "Повторить", + dictation: { + title: "Диктовка", + enabled: "Диктовка", + enabledHint: "Говорите в поле сообщения вместо набора текста", + }, + voiceMode: { + title: "Голосовой режим", + info: "Голосовой режим расшифровывает вашу речь и зачитывает ответы агента вслух.", + enabled: "Голосовой режим", + enabledHint: "Общайтесь с агентами без рук", + sttProvider: "Распознавание речи", + ttsProvider: "Синтез речи", + }, + openai: { + apiKey: "API-ключ", + apiKeyHint: "Хранится на хосте и больше не показывается.", + keyPlaceholder: "sk-...", + replaceKeyPlaceholder: "Новый ключ", + removeKey: "Удалить", + removeKeyTitle: "Удалить API-ключ OpenAI?", + removeKeyMessage: + "Функции, настроенные на OpenAI, перестанут работать, пока вы не добавите ключ.", + baseUrl: "Базовый URL", + baseUrlHint: + "Для серверов, совместимых с OpenAI. Оставьте пустым, чтобы использовать OpenAI.", + missingKeyWarning: "Выбран OpenAI, но API-ключ не задан.", + }, + models: { + loading: "Загрузка моделей...", + loadError: "Не удалось загрузить локальные модели", + downloaded: "Скачана", + inUse: "Используется", + recommended: "Рекомендуется", + download: "Скачать", + downloading: "Скачивание...", + downloadingPercent: "Скачивание {{percent}}%", + extracting: "Распаковка...", + delete: "Удалить", + deleteAction: "Удалить {{label}}", + deleteTitle: "Удалить {{label}}?", + deleteMessage: "Освободится {{size}}. Позже её можно скачать снова.", + languages: "Языки {{label}}", + moreLanguages: "{{languages}} и ещё {{count}}", + }, + providerSettings: { + title: "Провайдеры", + info: "Модели и учётные данные каждого провайдера. Локальные модели работают на этом хосте; размер и языки видны до загрузки.", + localStt: "Локальное распознавание речи", + localTts: "Локальный синтез речи", + openai: "OpenAI", + downloaded_one: "Загружено моделей: {{count}}", + downloaded_other: "Загружено моделей: {{count}}", + configured: "Настроено", + notConfigured: "Не настроено", + }, + }, general: { title: "Основные", sending: "Отправка", diff --git a/packages/app/src/i18n/resources/zh-CN.ts b/packages/app/src/i18n/resources/zh-CN.ts index ca221d7d3a..d0b31445b9 100644 --- a/packages/app/src/i18n/resources/zh-CN.ts +++ b/packages/app/src/i18n/resources/zh-CN.ts @@ -2084,6 +2084,7 @@ export const zhCN: TranslationResources = { connections: "连接", agents: "Agents", metadata: "元数据", + voice: "语音", workspaces: "Workspaces", providers: "Providers", usage: "使用情况", @@ -2105,6 +2106,75 @@ export const zhCN: TranslationResources = { docs: "文档", saveError: "无法更新元数据生成设置", }, + voice: { + offline: "连接到此主机以配置语音。", + updateHost: "请更新主机以配置语音。", + saveError: "无法更新语音设置", + provider: "提供方", + providers: { + local: "本地", + openai: "OpenAI", + }, + sttModel: "语音转文字模型", + ttsModel: "语音模型", + voice: "音色", + notDownloadedOption: "{{label}}（未下载）", + save: "保存", + retry: "重试", + dictation: { + title: "听写", + enabled: "听写", + enabledHint: "对着消息框说话，无需打字", + }, + voiceMode: { + title: "语音模式", + info: "语音模式会转写你说的话，并朗读智能体的回复。", + enabled: "语音模式", + enabledHint: "免手动与智能体对话", + sttProvider: "语音转文字", + ttsProvider: "文字转语音", + }, + openai: { + apiKey: "API 密钥", + apiKeyHint: "保存在主机上，不会再次显示。", + keyPlaceholder: "sk-...", + replaceKeyPlaceholder: "新密钥", + removeKey: "移除", + removeKeyTitle: "移除 OpenAI API 密钥？", + removeKeyMessage: "在添加密钥之前，设为 OpenAI 的功能将无法使用。", + baseUrl: "基础 URL", + baseUrlHint: "用于兼容 OpenAI 的服务器。留空则使用 OpenAI。", + missingKeyWarning: "已选择 OpenAI，但未设置 API 密钥。", + }, + models: { + loading: "正在加载模型...", + loadError: "无法加载本地模型", + downloaded: "已下载", + inUse: "使用中", + recommended: "推荐", + download: "下载", + downloading: "正在下载...", + downloadingPercent: "正在下载 {{percent}}%", + extracting: "正在解压...", + delete: "删除", + deleteAction: "删除 {{label}}", + deleteTitle: "删除 {{label}}？", + deleteMessage: "这将释放 {{size}}。之后可以重新下载。", + languages: "{{label}} 支持的语言", + moreLanguages: "{{languages}} 等另外 {{count}} 种", + }, + providerSettings: { + title: "提供方", + info: "各提供方的模型和凭据。本地模型在此主机上运行，下载前会显示大小和支持的语言。", + localStt: "本地语音转文字", + localTts: "本地文字转语音", + openai: "OpenAI", + downloaded_one: "已下载 {{count}} 个模型", + downloaded_other: "已下载 {{count}} 个模型", + configured: "已配置", + notConfigured: "未配置", + }, + }, general: { title: "通用", sending: "发送", diff --git a/packages/app/src/screens/settings-screen.tsx b/packages/app/src/screens/settings-screen.tsx index 31f5058940..55b67819fa 100644 --- a/packages/app/src/screens/settings-screen.tsx +++ b/packages/app/src/screens/settings-screen.tsx @@ -36,6 +36,7 @@ import { Code2, Smartphone, Sparkles, + Mic, Blocks, Globe, PanelLeft, @@ -117,6 +118,7 @@ import { PluginSettingsContent } from "@/plugins/settings"; import { useInstalledPlugins } from "@/plugins/registry"; import { HostPluginsPage } from "@/screens/settings/plugins-page"; import { MetadataGenerationPage } from "@/screens/settings/metadata-generation-page"; +import { HostVoicePage } from "@/screens/settings/voice/voice-page"; import ProjectsScreen from "@/screens/projects-screen"; import ProjectSettingsScreen from "@/screens/project-settings-screen"; import { SETTINGS_DESKTOP_SIDEBAR_WIDTH, useIsCompactFormFactor } from "@/constants/layout"; @@ -234,6 +236,7 @@ const HOST_SECTION_ITEMS: HostSectionItem[] = [ { id: "pair-device", labelKey: "openProject.tiles.pairDevice.title", icon: Smartphone }, { id: "agents", labelKey: "settings.hostSections.agents", icon: Bot }, { id: "metadata", labelKey: "settings.hostSections.metadata", icon: Sparkles }, + { id: "voice", labelKey: "settings.hostSections.voice", icon: Mic }, { id: "workspaces", labelKey: "settings.hostSections.workspaces", icon: FolderGit2 }, { id: "providers", labelKey: "settings.hostSections.providers", icon: Boxes }, { id: "usage", labelKey: "settings.hostSections.usage", icon: Gauge }, @@ -256,6 +259,8 @@ function renderHostSettingsContent( return ; case "metadata": return ; + case "voice": + return ; case "workspaces": return ; case "providers": diff --git a/packages/app/src/screens/settings/voice/use-local-speech-models.ts b/packages/app/src/screens/settings/voice/use-local-speech-models.ts new file mode 100644 index 0000000000..b68a7b8135 --- /dev/null +++ b/packages/app/src/screens/settings/voice/use-local-speech-models.ts @@ -0,0 +1,71 @@ +import { useCallback } from "react"; +import { useTranslation } from "react-i18next"; +import type { LocalSpeechModel } from "@getpaseo/protocol/speech/rpc-schemas"; +import { useFetchQuery } from "@/data/query"; +import { useHostRuntimeClient, useHostRuntimeIsConnected } from "@/runtime/host-runtime"; +import { isLocalModelBusy } from "./voice-settings"; + +const DOWNLOAD_POLL_INTERVAL_MS = 1000; + +const localSpeechModelsQueryKey = (serverId: string) => ["speech-local-models", serverId] as const; + +export interface LocalSpeechModels { + models: LocalSpeechModel[]; + isLoading: boolean; + error: Error | null; + refresh: () => void; + download: (modelId: string) => Promise; + remove: (modelId: string) => Promise; +} + +// The daemon reports download progress through the list, so the list polls while any +// model is downloading or extracting. +export function useLocalSpeechModels(serverId: string): LocalSpeechModels { + const { t } = useTranslation(); + const client = useHostRuntimeClient(serverId); + const isConnected = useHostRuntimeIsConnected(serverId); + const query = useFetchQuery({ + queryKey: localSpeechModelsQueryKey(serverId), + enabled: Boolean(client && isConnected), + dataShape: "list", + staleTimeMs: 0, + queryFn: () => { + if (!client) throw new Error(t("workspace.terminal.hostDisconnected")); + return client.listLocalSpeechModels(); + }, + refetchInterval: (current) => + current.state.data?.some(isLocalModelBusy) ? DOWNLOAD_POLL_INTERVAL_MS : false, + }); + const { refetch } = query; + + const refresh = useCallback(() => { + void refetch(); + }, [refetch]); + + const download = useCallback( + async (modelId: string) => { + if (!client) return; + await client.downloadLocalSpeechModel(modelId); + await refetch(); + }, + [client, refetch], + ); + + const remove = useCallback( + async (modelId: string) => { + if (!client) return; + await client.deleteLocalSpeechModel(modelId); + await refetch(); + }, + [client, refetch], + ); + + return { + models: query.data ?? [], + isLoading: query.isLoading, + error: query.error, + refresh, + download, + remove, + }; +} diff --git a/packages/app/src/screens/settings/voice/voice-page.tsx b/packages/app/src/screens/settings/voice/voice-page.tsx new file mode 100644 index 0000000000..553ac017f7 --- /dev/null +++ b/packages/app/src/screens/settings/voice/voice-page.tsx @@ -0,0 +1,703 @@ +import { useCallback, useMemo, useRef, useState } from "react"; +import { Alert as NativeAlert, Text, View } from "react-native"; +import { useTranslation } from "react-i18next"; +import type { TFunction } from "i18next"; +import { StyleSheet } from "react-native-unistyles"; +import { Download, Languages, Trash2 } from "lucide-react-native"; +import type { + LocalSpeechModel, + SpeechSettings, + SpeechSettingsPatch, +} from "@getpaseo/protocol/speech/rpc-schemas"; +import { + SettingsCard, + SettingsCollapsibleRow, + SettingsInfoTip, + SettingsRow, + SettingsSection, + SettingsSelect, +} from "@/components/settings"; +import { Alert } from "@/components/ui/alert"; +import { Button } from "@/components/ui/button"; +import { FormTextInput } from "@/components/ui/form-field"; +import { LoadingSpinner } from "@/components/ui/loading-spinner"; +import { StatusBadge } from "@/components/ui/status-badge"; +import { Switch } from "@/components/ui/switch"; +import type { EditingTextInputHandle } from "@/components/ui/text-input"; +import { useDaemonConfig } from "@/hooks/use-daemon-config"; +import { useHostFeature } from "@/runtime/host-features"; +import { useHostRuntimeIsConnected } from "@/runtime/host-runtime"; +import { confirmDialog } from "@/utils/confirm-dialog"; +import { useLocalSpeechModels, type LocalSpeechModels } from "./use-local-speech-models"; +import { + SPEECH_PROVIDERS, + buildModelOptions, + buildOpenAiVoiceOptions, + formatLanguages, + formatModelSize, + getLocalModelState, + resolveSelectedModel, + resolveSelectedVoice, + summarizeLocalModels, + toSpeechProvider, + usesOpenAi, + type SpeechModelKind, + type SpeechProvider, +} from "./voice-settings"; + +type SavePatch = (patch: SpeechSettingsPatch) => void; + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +export function HostVoicePage({ serverId }: { serverId: string }) { + const { t } = useTranslation(); + const isConnected = useHostRuntimeIsConnected(serverId); + const supported = useHostFeature(serverId, "speechSettings"); + const { config, patchConfig } = useDaemonConfig(serverId); + const localModels = useLocalSpeechModels(serverId); + const { refresh: refreshModels } = localModels; + + const save = useCallback( + (patch) => { + patchConfig({ speech: patch }) + .then(refreshModels) + .catch((error: unknown) => { + NativeAlert.alert(t("settings.voice.saveError"), errorMessage(error)); + }); + }, + [patchConfig, refreshModels, t], + ); + + if (!isConnected) { + return ; + } + if (!supported) { + return ; + } + const speech = config?.speech; + if (!speech) { + return ( + + + + ); + } + + return ( + + + + + + ); +} + +// --------------------------------------------------------------------------- +// Feature sections +// --------------------------------------------------------------------------- + +interface FeatureSectionProps { + speech: SpeechSettings; + localModels: LocalSpeechModel[]; + save: SavePatch; +} + +function DictationSection({ speech, localModels, save }: FeatureSectionProps) { + const { t } = useTranslation(); + const { dictation } = speech; + const setEnabled = useCallback((enabled: boolean) => save({ dictation: { enabled } }), [save]); + const setProvider = useCallback( + (provider: SpeechProvider) => save({ dictation: { stt: { provider } } }), + [save], + ); + const setModel = useCallback((model: string) => save({ dictation: { stt: { model } } }), [save]); + + return ( + + + + + + {dictation.enabled ? ( + + ) : null} + {dictation.enabled ? ( + + ) : null} + + + ); +} + +function VoiceModeSection({ speech, localModels, save }: FeatureSectionProps) { + const { t } = useTranslation(); + const { voiceMode } = speech; + const sttProvider = toSpeechProvider(voiceMode.stt.provider); + const ttsProvider = toSpeechProvider(voiceMode.tts.provider); + const setEnabled = useCallback((enabled: boolean) => save({ voiceMode: { enabled } }), [save]); + const setSttProvider = useCallback( + (provider: SpeechProvider) => save({ voiceMode: { stt: { provider } } }), + [save], + ); + const setSttModel = useCallback( + (model: string) => save({ voiceMode: { stt: { model } } }), + [save], + ); + const setTtsProvider = useCallback( + (provider: SpeechProvider) => save({ voiceMode: { tts: { provider } } }), + [save], + ); + const setTtsModel = useCallback( + (model: string) => save({ voiceMode: { tts: { model } } }), + [save], + ); + const setTtsVoice = useCallback( + (voice: string) => save({ voiceMode: { tts: { voice } } }), + [save], + ); + const voiceOptions = useMemo( + () => buildOpenAiVoiceOptions(resolveSelectedVoice(voiceMode.tts.voice)), + [voiceMode.tts.voice], + ); + + return ( + + + + + + {voiceMode.enabled ? ( + + ) : null} + {voiceMode.enabled ? ( + + ) : null} + {voiceMode.enabled ? ( + + ) : null} + {voiceMode.enabled ? ( + + ) : null} + {voiceMode.enabled && ttsProvider === "openai" ? ( + + ) : null} + + + ); +} + +function ProviderRow({ + label, + value, + onChange, + testID, +}: { + label: string; + value: SpeechProvider; + onChange: (provider: SpeechProvider) => void; + testID: string; +}) { + const { t } = useTranslation(); + const options = useMemo( + () => + SPEECH_PROVIDERS.map((provider) => ({ + value: provider, + label: t(`settings.voice.providers.${provider}`), + })), + [t], + ); + return ( + + ); +} + +function ModelRow({ + kind, + provider, + model, + localModels, + onChange, +}: { + kind: SpeechModelKind; + provider: SpeechProvider; + model: string | undefined; + localModels: LocalSpeechModel[]; + onChange: (model: string) => void; +}) { + const { t } = useTranslation(); + const selected = resolveSelectedModel({ kind, provider, model, localModels }); + const options = useMemo( + () => + buildModelOptions({ + kind, + provider, + selected, + localModels, + notDownloadedLabel: (label) => t("settings.voice.notDownloadedOption", { label }), + }), + [kind, localModels, provider, selected, t], + ); + return ( + + ); +} + +// --------------------------------------------------------------------------- +// Providers: one collapsible row per provider, each with a one-line summary +// --------------------------------------------------------------------------- + +function ProvidersSection({ + speech, + localModels, + save, +}: { + speech: SpeechSettings; + localModels: LocalSpeechModels; + save: SavePatch; +}) { + const { t } = useTranslation(); + const missingKey = usesOpenAi(speech) && !speech.openai.apiKeyConfigured; + return ( + + {missingKey ? ( + + + + ) : null} + + + + + + + ); +} + +function LocalProviderRow({ + kind, + localModels, +}: { + kind: SpeechModelKind; + localModels: LocalSpeechModels; +}) { + const { t } = useTranslation(); + const { models, isLoading, error, refresh, download, remove } = localModels; + const kindModels = useMemo(() => models.filter((model) => model.kind === kind), [kind, models]); + const label = + kind === "stt" + ? t("settings.voice.providerSettings.localStt") + : t("settings.voice.providerSettings.localTts"); + + if (error && kindModels.length === 0) { + return ( + + + + + + ); + } + return ( + + {kindModels.map((model) => ( + + ))} + + ); +} + +function localSummaryLabel( + models: readonly LocalSpeechModel[], + kind: SpeechModelKind, + t: TFunction, +): string { + const summary = summarizeLocalModels(models, kind); + switch (summary.kind) { + case "downloading": + return summary.percent === null + ? t("settings.voice.models.downloading") + : t("settings.voice.models.downloadingPercent", { percent: summary.percent }); + case "extracting": + return t("settings.voice.models.extracting"); + case "downloaded": + return t("settings.voice.providerSettings.downloaded", { count: summary.count }); + } +} + +function OpenAiProviderRow({ speech, save }: { speech: SpeechSettings; save: SavePatch }) { + const { t } = useTranslation(); + const { apiKeyConfigured, baseUrl } = speech.openai; + const keyInput = useRef(null); + const [draftKey, setDraftKey] = useState(""); + const [draftBaseUrl, setDraftBaseUrl] = useState(null); + + const saveKey = useCallback(() => { + save({ openai: { apiKey: draftKey } }); + keyInput.current?.replaceText(""); + setDraftKey(""); + }, [draftKey, save]); + const removeKey = useCallback(() => { + void confirmDialog({ + title: t("settings.voice.openai.removeKeyTitle"), + message: t("settings.voice.openai.removeKeyMessage"), + confirmLabel: t("settings.voice.openai.removeKey"), + destructive: true, + }).then((confirmed) => { + if (confirmed) save({ openai: { apiKey: null } }); + return undefined; + }); + }, [save, t]); + const saveBaseUrl = useCallback(() => { + if (draftBaseUrl === null) return; + save({ openai: { baseUrl: draftBaseUrl.trim() || null } }); + setDraftBaseUrl(null); + }, [draftBaseUrl, save]); + + const apiKeyInfo = useMemo( + () => ( + + ), + [t], + ); + const baseUrlInfo = useMemo( + () => ( + + ), + [t], + ); + + return ( + + + + + + {apiKeyConfigured ? ( + + ) : null} + + + + + + + + + + ); +} + +function LocalModelRow({ + model, + onDownload, + onDelete, +}: { + model: LocalSpeechModel; + onDownload: (modelId: string) => Promise; + onDelete: (modelId: string) => Promise; +}) { + const { t, i18n } = useTranslation(); + const state = getLocalModelState(model); + const [actionError, setActionError] = useState(null); + + const download = useCallback(() => { + setActionError(null); + onDownload(model.id).catch((error: unknown) => setActionError(errorMessage(error))); + }, [model.id, onDownload]); + const remove = useCallback(() => { + void confirmDialog({ + title: t("settings.voice.models.deleteTitle", { label: model.label }), + message: t("settings.voice.models.deleteMessage", { + size: formatModelSize(model.downloadSizeBytes), + }), + confirmLabel: t("settings.voice.models.delete"), + destructive: true, + }) + .then((confirmed) => { + if (!confirmed) return undefined; + setActionError(null); + return onDelete(model.id); + }) + .catch((error: unknown) => setActionError(errorMessage(error))); + }, [model.downloadSizeBytes, model.id, model.label, onDelete, t]); + + const languages = formatLanguages({ + languages: model.languages, + locale: i18n.language, + formatMore: (listed, count) => + t("settings.voice.models.moreLanguages", { languages: listed, count }), + }); + const details = useMemo( + () => ( + + + + {formatModelSize(model.downloadSizeBytes)} + {state.kind === "installed" ? ( + + ) : null} + {model.isDefault ? ( + + ) : null} + + ), + [ + languages, + model.description, + model.downloadSizeBytes, + model.id, + model.inUse, + model.isDefault, + model.label, + state.kind, + t, + ], + ); + + let control = null; + if (state.kind === "downloading") { + control = ( + + + + {state.percent === null + ? t("settings.voice.models.downloading") + : t("settings.voice.models.downloadingPercent", { percent: state.percent })} + + + ); + } else if (state.kind === "extracting") { + control = ( + + + {t("settings.voice.models.extracting")} + + ); + } else if (state.kind === "installed") { + control = model.inUse ? null : ( + + ); + } else { + control = ( + + ); + } + + return ( + + {control} + + ); +} + +const styles = StyleSheet.create((theme) => ({ + loading: { + paddingVertical: theme.spacing[8], + alignItems: "center", + }, + spinnerColor: { + color: theme.colors.foregroundMuted, + }, + sectionAlert: { + marginBottom: theme.spacing[3], + }, + inputControls: { + flexDirection: "row", + alignItems: "center", + flexWrap: "wrap", + gap: theme.spacing[2], + }, + input: { + minWidth: 220, + }, + details: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + }, + modelSize: { + color: theme.colors.foregroundMuted, + fontSize: theme.fontSize.sm, + fontVariant: ["tabular-nums"], + }, + progress: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + }, + progressText: { + color: theme.colors.foregroundMuted, + fontSize: theme.fontSize.sm, + fontVariant: ["tabular-nums"], + }, +})); diff --git a/packages/app/src/screens/settings/voice/voice-settings.test.ts b/packages/app/src/screens/settings/voice/voice-settings.test.ts new file mode 100644 index 0000000000..3dc828adbe --- /dev/null +++ b/packages/app/src/screens/settings/voice/voice-settings.test.ts @@ -0,0 +1,126 @@ +import { describe, expect, it } from "vitest"; +import type { LocalSpeechModel } from "@getpaseo/protocol/speech/rpc-schemas"; +import { + buildModelOptions, + formatLanguages, + formatModelSize, + getLocalModelState, + resolveSelectedModel, + summarizeLocalModels, +} from "./voice-settings"; + +function model(overrides: Partial): LocalSpeechModel { + return { + id: "parakeet-tdt-0.6b-v2-int8", + kind: "stt", + label: "Parakeet v2", + description: "Accurate English transcription.", + languages: ["en"], + downloadSizeBytes: 482_468_385, + isDefault: true, + inUse: true, + status: "installed", + ...overrides, + }; +} + +const catalog = [ + model({}), + model({ + id: "parakeet-tdt-110m-en-int8", + label: "Parakeet Small", + isDefault: false, + status: "not_installed", + inUse: false, + }), + model({ id: "kokoro-en-v0_19", kind: "tts", label: "Kokoro", inUse: false }), +]; + +describe("voice settings", () => { + it("falls back to the catalog default when no local model is saved", () => { + expect( + resolveSelectedModel({ + kind: "stt", + provider: "local", + model: undefined, + localModels: catalog, + }), + ).toBe("parakeet-tdt-0.6b-v2-int8"); + expect( + resolveSelectedModel({ + kind: "tts", + provider: "openai", + model: undefined, + localModels: catalog, + }), + ).toBe("tts-1"); + }); + + it("marks local models that still need a download", () => { + expect( + buildModelOptions({ + kind: "stt", + provider: "local", + selected: "parakeet-tdt-0.6b-v2-int8", + localModels: catalog, + notDownloadedLabel: (label) => `${label} (not downloaded)`, + }), + ).toEqual([ + { value: "parakeet-tdt-0.6b-v2-int8", label: "Parakeet v2" }, + { value: "parakeet-tdt-110m-en-int8", label: "Parakeet Small (not downloaded)" }, + ]); + }); + + it("keeps a custom OpenAI-compatible model id selectable", () => { + const options = buildModelOptions({ + kind: "stt", + provider: "openai", + selected: "my-whisper", + localModels: catalog, + notDownloadedLabel: (label) => label, + }); + expect(options.at(-1)).toEqual({ value: "my-whisper", label: "my-whisper" }); + }); + + it("reports download progress as a percentage", () => { + expect( + getLocalModelState(model({ status: "downloading", receivedBytes: 241, totalBytes: 482 })), + ).toEqual({ kind: "downloading", percent: 50 }); + expect(getLocalModelState(model({ status: "downloading" }))).toEqual({ + kind: "downloading", + percent: null, + }); + }); + + it("summarizes each local provider by its downloads", () => { + expect(summarizeLocalModels(catalog, "stt")).toEqual({ kind: "downloaded", count: 1 }); + expect(summarizeLocalModels(catalog, "tts")).toEqual({ kind: "downloaded", count: 1 }); + const downloading = [ + ...catalog, + model({ + id: "parakeet-tdt-0.6b-v3-int8", + status: "downloading", + receivedBytes: 1, + totalBytes: 4, + }), + ]; + expect(summarizeLocalModels(downloading, "stt")).toEqual({ kind: "downloading", percent: 25 }); + expect(summarizeLocalModels(downloading, "tts")).toEqual({ kind: "downloaded", count: 1 }); + }); + + it("formats download sizes", () => { + expect(formatModelSize(482_468_385)).toBe("482 MB"); + expect(formatModelSize(1_250_000_000)).toBe("1.3 GB"); + }); + + it("names a short language list and counts the rest of a long one", () => { + const formatMore = (languages: string, count: number) => `${languages} and ${count} more`; + expect(formatLanguages({ languages: ["en", "de"], locale: "en", formatMore })).toBe( + "English, German", + ); + const whisper = ["en", "zh", "de", "es", "ru", "ko", "fr", "ja", "pt", "tr", "pl", "ca"]; + expect(formatLanguages({ languages: whisper, locale: "en", formatMore })).toBe( + "English, Chinese, German, Spanish, Russian, Korean, French, Japanese, Portuguese, Turkish and 2 more", + ); + }); +}); diff --git a/packages/app/src/screens/settings/voice/voice-settings.ts b/packages/app/src/screens/settings/voice/voice-settings.ts new file mode 100644 index 0000000000..9e45616acc --- /dev/null +++ b/packages/app/src/screens/settings/voice/voice-settings.ts @@ -0,0 +1,173 @@ +import type { LocalSpeechModel, SpeechSettings } from "@getpaseo/protocol/speech/rpc-schemas"; + +// What the Voice page needs to know about speech providers and models, kept free of React +// so the option lists and model states are testable on their own. + +export type SpeechProvider = "local" | "openai"; +export type SpeechModelKind = "stt" | "tts"; + +export const SPEECH_PROVIDERS: readonly SpeechProvider[] = ["local", "openai"]; + +// OpenAI accepts free-form ids so self-hosted servers can use their own; these are the +// ones the page offers. A saved id outside the list is still shown as an option. +const OPENAI_MODELS: Record = { + stt: { + ids: ["whisper-1", "gpt-4o-transcribe", "gpt-4o-mini-transcribe"], + defaultId: "whisper-1", + }, + tts: { ids: ["tts-1", "tts-1-hd", "gpt-4o-mini-tts"], defaultId: "tts-1" }, +}; + +const OPENAI_VOICES = ["alloy", "ash", "coral", "echo", "fable", "nova", "onyx", "sage", "shimmer"]; +const DEFAULT_OPENAI_VOICE = "alloy"; + +export interface SpeechModelOption { + value: string; + label: string; +} + +export function toSpeechProvider(provider: string): SpeechProvider { + return provider === "openai" ? "openai" : "local"; +} + +export function resolveSelectedModel(input: { + kind: SpeechModelKind; + provider: SpeechProvider; + model: string | undefined; + localModels: readonly LocalSpeechModel[]; +}): string { + if (input.model) return input.model; + if (input.provider === "openai") return OPENAI_MODELS[input.kind].defaultId; + return input.localModels.find((m) => m.kind === input.kind && m.isDefault)?.id ?? ""; +} + +export function buildModelOptions(input: { + kind: SpeechModelKind; + provider: SpeechProvider; + selected: string; + localModels: readonly LocalSpeechModel[]; + notDownloadedLabel: (label: string) => string; +}): SpeechModelOption[] { + if (input.provider === "openai") { + return withSelected( + OPENAI_MODELS[input.kind].ids.map((id) => ({ value: id, label: id })), + input.selected, + ); + } + return input.localModels + .filter((model) => model.kind === input.kind) + .map((model) => ({ + value: model.id, + label: model.status === "installed" ? model.label : input.notDownloadedLabel(model.label), + })); +} + +export function resolveSelectedVoice(voice: string | undefined): string { + return voice ?? DEFAULT_OPENAI_VOICE; +} + +export function buildOpenAiVoiceOptions(selected: string): SpeechModelOption[] { + return withSelected( + OPENAI_VOICES.map((voice) => ({ value: voice, label: voice })), + selected, + ); +} + +function withSelected(options: SpeechModelOption[], selected: string): SpeechModelOption[] { + if (!selected || options.some((option) => option.value === selected)) return options; + return [...options, { value: selected, label: selected }]; +} + +export function usesOpenAi(settings: SpeechSettings): boolean { + return ( + (settings.dictation.enabled && settings.dictation.stt.provider === "openai") || + (settings.voiceMode.enabled && + (settings.voiceMode.stt.provider === "openai" || + settings.voiceMode.tts.provider === "openai")) + ); +} + +export type LocalModelState = + | { kind: "installed" } + | { kind: "not_installed" } + | { kind: "downloading"; percent: number | null } + | { kind: "extracting" } + | { kind: "failed"; error: string }; + +export function getLocalModelState(model: LocalSpeechModel): LocalModelState { + switch (model.status) { + case "installed": + return { kind: "installed" }; + case "downloading": { + const total = model.totalBytes ?? 0; + const received = model.receivedBytes ?? 0; + return { + kind: "downloading", + percent: total > 0 ? Math.min(100, Math.floor((received / total) * 100)) : null, + }; + } + case "extracting": + return { kind: "extracting" }; + case "failed": + return { kind: "failed", error: model.error ?? "" }; + default: + return { kind: "not_installed" }; + } +} + +export function isLocalModelBusy(model: LocalSpeechModel): boolean { + return model.status === "downloading" || model.status === "extracting"; +} + +// What a collapsed local provider row says: an active download wins over the count. +export type LocalModelsSummary = + | { kind: "downloading"; percent: number | null } + | { kind: "extracting" } + | { kind: "downloaded"; count: number }; + +export function summarizeLocalModels( + models: readonly LocalSpeechModel[], + kind: SpeechModelKind, +): LocalModelsSummary { + const kindModels = models.filter((model) => model.kind === kind); + const busy = kindModels.find(isLocalModelBusy); + if (busy) { + const state = getLocalModelState(busy); + if (state.kind === "downloading") return state; + return { kind: "extracting" }; + } + return { + kind: "downloaded", + count: kindModels.filter((model) => model.status === "installed").length, + }; +} + +export function formatModelSize(bytes: number): string { + const megabytes = bytes / 1_000_000; + if (megabytes >= 1000) return `${(megabytes / 1000).toFixed(1)} GB`; + return `${Math.round(megabytes)} MB`; +} + +// Multilingual models list close to a hundred languages; past this many the tooltip names +// the first ones and counts the rest. +const MAX_LISTED_LANGUAGES = 10; + +export function formatLanguages(input: { + languages: readonly string[]; + locale: string; + formatMore: (languages: string, count: number) => string; +}): string { + const names = languageNames(input.locale); + const listed = input.languages + .slice(0, MAX_LISTED_LANGUAGES) + .map((code) => names?.of(code) ?? code.toUpperCase()) + .join(", "); + const hidden = input.languages.length - MAX_LISTED_LANGUAGES; + return hidden > 0 ? input.formatMore(listed, hidden) : listed; +} + +// Hermes ships without Intl.DisplayNames; the language code is the fallback label there. +function languageNames(locale: string): Intl.DisplayNames | null { + if (typeof Intl.DisplayNames !== "function") return null; + return new Intl.DisplayNames([locale], { type: "language" }); +} diff --git a/packages/app/src/utils/host-routes.ts b/packages/app/src/utils/host-routes.ts index c13d6f25f9..c422c5ce6b 100644 --- a/packages/app/src/utils/host-routes.ts +++ b/packages/app/src/utils/host-routes.ts @@ -517,6 +517,7 @@ export const HOST_SECTION_SLUGS = [ "pair-device", "agents", "metadata", + "voice", "workspaces", "providers", "usage", diff --git a/packages/app/vitest.config.ts b/packages/app/vitest.config.ts index e2d67aca70..cfafa0da93 100644 --- a/packages/app/vitest.config.ts +++ b/packages/app/vitest.config.ts @@ -91,6 +91,8 @@ export default defineConfig({ "react-native-keyboard-controller", ], }, + // babel-preset-expo compiles JSX with the automatic runtime, so app modules don't import React. + esbuild: { jsx: "automatic" }, // The globals a React Native bundler defines, which esbuild is no longer there to supply for // the package excluded above. define: { diff --git a/packages/client/src/daemon-client.ts b/packages/client/src/daemon-client.ts index e0e4d9ddc5..384c4e391b 100644 --- a/packages/client/src/daemon-client.ts +++ b/packages/client/src/daemon-client.ts @@ -12,6 +12,7 @@ import { CreationClient } from "./creation/index.js"; import type { CreationSnapshot } from "@getpaseo/protocol/messages"; import type { z } from "zod"; import type { SessionEventSubscription } from "@getpaseo/protocol/messages"; +import type { LocalSpeechModel } from "@getpaseo/protocol/speech/rpc-schemas"; import type { ClientCapability } from "@getpaseo/protocol/client-capabilities"; import type { AgentAttentionNotificationPayload } from "@getpaseo/protocol/agent-attention-notification"; import { MAX_WEBSOCKET_MESSAGE_BYTES } from "@getpaseo/protocol/websocket-message-limit"; @@ -5264,6 +5265,34 @@ export class DaemonClient { }); } + async listLocalSpeechModels(requestId?: string): Promise { + const payload = await this.sendCorrelatedSessionRequest({ + requestId, + message: { type: "speech.local_models.list.request" }, + responseType: "speech.local_models.list.response", + }); + if (payload.error) throw new Error(payload.error); + return payload.models; + } + + async downloadLocalSpeechModel(modelId: string, requestId?: string): Promise { + const payload = await this.sendCorrelatedSessionRequest({ + requestId, + message: { type: "speech.local_models.download.request", modelId }, + responseType: "speech.local_models.download.response", + }); + if (payload.error) throw new Error(payload.error); + } + + async deleteLocalSpeechModel(modelId: string, requestId?: string): Promise { + const payload = await this.sendCorrelatedSessionRequest({ + requestId, + message: { type: "speech.local_models.delete.request", modelId }, + responseType: "speech.local_models.delete.response", + }); + if (payload.error) throw new Error(payload.error); + } + sendBrowserAutomationExecuteResponse(response: BrowserAutomationExecuteResponse): void { this.sendSessionMessageStrict(response); } diff --git a/packages/protocol/src/messages.ts b/packages/protocol/src/messages.ts index a7a2182d94..9c944d74f2 100644 --- a/packages/protocol/src/messages.ts +++ b/packages/protocol/src/messages.ts @@ -63,6 +63,16 @@ import { ScheduleRunOnceResponseSchema, ScheduleUpdateResponseSchema, } from "./schedule/rpc-schemas.js"; +import { + SpeechLocalModelsDeleteRequestSchema, + SpeechLocalModelsDeleteResponseSchema, + SpeechLocalModelsDownloadRequestSchema, + SpeechLocalModelsDownloadResponseSchema, + SpeechLocalModelsListRequestSchema, + SpeechLocalModelsListResponseSchema, + SpeechSettingsPatchSchema, + SpeechSettingsSchema, +} from "./speech/rpc-schemas.js"; import { LoopRunRequestSchema, LoopListRequestSchema, @@ -208,6 +218,8 @@ export const MutableDaemonConfigSchema = z skills: z.object({ selection: AgentSkillSelectionSchema.optional() }).strict().optional(), pluginsEnabled: z.boolean().optional(), plugins: z.record(PluginIdSchema, PluginSourceSchema).optional(), + // COMPAT(speechSettings): added in v0.12.0, optional because older daemons omit it. + speech: SpeechSettingsSchema.optional(), }) .passthrough(); @@ -228,6 +240,7 @@ export const MutableDaemonConfigPatchSchema = z agentProfiles: z.array(AgentProfileSchema).optional(), pluginsEnabled: z.boolean().optional(), plugins: z.record(PluginIdSchema, PluginSourceSchema).optional(), + speech: SpeechSettingsPatchSchema.optional(), }) .partial() .passthrough(); @@ -3390,6 +3403,9 @@ export const SessionInboundMessageSchema = z.discriminatedUnion("type", [ ChatWaitRequestSchema, ScheduleCreateRequestSchema, ScheduleListRequestSchema, + SpeechLocalModelsListRequestSchema, + SpeechLocalModelsDownloadRequestSchema, + SpeechLocalModelsDeleteRequestSchema, ScheduleInspectRequestSchema, ScheduleLogsRequestSchema, SchedulePauseRequestSchema, @@ -3620,6 +3636,8 @@ export const ServerInfoStatusPayloadSchema = z daemonStatusRpc: z.boolean().optional(), // COMPAT(daemonConfigReload): added in v0.4.0, remove gate after 2027-02-14. daemonConfigReload: z.boolean().optional(), + // COMPAT(speechSettings): added in v0.12.0, remove gate after 2027-04-18. + speechSettings: z.boolean().optional(), // COMPAT(relayConfig): added in v0.2.6, remove gate after 2027-01-31. relayConfig: z.boolean().optional(), // COMPAT(pushTokenRevocation): added in v0.3.2, remove gate after 2027-02-10. @@ -7061,6 +7079,9 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [ ChatWaitResponseSchema, ScheduleCreateResponseSchema, ScheduleListResponseSchema, + SpeechLocalModelsListResponseSchema, + SpeechLocalModelsDownloadResponseSchema, + SpeechLocalModelsDeleteResponseSchema, ScheduleInspectResponseSchema, ScheduleLogsResponseSchema, SchedulePauseResponseSchema, diff --git a/packages/protocol/src/speech/rpc-schemas.ts b/packages/protocol/src/speech/rpc-schemas.ts new file mode 100644 index 0000000000..9e8dec9749 --- /dev/null +++ b/packages/protocol/src/speech/rpc-schemas.ts @@ -0,0 +1,165 @@ +import { z } from "zod"; + +// --------------------------------------------------------------------------- +// Speech settings: the `speech` section of the mutable daemon config. +// --------------------------------------------------------------------------- + +// Provider ids stay plain strings so an old client can parse a daemon that adds one. +const SpeechSttSettingsSchema = z + .object({ + provider: z.string(), + // Omitted means the provider's default model. + model: z.string().optional(), + language: z.string().optional(), + }) + .passthrough(); + +const SpeechTtsSettingsSchema = z + .object({ + provider: z.string(), + model: z.string().optional(), + voice: z.string().optional(), + }) + .passthrough(); + +export const SpeechSettingsSchema = z + .object({ + dictation: z + .object({ + enabled: z.boolean(), + stt: SpeechSttSettingsSchema, + }) + .passthrough(), + voiceMode: z + .object({ + enabled: z.boolean(), + stt: SpeechSttSettingsSchema, + tts: SpeechTtsSettingsSchema, + }) + .passthrough(), + openai: z + .object({ + // The key itself is write-only; readers only learn whether one is set. + apiKeyConfigured: z.boolean(), + baseUrl: z.string().optional(), + }) + .passthrough(), + }) + .passthrough(); + +// null clears a field back to its default. Changing `provider` without `model` clears +// the model, because model ids belong to one provider. +const SpeechSttSettingsPatchSchema = z + .object({ + provider: z.string().optional(), + model: z.string().nullable().optional(), + language: z.string().nullable().optional(), + }) + .passthrough(); + +const SpeechTtsSettingsPatchSchema = z + .object({ + provider: z.string().optional(), + model: z.string().nullable().optional(), + voice: z.string().nullable().optional(), + }) + .passthrough(); + +export const SpeechSettingsPatchSchema = z + .object({ + dictation: z + .object({ + enabled: z.boolean().optional(), + stt: SpeechSttSettingsPatchSchema.optional(), + }) + .passthrough() + .optional(), + voiceMode: z + .object({ + enabled: z.boolean().optional(), + stt: SpeechSttSettingsPatchSchema.optional(), + tts: SpeechTtsSettingsPatchSchema.optional(), + }) + .passthrough() + .optional(), + openai: z + .object({ + apiKey: z.string().nullable().optional(), + baseUrl: z.string().nullable().optional(), + }) + .passthrough() + .optional(), + }) + .passthrough(); + +export type SpeechSettings = z.infer; +export type SpeechSettingsPatch = z.infer; + +// --------------------------------------------------------------------------- +// Local speech models: catalog, install state, download, delete. +// --------------------------------------------------------------------------- + +export const LocalSpeechModelSchema = z + .object({ + id: z.string(), + // "stt" or "tts". + kind: z.string(), + label: z.string(), + description: z.string(), + languages: z.array(z.string()), + downloadSizeBytes: z.number(), + isDefault: z.boolean(), + // Selected by an enabled feature; the daemon keeps it installed. + inUse: z.boolean(), + // "installed" | "not_installed" | "downloading" | "extracting" | "failed". + status: z.string(), + receivedBytes: z.number().optional(), + totalBytes: z.number().optional(), + error: z.string().optional(), + }) + .passthrough(); + +export type LocalSpeechModel = z.infer; + +export const SpeechLocalModelsListRequestSchema = z.object({ + type: z.literal("speech.local_models.list.request"), + requestId: z.string(), +}); + +export const SpeechLocalModelsDownloadRequestSchema = z.object({ + type: z.literal("speech.local_models.download.request"), + requestId: z.string(), + modelId: z.string(), +}); + +export const SpeechLocalModelsDeleteRequestSchema = z.object({ + type: z.literal("speech.local_models.delete.request"), + requestId: z.string(), + modelId: z.string(), +}); + +export const SpeechLocalModelsListResponseSchema = z.object({ + type: z.literal("speech.local_models.list.response"), + payload: z.object({ + requestId: z.string(), + models: z.array(LocalSpeechModelSchema), + error: z.string().nullable(), + }), +}); + +// Resolves once the download starts; poll the list for progress. +export const SpeechLocalModelsDownloadResponseSchema = z.object({ + type: z.literal("speech.local_models.download.response"), + payload: z.object({ + requestId: z.string(), + error: z.string().nullable(), + }), +}); + +export const SpeechLocalModelsDeleteResponseSchema = z.object({ + type: z.literal("speech.local_models.delete.response"), + payload: z.object({ + requestId: z.string(), + error: z.string().nullable(), + }), +}); diff --git a/packages/server/src/server/authorization/operation-permissions.ts b/packages/server/src/server/authorization/operation-permissions.ts index e7e77bc0d3..bedd6f79c6 100644 --- a/packages/server/src/server/authorization/operation-permissions.ts +++ b/packages/server/src/server/authorization/operation-permissions.ts @@ -174,6 +174,9 @@ const INBOUND_PERMISSION = { set_agent_model_request: "workspace.write", set_agent_thinking_request: "workspace.write", set_daemon_config_request: "daemon.manage", + "speech.local_models.list.request": "daemon.read", + "speech.local_models.download.request": "daemon.manage", + "speech.local_models.delete.request": "daemon.manage", set_voice_mode: "workspace.write", shutdown_server_request: "daemon.manage", start_workspace_script_request: "workspace.write", @@ -399,6 +402,9 @@ const OUTBOUND_PERMISSION = { set_agent_model_response: "workspace.write", set_agent_thinking_response: "workspace.write", set_daemon_config_response: "daemon.manage", + "speech.local_models.list.response": "daemon.read", + "speech.local_models.download.response": "daemon.manage", + "speech.local_models.delete.response": "daemon.manage", set_voice_mode_response: "workspace.write", start_workspace_script_response: "workspace.write", stash_list_response: "workspace.read", diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 727c7b7899..12bd803da9 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -131,6 +131,7 @@ import type { OpenAiSpeechProviderConfig } from "./speech/providers/openai/confi import type { LocalSpeechProviderConfig } from "./speech/providers/local/config.js"; import type { RequestedSpeechProviders } from "./speech/speech-types.js"; import { createSpeechService } from "./speech/speech-runtime.js"; +import { createSpeechSettingsOwner } from "./speech/speech-settings.js"; import { AgentManager } from "./agent/agent-manager.js"; import { AgentStorage } from "./agent/agent-storage.js"; import { attachAgentStoragePersistence } from "./persistence-hooks.js"; @@ -370,6 +371,8 @@ export interface PaseoSpeechSttLanguages { export interface PaseoSpeechConfig { providers: RequestedSpeechProviders; + // Where local models live, whether or not a feature currently uses the local provider. + localModelsDir: string; sttLanguages?: PaseoSpeechSttLanguages; local?: PaseoLocalSpeechConfig; } @@ -537,6 +540,10 @@ function resolveExpressTrustProxySetting(config: PaseoDaemonConfig): true | stri return config.trustedProxies ?? ["loopback"]; } +function resolveReloadEnv(config: PaseoDaemonConfig): NodeJS.ProcessEnv { + return config.configReload?.env ?? process.env; +} + function createInitialMutableDaemonConfig(config: PaseoDaemonConfig): MutableDaemonConfig { const providers = config.providerOverrides ?? {}; @@ -596,13 +603,24 @@ export async function createPaseoDaemon( const elapsed = () => `${(performance.now() - bootstrapStart).toFixed(0)}ms`; const daemonVersion = config.daemonVersion ?? resolveDaemonVersion(import.meta.url); const initialMutableConfig = createInitialMutableDaemonConfig(config); + const speechService = createSpeechService({ + logger, + openaiConfig: config.openai, + speechConfig: config.speech, + }); + logger.info({ elapsed: elapsed() }, "Speech service created"); const daemonConfigStore = new DaemonConfigStore(config.paseoHome, initialMutableConfig, logger, { relayEnabledMutable: config.relayEnabledMutable ?? true, startupPersisted: config.configReload?.startupPersisted, + speech: createSpeechSettingsOwner({ + paseoHome: config.paseoHome, + env: resolveReloadEnv(config), + speechService, + }), reloadSource: { resolve: (persisted) => { const reloaded = resolveConfigFromPersisted(config.paseoHome, persisted, { - env: config.configReload?.env ?? process.env, + env: resolveReloadEnv(config), cli: config.configReload?.cli, relayEnabledFallback: config.configReload?.relayEnabledFallback, }); @@ -1591,13 +1609,6 @@ export async function createPaseoDaemon( logger.info({ route: agentMcpRoute, enabled: mcpEnabled }, "Agent MCP route mounted"); } - const speechService = createSpeechService({ - logger, - openaiConfig: config.openai, - speechConfig: config.speech, - }); - logger.info({ elapsed: elapsed() }, "Speech service created"); - logger.info({ elapsed: elapsed() }, "Bootstrap complete, ready to start listening"); const start = async () => { diff --git a/packages/server/src/server/daemon-client.e2e.test.ts b/packages/server/src/server/daemon-client.e2e.test.ts index b6518d773d..a442dfe6a9 100644 --- a/packages/server/src/server/daemon-client.e2e.test.ts +++ b/packages/server/src/server/daemon-client.e2e.test.ts @@ -1318,9 +1318,9 @@ test("emits disabled voice capability reasons on fresh daemon startup", async () expect(voice).toBeTruthy(); expect(voice?.dictation.enabled).toBe(false); - expect(voice?.dictation.reason).toBe("Dictation is disabled in daemon config."); + expect(voice?.dictation.reason).toBe("Dictation is off. Turn it on in Settings > Voice."); expect(voice?.voice.enabled).toBe(false); - expect(voice?.voice.reason).toBe("Realtime voice is disabled in daemon config."); + expect(voice?.voice.reason).toBe("Voice mode is off. Turn it on in Settings > Voice."); } finally { await client.close().catch(() => undefined); await isolatedCtx.cleanup(); diff --git a/packages/server/src/server/daemon-config-store.test.ts b/packages/server/src/server/daemon-config-store.test.ts index 3629a0fd18..61ee2a1dd0 100644 --- a/packages/server/src/server/daemon-config-store.test.ts +++ b/packages/server/src/server/daemon-config-store.test.ts @@ -5,6 +5,7 @@ import { afterEach, describe, expect, test } from "vitest"; import { DaemonConfigStore, applyMutableProviderConfigToOverrides } from "./daemon-config-store.js"; import { loadPersistedConfig } from "./persisted-config.js"; +import { readSpeechSettings } from "./speech/speech-settings.js"; import type { PersistedConfig } from "./persisted-config.js"; import type { MutableDaemonConfig } from "@getpaseo/protocol/messages"; @@ -1193,3 +1194,110 @@ describe("DaemonConfigStore reload", () => { }); }); }); + +describe("DaemonConfigStore speech settings", () => { + const tempDirs: string[] = []; + + afterEach(() => { + for (const dir of tempDirs) rmSync(dir, { recursive: true, force: true }); + }); + + function createSpeechStore(initialPersisted?: PersistedConfig) { + const paseoHome = mkdtempSync(path.join(tmpdir(), "paseo-daemon-config-speech-")); + tempDirs.push(paseoHome); + if (initialPersisted) { + writeFileSync( + path.join(paseoHome, "config.json"), + `${JSON.stringify(initialPersisted, null, 2)}\n`, + ); + } + const persisted = loadPersistedConfig(paseoHome); + const applied: PersistedConfig[] = []; + const speech = { + read: (next: PersistedConfig) => readSpeechSettings(next, {}), + apply: (next: PersistedConfig) => applied.push(next), + }; + const store = new DaemonConfigStore(paseoHome, reloadableConfig(persisted), undefined, { + speech, + reloadSource: { + resolve: (next) => ({ mutable: reloadableConfig(next), overrideControlledPaths: [] }), + }, + }); + return { paseoHome, store, applied, persisted }; + } + + test("keeps voice mode off until it is turned on", () => { + const { paseoHome, store } = createSpeechStore(); + + expect(store.get().speech?.dictation.enabled).toBe(true); + expect(store.get().speech?.voiceMode.enabled).toBe(false); + + const result = store.patch({ speech: { voiceMode: { enabled: true } } }); + + expect(result.speech?.voiceMode.enabled).toBe(true); + expect(loadPersistedConfig(paseoHome).features?.voiceMode?.enabled).toBe(true); + }); + + test("stores the OpenAI key without ever returning it", () => { + const { paseoHome, store, applied } = createSpeechStore(); + + const result = store.patch({ + speech: { dictation: { stt: { provider: "openai" } }, openai: { apiKey: " sk-test " } }, + }); + + expect(result.speech?.openai).toEqual({ apiKeyConfigured: true }); + expect(JSON.stringify(result)).not.toContain("sk-test"); + expect(loadPersistedConfig(paseoHome).providers?.openai?.apiKey).toBe("sk-test"); + expect(applied).toHaveLength(1); + }); + + test("applies a replaced key even though the visible settings do not change", () => { + const { store, applied } = createSpeechStore({ + version: 1, + providers: { openai: { apiKey: "sk-old" } }, + }); + + store.patch({ speech: { openai: { apiKey: "sk-new" } } }); + + expect(applied.map((next) => next.providers?.openai?.apiKey)).toEqual(["sk-new"]); + }); + + test("switching provider clears the previous provider's model", () => { + const { paseoHome, store } = createSpeechStore({ + version: 1, + features: { dictation: { stt: { provider: "local", model: "parakeet-tdt-0.6b-v3-int8" } } }, + }); + + const result = store.patch({ speech: { dictation: { stt: { provider: "openai" } } } }); + + expect(result.speech?.dictation.stt).toEqual({ provider: "openai" }); + expect(loadPersistedConfig(paseoHome).features?.dictation?.stt).toEqual({ + provider: "openai", + }); + }); + + test("rejects a local model id outside the catalog without persisting it", () => { + const { paseoHome, store, applied } = createSpeechStore(); + + expect(() => + store.patch({ speech: { voiceMode: { tts: { provider: "local", model: "nope" } } } }), + ).toThrow("Unknown local voice model: nope"); + expect(loadPersistedConfig(paseoHome).features?.voiceMode).toBeUndefined(); + expect(applied).toEqual([]); + }); + + test("reload applies hand-edited speech settings without a restart", () => { + const { paseoHome, store, applied, persisted } = createSpeechStore(); + writeFileSync( + path.join(paseoHome, "config.json"), + `${JSON.stringify({ ...persisted, features: { voiceMode: { enabled: false } } }, null, 2)}\n`, + ); + + const result = store.reload(); + + expect(result.appliedPaths).toContain("features.voiceMode"); + expect(result.restartRequiredPaths).toEqual([]); + expect(store.get().speech?.voiceMode.enabled).toBe(false); + expect(applied).toHaveLength(1); + }); +}); diff --git a/packages/server/src/server/daemon-config-store.ts b/packages/server/src/server/daemon-config-store.ts index 3e285bded0..432c560a32 100644 --- a/packages/server/src/server/daemon-config-store.ts +++ b/packages/server/src/server/daemon-config-store.ts @@ -9,6 +9,8 @@ import { MutableDaemonConfigPatchSchema, } from "@getpaseo/protocol/messages"; import type { AgentSkillSelection } from "@getpaseo/protocol/messages"; +import type { SpeechSettings, SpeechSettingsPatch } from "@getpaseo/protocol/speech/rpc-schemas"; +import { mergeSpeechSettingsPatch, type SpeechSettingsOwner } from "./speech/speech-settings.js"; export type { MutableDaemonConfig, MutableDaemonConfigPatch } from "@getpaseo/protocol/messages"; @@ -31,8 +33,11 @@ interface SupportedMutableConfigPatch { skills?: MutableDaemonConfig["skills"]; pluginsEnabled?: boolean; plugins?: MutableDaemonConfig["plugins"]; + speech?: SpeechSettingsPatch; } +const SPEECH_PERSISTED_PATHS = ["features.dictation", "features.voiceMode", "providers.openai"]; + interface LoggerLike { child(bindings: Record): LoggerLike; info(...args: unknown[]): void; @@ -276,6 +281,7 @@ function pickSupportedPatchFields(patch: MutableDaemonConfigPatch): SupportedMut ...(patch.agentProfiles !== undefined ? { agentProfiles: patch.agentProfiles } : {}), ...(patch.pluginsEnabled !== undefined ? { pluginsEnabled: patch.pluginsEnabled } : {}), ...(patch.plugins !== undefined ? { plugins: patch.plugins } : {}), + ...(patch.speech !== undefined ? { speech: patch.speech } : {}), }; } @@ -318,6 +324,7 @@ export class DaemonConfigStore { private readonly relayEnabledMutable: boolean; private readonly reloadSource: DaemonConfigReloadSource | undefined; private readonly startupPersisted: PersistedConfig; + private readonly speechOwner: SpeechSettingsOwner | undefined; private lastKnownPersisted: PersistedConfig; constructor( @@ -328,18 +335,21 @@ export class DaemonConfigStore { relayEnabledMutable?: boolean; reloadSource?: DaemonConfigReloadSource; startupPersisted?: PersistedConfig; + speech?: SpeechSettingsOwner; } = {}, ) { this.paseoHome = paseoHome; this.logger = getLogger(logger); + this.relayEnabledMutable = options.relayEnabledMutable ?? true; + this.reloadSource = options.reloadSource; + this.speechOwner = options.speech; + this.startupPersisted = options.startupPersisted ?? loadPersistedConfig(paseoHome, this.logger); + this.lastKnownPersisted = this.startupPersisted; this.current = MutableDaemonConfigSchema.parse({ ...initial, relay: initial.relay ?? { enabled: true }, + ...this.readSpeech(this.startupPersisted), }); - this.relayEnabledMutable = options.relayEnabledMutable ?? true; - this.reloadSource = options.reloadSource; - this.startupPersisted = options.startupPersisted ?? loadPersistedConfig(paseoHome, this.logger); - this.lastKnownPersisted = this.startupPersisted; } public get(): MutableDaemonConfig { @@ -361,9 +371,11 @@ export class DaemonConfigStore { "Relay is controlled by a daemon launch override. Remove PASEO_RELAY_ENABLED or the relay CLI flag before changing it here.", ); } - const { removeProviders = [], ...configPatch } = parsedPatch; + const { removeProviders = [], ...patch } = parsedPatch; + const { speech: speechPatch, ...configPatch } = patch; const removedProviders = Array.from(new Set(removeProviders)); const merged = deepMerge(this.current, configPatch); + if (speechPatch !== undefined) merged.speech = this.readPatchedSpeech(speechPatch); if (parsedPatch.skills?.selection !== undefined) { merged.skills = { selection: parsedPatch.skills.selection }; } @@ -377,30 +389,47 @@ export class DaemonConfigStore { const configChanged = !isEqualValue(this.current, next); - if (!configChanged && removedProviders.length === 0) { + // A replaced API key leaves the speech view unchanged, so a speech patch always persists. + if (!configChanged && removedProviders.length === 0 && speechPatch === undefined) { return this.current; } const { previous: persistedBeforePatch, knownNext } = this.persistConfig( - configPatch, + patch, removedProviders, ); - if (!configChanged) { - this.lastKnownPersisted = knownNext; - return this.current; - } - - try { - this.applyReplacement(next, { removedProviders }); - this.lastKnownPersisted = knownNext; - } catch (error) { - savePersistedConfig(this.paseoHome, persistedBeforePatch, this.logger); - throw error; + const speechChanged = this.speechPersistedChanged(this.lastKnownPersisted, knownNext); + if (configChanged) { + try { + this.applyReplacement(next, { removedProviders }); + } catch (error) { + savePersistedConfig(this.paseoHome, persistedBeforePatch, this.logger); + throw error; + } } + this.lastKnownPersisted = knownNext; + if (speechChanged) this.speechOwner?.apply(knownNext); return this.current; } + private readSpeech(persisted: PersistedConfig): Pick { + return this.speechOwner ? { speech: this.speechOwner.read(persisted) } : {}; + } + + private readPatchedSpeech(speechPatch: SpeechSettingsPatch): SpeechSettings { + if (!this.speechOwner) { + throw new Error("Speech settings are unavailable for this daemon instance"); + } + return this.speechOwner.read(mergeSpeechSettingsPatch(this.lastKnownPersisted, speechPatch)); + } + + private speechPersistedChanged(previous: PersistedConfig, next: PersistedConfig): boolean { + return diffPaths(previous, next).some((path) => + SPEECH_PERSISTED_PATHS.some((owner) => pathBelongsTo(path, owner)), + ); + } + public reload(): DaemonConfigReloadResult { if (!this.reloadSource) { throw new Error("Daemon config reload is unavailable for this daemon instance"); @@ -413,6 +442,7 @@ export class DaemonConfigStore { const desired = MutableDaemonConfigSchema.parse({ ...resolved.mutable, plugins: this.current.plugins, + ...this.readSpeech(persisted), }); const changedSinceLastApply = diffPaths(this.lastKnownPersisted, persisted); const overrideControlledPaths = compactOwnedPaths( @@ -421,7 +451,7 @@ export class DaemonConfigStore { ), resolved.overrideControlledPaths, ); - const appliedPaths = RELOADABLE_PATHS.filter((persistedPath) => { + const appliedPaths: string[] = RELOADABLE_PATHS.filter((persistedPath) => { if (resolved.overrideControlledPaths.some((owner) => pathBelongsTo(persistedPath, owner))) { return false; } @@ -438,6 +468,8 @@ export class DaemonConfigStore { diffPaths(this.startupPersisted, persisted).filter((path) => { if (path === "$schema" || path === "version") return false; if (RELOADABLE_PATHS.some((owner) => pathBelongsTo(path, owner))) return false; + if (this.speechOwner && SPEECH_PERSISTED_PATHS.some((owner) => pathBelongsTo(path, owner))) + return false; return !resolved.overrideControlledPaths.some((owner) => pathBelongsTo(path, owner)); }), [], @@ -446,8 +478,17 @@ export class DaemonConfigStore { const removedProviders = Object.keys(this.current.providers).filter( (provider) => !(provider in desired.providers), ); + const speechChanged = this.speechPersistedChanged(this.lastKnownPersisted, persisted); this.applyReplacement(desired, { removedProviders }); this.lastKnownPersisted = persisted; + if (speechChanged && this.speechOwner) { + this.speechOwner.apply(persisted); + appliedPaths.push( + ...SPEECH_PERSISTED_PATHS.filter((owner) => + changedSinceLastApply.some((path) => pathBelongsTo(path, owner)), + ), + ); + } return { appliedPaths: [...appliedPaths].sort(), @@ -584,8 +625,9 @@ function mergeMutablePatchIntoPersistedConfig(params: { const { persisted, patch, removeProviders, persistRelayEnabled } = params; const daemon = mergeMutableDaemonPatch(persisted.daemon, patch, persistRelayEnabled); const agents = mergeMutableAgentPatch(persisted.agents, patch, removeProviders); + const withSpeech = patch.speech ? mergeSpeechSettingsPatch(persisted, patch.speech) : persisted; return { - ...persisted, + ...withSpeech, ...(patch.pluginsEnabled !== undefined ? { pluginsEnabled: patch.pluginsEnabled } : {}), ...(patch.plugins !== undefined ? { plugins: patch.plugins } : {}), ...(daemon ? { daemon } : { daemon: undefined }), diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 08d3f0eefb..aea8a38dc9 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -182,6 +182,10 @@ import { import { ScheduleSession } from "./session/schedule/schedule-session.js"; import { ProviderCatalogSession } from "./session/provider/provider-catalog-session.js"; import { UsageSession } from "./session/usage/usage-session.js"; +import { + SpeechModelsSession, + type SpeechModelsSessionOptions, +} from "./session/speech/speech-models-session.js"; import { WorkspaceFilesSession } from "./session/files/workspace-files-session.js"; import { AgentConfigSession } from "./session/agent-config/agent-config-session.js"; import { ProjectConfigSession } from "./session/project-config/project-config-session.js"; @@ -525,6 +529,7 @@ export interface SessionOptions { stt: Resolvable; sttLanguage?: string; tts: Resolvable; + speechModels?: SpeechModelsSessionOptions["runtime"]; terminalManager: TerminalManager | null; providerSnapshotManager: ProviderSnapshotManager; hubExecutionAgents?: HubExecutionAgents; @@ -792,6 +797,7 @@ export class Session { private readonly scheduleSession: ScheduleSession; private readonly providerCatalogSession: ProviderCatalogSession; private readonly usageSession: UsageSession; + private readonly speechModelsSession: SpeechModelsSession; private readonly workspaceFilesSession: WorkspaceFilesSession; private readonly agentConfigSession: AgentConfigSession; private readonly projectConfigSession: ProjectConfigSession; @@ -840,6 +846,7 @@ export class Session { stt, sttLanguage, tts, + speechModels, terminalManager, providerSnapshotManager, serviceProxy, @@ -1005,6 +1012,10 @@ export class Session { runtime: pluginRuntime, logger: this.sessionLogger, }); + this.speechModelsSession = new SpeechModelsSession({ + emit: (msg) => this.emit(msg), + runtime: speechModels, + }); this.agentConfigSession = new AgentConfigSession({ host: { emit: (msg) => this.emit(msg), @@ -2336,12 +2347,19 @@ export class Session { this.dispatchOrchestrationSkillsMessage(msg) ?? this.dispatchPluginDirectoryMessage(msg) ?? this.dispatchPluginMessage(msg) ?? - this.dispatchTerminalMessage(msg) ?? - this.dispatchScheduleMessage(msg) ?? + this.dispatchHostServiceMessage(msg) ?? this.dispatchMiscMessage(msg); if (promise) await promise; } + private dispatchHostServiceMessage(msg: SessionInboundMessage): Promise | undefined { + return ( + this.dispatchTerminalMessage(msg) ?? + this.dispatchScheduleMessage(msg) ?? + this.speechModelsSession.dispatch(msg) + ); + } + private dispatchWorkspaceLifecycleMessage(msg: SessionInboundMessage): Promise | undefined { return ( this.dispatchWorkspaceStateMessage(msg) ?? diff --git a/packages/server/src/server/session/speech/speech-models-session.test.ts b/packages/server/src/server/session/speech/speech-models-session.test.ts new file mode 100644 index 0000000000..9624475ad4 --- /dev/null +++ b/packages/server/src/server/session/speech/speech-models-session.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, test } from "vitest"; + +import type { SessionOutboundMessage } from "../../messages.js"; +import { + listLocalSpeechModels, + type LocalSpeechModelId, +} from "../../speech/providers/local/models.js"; +import { + LocalSpeechModelInUseError, + type LocalSpeechModelStatus, +} from "../../speech/speech-runtime.js"; +import { SpeechModelsSession } from "./speech-models-session.js"; + +const [firstModel] = listLocalSpeechModels(); + +function createSession(models: LocalSpeechModelStatus[], inUse: LocalSpeechModelId[] = []) { + const emitted: SessionOutboundMessage[] = []; + const downloads: LocalSpeechModelId[] = []; + const session = new SpeechModelsSession({ + emit: (message) => emitted.push(message), + runtime: { + listLocalModels: async () => models, + downloadLocalModel: (modelId) => { + downloads.push(modelId); + }, + deleteLocalModel: async (modelId) => { + if (inUse.includes(modelId)) throw new LocalSpeechModelInUseError(modelId); + }, + }, + }); + return { session, emitted, downloads }; +} + +describe("SpeechModelsSession", () => { + test("lists catalog details and download progress before anything is installed", async () => { + const { session, emitted } = createSession([ + { + spec: firstModel, + state: { status: "downloading", receivedBytes: 10, totalBytes: 40 }, + inUse: true, + }, + ]); + + await session.dispatch({ type: "speech.local_models.list.request", requestId: "r1" }); + + expect(emitted).toEqual([ + { + type: "speech.local_models.list.response", + payload: { + requestId: "r1", + error: null, + models: [ + { + id: firstModel.id, + kind: "stt", + label: firstModel.label, + description: firstModel.description, + languages: [...firstModel.languages], + downloadSizeBytes: firstModel.downloadSizeBytes, + isDefault: true, + inUse: true, + status: "downloading", + receivedBytes: 10, + totalBytes: 40, + }, + ], + }, + }, + ]); + }); + + test("rejects a model id outside the catalog without starting a download", () => { + const { session, emitted, downloads } = createSession([]); + + void session.dispatch({ + type: "speech.local_models.download.request", + requestId: "r2", + modelId: "../../etc", + }); + + expect(downloads).toEqual([]); + expect(emitted).toEqual([ + { + type: "speech.local_models.download.response", + payload: { requestId: "r2", error: "Unknown local speech model: ../../etc" }, + }, + ]); + }); + + test("reports why an in-use model cannot be deleted", async () => { + const { session, emitted } = createSession([], [firstModel.id]); + + await session.dispatch({ + type: "speech.local_models.delete.request", + requestId: "r3", + modelId: firstModel.id, + }); + + expect(emitted).toEqual([ + { + type: "speech.local_models.delete.response", + payload: { requestId: "r3", error: new LocalSpeechModelInUseError(firstModel.id).message }, + }, + ]); + }); +}); diff --git a/packages/server/src/server/session/speech/speech-models-session.ts b/packages/server/src/server/session/speech/speech-models-session.ts new file mode 100644 index 0000000000..d987623a8d --- /dev/null +++ b/packages/server/src/server/session/speech/speech-models-session.ts @@ -0,0 +1,124 @@ +import type { LocalSpeechModel } from "@getpaseo/protocol/speech/rpc-schemas"; + +import type { SessionInboundMessage, SessionOutboundMessage } from "../../messages.js"; +import { LocalSpeechModelIdSchema } from "../../speech/providers/local/models.js"; +import type { LocalSpeechModelStatus, SpeechService } from "../../speech/speech-runtime.js"; + +type SpeechModelsRuntime = Pick< + SpeechService, + "listLocalModels" | "downloadLocalModel" | "deleteLocalModel" +>; + +export interface SpeechModelsSessionOptions { + emit(message: SessionOutboundMessage): void; + // Absent when the daemon runs without speech. + runtime?: SpeechModelsRuntime | null; +} + +export class SpeechModelsSession { + constructor(private readonly options: SpeechModelsSessionOptions) {} + + // Handles the speech.local_models.* requests; returns undefined for any other message. + dispatch(msg: SessionInboundMessage): Promise | undefined { + switch (msg.type) { + case "speech.local_models.list.request": + return this.handleList(msg); + case "speech.local_models.download.request": + this.handleDownload(msg); + return Promise.resolve(); + case "speech.local_models.delete.request": + return this.handleDelete(msg); + default: + return undefined; + } + } + + private async handleList( + msg: Extract, + ): Promise { + try { + const models = await this.requireRuntime().listLocalModels(); + this.options.emit({ + type: "speech.local_models.list.response", + payload: { requestId: msg.requestId, models: models.map(toWireModel), error: null }, + }); + } catch (error) { + this.options.emit({ + type: "speech.local_models.list.response", + payload: { requestId: msg.requestId, models: [], error: describeError(error) }, + }); + } + } + + private handleDownload( + msg: Extract, + ): void { + try { + this.requireRuntime().downloadLocalModel(parseModelId(msg.modelId)); + this.options.emit({ + type: "speech.local_models.download.response", + payload: { requestId: msg.requestId, error: null }, + }); + } catch (error) { + this.options.emit({ + type: "speech.local_models.download.response", + payload: { requestId: msg.requestId, error: describeError(error) }, + }); + } + } + + private async handleDelete( + msg: Extract, + ): Promise { + try { + await this.requireRuntime().deleteLocalModel(parseModelId(msg.modelId)); + this.options.emit({ + type: "speech.local_models.delete.response", + payload: { requestId: msg.requestId, error: null }, + }); + } catch (error) { + this.options.emit({ + type: "speech.local_models.delete.response", + payload: { requestId: msg.requestId, error: describeError(error) }, + }); + } + } + + private requireRuntime(): SpeechModelsRuntime { + if (!this.options.runtime) { + throw new Error("Speech is unavailable on this daemon"); + } + return this.options.runtime; + } +} + +function parseModelId(modelId: string) { + const parsed = LocalSpeechModelIdSchema.safeParse(modelId); + if (!parsed.success) { + throw new Error(`Unknown local speech model: ${modelId}`); + } + return parsed.data; +} + +function toWireModel(model: LocalSpeechModelStatus): LocalSpeechModel { + const { spec, state, inUse } = model; + return { + id: spec.id, + kind: spec.kind === "tts" ? "tts" : "stt", + label: spec.label, + description: spec.description, + languages: [...spec.languages], + downloadSizeBytes: spec.downloadSizeBytes, + isDefault: spec.defaultFor !== undefined, + inUse, + status: state.status, + ...(state.status === "downloading" + ? { receivedBytes: state.receivedBytes, totalBytes: state.totalBytes } + : {}), + ...(state.status === "failed" ? { error: state.error } : {}), + }; +} + +function describeError(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/packages/server/src/server/speech/providers/local/config.ts b/packages/server/src/server/speech/providers/local/config.ts index 8e7d500dc2..e15a8cae64 100644 --- a/packages/server/src/server/speech/providers/local/config.ts +++ b/packages/server/src/server/speech/providers/local/config.ts @@ -29,6 +29,7 @@ export interface LocalSpeechProviderConfig { export interface ResolvedLocalSpeechConfig { local: LocalSpeechProviderConfig | undefined; + modelsDir: string; sttLanguages: LocalSpeechSttLanguageConfig; } @@ -200,11 +201,10 @@ export function resolveLocalSpeechConfig(params: { buildLocalSpeechResolutionInput({ ...params, includeProviderConfig }), ); - const resolvedVoiceTtsSpeakerId = - parsed.voiceLocalTtsSpeakerId ?? - (parsed.voiceLocalTtsModel === "kokoro-en-v0_19" ? 0 : undefined); + const resolvedVoiceTtsSpeakerId = parsed.voiceLocalTtsSpeakerId ?? 0; return { + modelsDir: parsed.modelsDir, sttLanguages: { dictation: parsed.dictationLanguage, voice: parsed.voiceLanguage, diff --git a/packages/server/src/server/speech/providers/local/model-library.ts b/packages/server/src/server/speech/providers/local/model-library.ts new file mode 100644 index 0000000000..4ef19e3fa9 --- /dev/null +++ b/packages/server/src/server/speech/providers/local/model-library.ts @@ -0,0 +1,137 @@ +import type { Logger } from "pino"; + +import { + deleteSherpaOnnxModel, + ensureSherpaOnnxModel, + isSherpaOnnxModelInstalled, +} from "./sherpa/model-downloader.js"; +import { + listLocalSpeechModels, + type LocalSpeechModelId, + type LocalSpeechModelSpec, +} from "./models.js"; + +export type LocalSpeechModelInstallState = + | { status: "installed" } + | { status: "not_installed" } + | { status: "downloading"; receivedBytes: number; totalBytes: number } + | { status: "extracting" } + | { status: "failed"; error: string }; + +export interface LocalSpeechModelEntry { + spec: LocalSpeechModelSpec; + state: LocalSpeechModelInstallState; +} + +type InFlightState = Extract< + LocalSpeechModelInstallState, + { status: "downloading" | "extracting" } +>; + +interface InFlightDownload { + promise: Promise; + state: InFlightState; +} + +export class LocalSpeechModelBusyError extends Error { + constructor(public readonly modelId: LocalSpeechModelId) { + super(`${modelId} is downloading. Wait for the download to finish before deleting it.`); + this.name = "LocalSpeechModelBusyError"; + } +} + +// Owns the models directory: which catalog models are installed, which are downloading, +// and the last failure per model. Concurrent downloads of one model share a single transfer. +export class LocalSpeechModelLibrary { + private readonly inFlight = new Map(); + private readonly failures = new Map(); + private readonly abortController = new AbortController(); + private readonly logger: Logger; + + constructor( + private readonly modelsDir: string, + logger: Logger, + ) { + this.logger = logger.child({ module: "speech", component: "local-model-library" }); + } + + getModelsDir(): string { + return this.modelsDir; + } + + async list(): Promise { + return Promise.all( + listLocalSpeechModels().map(async (spec) => ({ spec, state: await this.getState(spec.id) })), + ); + } + + async listMissing(modelIds: LocalSpeechModelId[]): Promise { + const installed = await Promise.all( + modelIds.map((modelId) => isSherpaOnnxModelInstalled(this.modelsDir, modelId)), + ); + return modelIds.filter((_modelId, index) => !installed[index]); + } + + download(modelId: LocalSpeechModelId): Promise { + const existing = this.inFlight.get(modelId); + if (existing) { + return existing.promise; + } + this.failures.delete(modelId); + const entry: InFlightDownload = { + state: { status: "downloading", receivedBytes: 0, totalBytes: 0 }, + promise: Promise.resolve(""), + }; + entry.promise = ensureSherpaOnnxModel({ + modelsDir: this.modelsDir, + modelId, + logger: this.logger, + signal: this.abortController.signal, + onProgress: (progress) => { + entry.state = + progress.phase === "downloading" + ? { + status: "downloading", + receivedBytes: progress.receivedBytes, + totalBytes: progress.totalBytes, + } + : { status: "extracting" }; + }, + }) + .catch((error: unknown) => { + this.failures.set(modelId, error instanceof Error ? error.message : String(error)); + throw error; + }) + .finally(() => { + this.inFlight.delete(modelId); + }); + this.inFlight.set(modelId, entry); + return entry.promise; + } + + async delete(modelId: LocalSpeechModelId): Promise { + if (this.inFlight.has(modelId)) { + throw new LocalSpeechModelBusyError(modelId); + } + this.failures.delete(modelId); + await deleteSherpaOnnxModel(this.modelsDir, modelId); + this.logger.info({ modelId }, "Deleted local speech model"); + } + + async stop(): Promise { + this.abortController.abort(); + await Promise.allSettled(Array.from(this.inFlight.values(), (entry) => entry.promise)); + } + + private async getState(modelId: LocalSpeechModelId): Promise { + const inFlight = this.inFlight.get(modelId); + if (inFlight) { + return inFlight.state; + } + if (await isSherpaOnnxModelInstalled(this.modelsDir, modelId)) { + return { status: "installed" }; + } + const failure = this.failures.get(modelId); + return failure ? { status: "failed", error: failure } : { status: "not_installed" }; + } +} diff --git a/packages/server/src/server/speech/providers/local/models.test.ts b/packages/server/src/server/speech/providers/local/models.test.ts index e40cc25384..be010dcae9 100644 --- a/packages/server/src/server/speech/providers/local/models.test.ts +++ b/packages/server/src/server/speech/providers/local/models.test.ts @@ -13,12 +13,8 @@ import path from "node:path"; import pino from "pino"; import { describe, expect, onTestFinished, test } from "vitest"; -import { - ensureLocalSpeechModels, - getLocalSpeechModelDir, - listLocalSpeechModels, - listMissingLocalSpeechModels, -} from "./models.js"; +import { LocalSpeechModelLibrary } from "./model-library.js"; +import { getLocalSpeechModelDir, listLocalSpeechModels } from "./models.js"; const logger = pino({ level: "silent" }); const modelId = "parakeet-tdt-0.6b-v2-int8"; @@ -55,16 +51,31 @@ describe("local speech models", () => { const encoder = path.join(modelDir, spec.requiredFiles[0]); truncateSync(encoder, 1_000); - expect(await listMissingLocalSpeechModels({ modelsDir, modelIds: [modelId] })).toEqual([ - modelId, - ]); + const library = new LocalSpeechModelLibrary(modelsDir, logger); + expect(await library.listMissing([modelId])).toEqual([modelId]); - await ensureLocalSpeechModels({ modelsDir, modelIds: [modelId], logger }); + await library.download(modelId); expect(readFileSync(encoder, "utf8")).toBe( `complete ${spec.requiredFiles[0]}`.padEnd(64_000, "."), ); expect(existsSync(archive)).toBe(false); - expect(await listMissingLocalSpeechModels({ modelsDir, modelIds: [modelId] })).toEqual([]); + expect(await library.listMissing([modelId])).toEqual([]); + }); + + test("lists install state and deletes an installed model", async () => { + const { modelsDir } = setupModelsDirWithArchive(); + const library = new LocalSpeechModelLibrary(modelsDir, logger); + const stateOf = async () => + (await library.list()).find((entry) => entry.spec.id === modelId)?.state; + + expect(await stateOf()).toEqual({ status: "not_installed" }); + await library.download(modelId); + expect(await stateOf()).toEqual({ status: "installed" }); + + await library.delete(modelId); + + expect(await stateOf()).toEqual({ status: "not_installed" }); + expect(existsSync(getLocalSpeechModelDir(modelsDir, modelId))).toBe(false); }); }); diff --git a/packages/server/src/server/speech/providers/local/models.ts b/packages/server/src/server/speech/providers/local/models.ts index a884be86d7..7d4c7f56b5 100644 --- a/packages/server/src/server/speech/providers/local/models.ts +++ b/packages/server/src/server/speech/providers/local/models.ts @@ -1,11 +1,8 @@ -import { - ensureSherpaOnnxModels, - getSherpaOnnxModelDir, - isSherpaOnnxModelInstalled, -} from "./sherpa/model-downloader.js"; +import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./sherpa/model-downloader.js"; import { DEFAULT_LOCAL_STT_MODEL, DEFAULT_LOCAL_TTS_MODEL, + LocalSpeechModelIdSchema, LocalSttModelIdSchema, LocalTtsModelIdSchema, listSherpaOnnxModels, @@ -17,6 +14,7 @@ import { export { DEFAULT_LOCAL_STT_MODEL, DEFAULT_LOCAL_TTS_MODEL, + LocalSpeechModelIdSchema, LocalSttModelIdSchema, LocalTtsModelIdSchema, type LocalSpeechModelId, @@ -34,16 +32,6 @@ export function getLocalSpeechModelDir(modelsDir: string, modelId: LocalSpeechMo return getSherpaOnnxModelDir(modelsDir, modelId); } -export async function listMissingLocalSpeechModels(options: { - modelsDir: string; - modelIds: LocalSpeechModelId[]; -}): Promise { - const installed = await Promise.all( - options.modelIds.map((modelId) => isSherpaOnnxModelInstalled(options.modelsDir, modelId)), - ); - return options.modelIds.filter((_modelId, index) => !installed[index]); -} - export async function ensureLocalSpeechModels(options: { modelsDir: string; modelIds: LocalSpeechModelId[]; diff --git a/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts b/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts index 06d838c49a..b142f52dd6 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts @@ -4,43 +4,467 @@ export type SherpaOnnxModelKind = "stt-offline" | "tts"; type DefaultModelRole = "stt" | "tts"; -interface SherpaOnnxCatalogEntry { - kind: SherpaOnnxModelKind; +// How sherpa-onnx loads a model, with file names relative to the extracted directory. +export type SttEngine = + | { family: "transducer"; encoder: string; decoder: string; joiner: string; tokens: string } + | { family: "whisper"; encoder: string; decoder: string; tokens: string } + | { family: "sense_voice"; model: string; tokens: string } + | { family: "dolphin"; model: string; tokens: string } + | { family: "nemo_ctc"; model: string; tokens: string }; + +export type TtsEngine = + | { + family: "kokoro"; + model: string; + voices: string; + tokens: string; + dataDir: string; + // Multi-language Kokoro models read pronunciations from these instead of espeak alone. + lexicons?: readonly string[]; + dictDir?: string; + ruleFsts?: readonly string[]; + } + | { family: "kitten"; model: string; voices: string; tokens: string; dataDir: string } + | { family: "vits"; model: string; tokens: string; dataDir: string }; + +interface SherpaOnnxCatalogEntryBase { + label: string; + // One user-facing sentence shown before download. + description: string; + // BCP 47 language tags the model transcribes or speaks. + languages: readonly string[]; + // Exact size of the release archive. The downloader rejects any other size. + downloadSizeBytes: number; archiveUrl: string; extractedDir: string; - requiredFiles: string[]; - description: string; defaultFor?: DefaultModelRole; } +type SherpaOnnxCatalogEntry = + | (SherpaOnnxCatalogEntryBase & { kind: "stt-offline"; engine: SttEngine }) + | (SherpaOnnxCatalogEntryBase & { kind: "tts"; engine: TtsEngine }); + +// Every sherpa-onnx release archive extracts to a directory named after the archive. +function releaseArchive(tag: "asr-models" | "tts-models", name: string) { + return { + archiveUrl: `https://github.com/k2-fsa/sherpa-onnx/releases/download/${tag}/${name}.tar.bz2`, + extractedDir: name, + }; +} + +const PARAKEET_INT8_ENGINE = { + family: "transducer", + encoder: "encoder.int8.onnx", + decoder: "decoder.int8.onnx", + joiner: "joiner.int8.onnx", + tokens: "tokens.txt", +} as const satisfies SttEngine; + +function whisperEngine(size: string): SttEngine { + return { + family: "whisper", + encoder: `${size}-encoder.int8.onnx`, + decoder: `${size}-decoder.int8.onnx`, + tokens: `${size}-tokens.txt`, + }; +} + +function piperEngine(voice: string): TtsEngine { + return { + family: "vits", + model: `${voice}.onnx`, + tokens: "tokens.txt", + dataDir: "espeak-ng-data", + }; +} + +// Parakeet TDT v3 transcribes these 25 European languages and detects which one is spoken. +const PARAKEET_V3_LANGUAGES = [ + "bg", + "hr", + "cs", + "da", + "nl", + "en", + "et", + "fi", + "fr", + "de", + "el", + "hu", + "it", + "lv", + "lt", + "mt", + "pl", + "pt", + "ro", + "sk", + "sl", + "es", + "sv", + "ru", + "uk", +] as const; + +// The 99 languages multilingual Whisper models transcribe, detecting which one is spoken. +const WHISPER_LANGUAGES = [ + "en", + "zh", + "de", + "es", + "ru", + "ko", + "fr", + "ja", + "pt", + "tr", + "pl", + "ca", + "nl", + "ar", + "sv", + "it", + "id", + "hi", + "fi", + "vi", + "he", + "uk", + "el", + "ms", + "cs", + "ro", + "da", + "hu", + "ta", + "no", + "th", + "ur", + "hr", + "bg", + "lt", + "la", + "mi", + "ml", + "cy", + "sk", + "te", + "fa", + "lv", + "bn", + "sr", + "az", + "sl", + "kn", + "et", + "mk", + "br", + "eu", + "is", + "hy", + "ne", + "mn", + "bs", + "kk", + "sq", + "sw", + "gl", + "mr", + "pa", + "si", + "km", + "sn", + "yo", + "so", + "af", + "oc", + "ka", + "be", + "tg", + "sd", + "gu", + "am", + "yi", + "lo", + "uz", + "fo", + "ht", + "ps", + "tk", + "nn", + "mt", + "sa", + "lb", + "my", + "bo", + "tl", + "mg", + "as", + "tt", + "haw", + "ln", + "ha", + "ba", + "jv", + "su", +] as const; + +// Dolphin's 40 Eastern languages. It also covers 22 Chinese dialects under "zh". +const DOLPHIN_LANGUAGES = [ + "zh", + "ja", + "th", + "ru", + "ko", + "id", + "vi", + "yue", + "hi", + "ur", + "ms", + "uz", + "ar", + "fa", + "bn", + "ta", + "te", + "ug", + "gu", + "my", + "tl", + "kk", + "or", + "ne", + "mn", + "km", + "jv", + "lo", + "si", + "fil", + "ps", + "pa", + "kab", + "ba", + "ks", + "tg", + "su", + "mr", + "ky", + "az", +] as const; + export const SHERPA_ONNX_MODEL_CATALOG = { "parakeet-tdt-0.6b-v2-int8": { kind: "stt-offline", - archiveUrl: - "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8.tar.bz2", - extractedDir: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8", - requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", "tokens.txt"], - description: "NVIDIA Parakeet TDT v2 (offline NeMo transducer, English).", + label: "Parakeet v2", + description: "Accurate English transcription that runs well on a laptop CPU.", + languages: ["en"], + downloadSizeBytes: 482_468_385, + ...releaseArchive("asr-models", "sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8"), + engine: PARAKEET_INT8_ENGINE, defaultFor: "stt", }, + "parakeet-tdt-110m-en-int8": { + kind: "stt-offline", + label: "Parakeet Small", + description: "Small, fast English model with punctuation. A quick download for any laptop.", + languages: ["en"], + downloadSizeBytes: 108_035_095, + ...releaseArchive("asr-models", "sherpa-onnx-nemo-parakeet_tdt_transducer_110m-en-36000-int8"), + engine: PARAKEET_INT8_ENGINE, + }, "parakeet-tdt-0.6b-v3-int8": { kind: "stt-offline", - archiveUrl: - "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2", - extractedDir: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8", - requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", "tokens.txt"], - description: - "NVIDIA Parakeet TDT v3 (offline NeMo transducer, 25 European languages, auto-detected).", + label: "Parakeet v3", + description: "Parakeet for 25 European languages, detecting which one you speak.", + languages: PARAKEET_V3_LANGUAGES, + downloadSizeBytes: 487_170_055, + ...releaseArchive("asr-models", "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8"), + engine: PARAKEET_INT8_ENGINE, + }, + "whisper-tiny-en": { + kind: "stt-offline", + label: "Whisper Tiny (English)", + description: "Small English Whisper model. Quick to download, less accurate than Parakeet.", + languages: ["en"], + downloadSizeBytes: 118_071_777, + ...releaseArchive("asr-models", "sherpa-onnx-whisper-tiny.en"), + engine: whisperEngine("tiny.en"), + }, + "whisper-base": { + kind: "stt-offline", + label: "Whisper Base", + description: "Compact Whisper model for 99 languages, detecting which one you speak.", + languages: WHISPER_LANGUAGES, + downloadSizeBytes: 207_557_382, + ...releaseArchive("asr-models", "sherpa-onnx-whisper-base"), + engine: whisperEngine("base"), + }, + "whisper-turbo": { + kind: "stt-offline", + label: "Whisper Turbo", + description: "Whisper's most accurate model for 99 languages and Cantonese. Slower on a CPU.", + languages: [...WHISPER_LANGUAGES, "yue"], + downloadSizeBytes: 563_790_207, + ...releaseArchive("asr-models", "sherpa-onnx-whisper-turbo"), + engine: whisperEngine("turbo"), + }, + "sense-voice-int8": { + kind: "stt-offline", + label: "SenseVoice", + description: "Chinese, Cantonese, English, Japanese and Korean, detecting which one you speak.", + languages: ["zh", "yue", "en", "ja", "ko"], + downloadSizeBytes: 165_783_878, + ...releaseArchive("asr-models", "sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2025-09-09"), + engine: { family: "sense_voice", model: "model.int8.onnx", tokens: "tokens.txt" }, + }, + "dolphin-base-int8": { + kind: "stt-offline", + label: "Dolphin", + description: "Small model for 40 Asian and Middle Eastern languages and Chinese dialects.", + languages: DOLPHIN_LANGUAGES, + downloadSizeBytes: 80_671_385, + ...releaseArchive("asr-models", "sherpa-onnx-dolphin-base-ctc-multi-lang-int8-2025-04-02"), + engine: { family: "dolphin", model: "model.int8.onnx", tokens: "tokens.txt" }, + }, + "giga-am-v3-ru": { + kind: "stt-offline", + label: "GigaAM v3 (Russian)", + description: "Russian transcription with punctuation.", + languages: ["ru"], + downloadSizeBytes: 170_197_019, + ...releaseArchive( + "asr-models", + "sherpa-onnx-nemo-transducer-punct-giga-am-v3-russian-2025-12-16", + ), + engine: { + family: "transducer", + encoder: "encoder.int8.onnx", + decoder: "decoder.onnx", + joiner: "joiner.onnx", + tokens: "tokens.txt", + }, + }, + "parakeet-tdt-ctc-0.6b-ja-int8": { + kind: "stt-offline", + label: "Parakeet (Japanese)", + description: "Japanese transcription with punctuation.", + languages: ["ja"], + downloadSizeBytes: 489_389_564, + ...releaseArchive("asr-models", "sherpa-onnx-nemo-parakeet-tdt_ctc-0.6b-ja-35000-int8"), + engine: { family: "nemo_ctc", model: "model.int8.onnx", tokens: "tokens.txt" }, }, "kokoro-en-v0_19": { kind: "tts", - archiveUrl: - "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2", - extractedDir: "kokoro-en-v0_19", - requiredFiles: ["model.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"], - description: "Kokoro TTS (higher quality; larger).", + label: "Kokoro (English)", + description: "Natural-sounding English voice for voice mode replies.", + languages: ["en"], + downloadSizeBytes: 319_625_534, + ...releaseArchive("tts-models", "kokoro-en-v0_19"), + engine: { + family: "kokoro", + model: "model.onnx", + voices: "voices.bin", + tokens: "tokens.txt", + dataDir: "espeak-ng-data", + }, defaultFor: "tts", }, + "kokoro-int8-en-v0_19": { + kind: "tts", + label: "Kokoro Lite (English)", + description: "The same Kokoro English voices, quantized to a third of the download.", + languages: ["en"], + downloadSizeBytes: 103_248_205, + ...releaseArchive("tts-models", "kokoro-int8-en-v0_19"), + engine: { + family: "kokoro", + model: "model.int8.onnx", + voices: "voices.bin", + tokens: "tokens.txt", + dataDir: "espeak-ng-data", + }, + }, + "kokoro-int8-multi-lang-v1_1": { + kind: "tts", + label: "Kokoro (Chinese and English)", + description: "Kokoro voices that speak both Mandarin Chinese and English.", + languages: ["zh", "en"], + downloadSizeBytes: 147_031_220, + ...releaseArchive("tts-models", "kokoro-int8-multi-lang-v1_1"), + engine: { + family: "kokoro", + model: "model.int8.onnx", + voices: "voices.bin", + tokens: "tokens.txt", + dataDir: "espeak-ng-data", + lexicons: ["lexicon-us-en.txt", "lexicon-zh.txt"], + dictDir: "dict", + ruleFsts: ["date-zh.fst", "phone-zh.fst", "number-zh.fst"], + }, + }, + // Kitten v0.8 needs a newer sherpa-onnx than we pin; v0.2 loads on 1.12.28. + "kitten-nano-en-v0_2-fp16": { + kind: "tts", + label: "Kitten Nano (English)", + description: "Tiny, very fast English voice.", + languages: ["en"], + downloadSizeBytes: 26_586_708, + ...releaseArchive("tts-models", "kitten-nano-en-v0_2-fp16"), + engine: { + family: "kitten", + model: "model.fp16.onnx", + voices: "voices.bin", + tokens: "tokens.txt", + dataDir: "espeak-ng-data", + }, + }, + "piper-en-us-libritts-r": { + kind: "tts", + label: "Piper LibriTTS (US English)", + description: "Lightweight American English voice.", + languages: ["en-US"], + downloadSizeBytes: 23_398_348, + ...releaseArchive("tts-models", "vits-piper-en_US-libritts_r-medium-int8"), + engine: piperEngine("en_US-libritts_r-medium"), + }, + "piper-en-gb-alba": { + kind: "tts", + label: "Piper Alba (British English)", + description: "Lightweight British English voice.", + languages: ["en-GB"], + downloadSizeBytes: 21_104_326, + ...releaseArchive("tts-models", "vits-piper-en_GB-alba-medium-int8"), + engine: piperEngine("en_GB-alba-medium"), + }, + "piper-de-thorsten": { + kind: "tts", + label: "Piper Thorsten (German)", + description: "Lightweight German voice.", + languages: ["de"], + downloadSizeBytes: 20_949_833, + ...releaseArchive("tts-models", "vits-piper-de_DE-thorsten-medium-int8"), + engine: piperEngine("de_DE-thorsten-medium"), + }, + "piper-fr-siwis": { + kind: "tts", + label: "Piper Siwis (French)", + description: "Lightweight French voice.", + languages: ["fr"], + downloadSizeBytes: 20_914_888, + ...releaseArchive("tts-models", "vits-piper-fr_FR-siwis-medium-int8"), + engine: piperEngine("fr_FR-siwis-medium"), + }, + "piper-es-davefx": { + kind: "tts", + label: "Piper Davefx (Spanish)", + description: "Lightweight Spanish voice.", + languages: ["es"], + downloadSizeBytes: 21_171_632, + ...releaseArchive("tts-models", "vits-piper-es_ES-davefx-medium-int8"), + engine: piperEngine("es_ES-davefx-medium"), + }, } as const satisfies Record; export type SherpaOnnxModelId = keyof typeof SHERPA_ONNX_MODEL_CATALOG; @@ -97,24 +521,44 @@ function createModelIdSchema(modelIds: readonly T[]): z.ZodTyp .transform((value) => value as T); } +export const LocalSpeechModelIdSchema = createModelIdSchema(ALL_MODEL_IDS); export const LocalSttModelIdSchema = createModelIdSchema(LOCAL_STT_MODEL_IDS); export const LocalTtsModelIdSchema = createModelIdSchema(LOCAL_TTS_MODEL_IDS); export type SherpaOnnxModelSpec = SherpaOnnxCatalogEntry & { id: SherpaOnnxModelId; + // Every file and directory that must exist for the model to count as installed. + requiredFiles: string[]; }; export function listSherpaOnnxModels(): SherpaOnnxModelSpec[] { - return ALL_MODEL_IDS.map((id) => Object.assign({ id }, SHERPA_ONNX_MODEL_CATALOG[id])); + return ALL_MODEL_IDS.map(getSherpaOnnxModelSpec); } export function getSherpaOnnxModelSpec(id: SherpaOnnxModelId): SherpaOnnxModelSpec { - const spec = SHERPA_ONNX_MODEL_CATALOG[id]; + const spec: SherpaOnnxCatalogEntry | undefined = SHERPA_ONNX_MODEL_CATALOG[id]; if (!spec) { throw new Error(`Unknown local speech model id: ${id}`); } return { id, ...spec, + requiredFiles: engineFiles(spec.engine), }; } + +// Every path an engine names, so the downloader can verify an install. +export function engineFiles(engine: SttEngine | TtsEngine): string[] { + return Object.entries(engine).flatMap(([key, value]) => { + if (key === "family") return []; + return typeof value === "string" ? [value] : (value as readonly string[]); + }); +} + +export function getLocalSttEngine(id: LocalSttModelId): SttEngine { + return SHERPA_ONNX_MODEL_CATALOG[id].engine; +} + +export function getLocalTtsEngine(id: LocalTtsModelId): TtsEngine { + return SHERPA_ONNX_MODEL_CATALOG[id].engine; +} diff --git a/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts b/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts index cdd286389f..b7ba511224 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts @@ -8,11 +8,16 @@ import type pino from "pino"; import { getSherpaOnnxModelSpec, type SherpaOnnxModelId } from "./model-catalog.js"; import { spawnProcess } from "../../../../../utils/spawn.js"; +export type SherpaOnnxModelInstallProgress = + | { phase: "downloading"; receivedBytes: number; totalBytes: number } + | { phase: "extracting" }; + export interface EnsureSherpaOnnxModelOptions { modelsDir: string; modelId: SherpaOnnxModelId; logger: pino.Logger; signal?: AbortSignal; + onProgress?: (progress: SherpaOnnxModelInstallProgress) => void; } export function getSherpaOnnxModelDir(modelsDir: string, modelId: SherpaOnnxModelId): string { @@ -41,7 +46,9 @@ async function hasRequiredFiles(modelDir: string, requiredFiles: string[]): Prom interface DownloadToFileOptions { url: string; outputPath: string; + expectedBytes: number; signal?: AbortSignal; + onProgress?: (receivedBytes: number, totalBytes: number) => void; } async function downloadToFile(options: DownloadToFileOptions): Promise { @@ -60,6 +67,13 @@ async function downloadToFile(options: DownloadToFileOptions): Promise { // The fetch ReadableStream type is slightly different from what Readable.fromWeb expects // eslint-disable-next-line @typescript-eslint/no-explicit-any const nodeStream = Readable.fromWeb(res.body as any); + const contentLength = Number(res.headers.get("content-length")); + const totalBytes = contentLength > 0 ? contentLength : options.expectedBytes; + let receivedBytes = 0; + nodeStream.on("data", (chunk: Buffer) => { + receivedBytes += chunk.length; + options.onProgress?.(receivedBytes, totalBytes); + }); try { await pipeline(nodeStream, createWriteStream(tmpPath), { signal: options.signal }); @@ -157,9 +171,13 @@ export async function ensureSherpaOnnxModel( await downloadToFile({ url: spec.archiveUrl, outputPath: archivePath, + expectedBytes: spec.downloadSizeBytes, signal: options.signal, + onProgress: (receivedBytes, totalBytes) => + options.onProgress?.({ phase: "downloading", receivedBytes, totalBytes }), }); } + options.onProgress?.({ phase: "extracting" }); logger.info( { @@ -213,6 +231,16 @@ export async function ensureSherpaOnnxModel( } } +export async function deleteSherpaOnnxModel( + modelsDir: string, + modelId: SherpaOnnxModelId, +): Promise { + const { modelDir, archivePath, extractionDir } = getSherpaOnnxModelPaths(modelsDir, modelId); + await rm(modelDir, { recursive: true, force: true }); + await rm(archivePath, { force: true }); + await rm(extractionDir, { recursive: true, force: true }); +} + export async function ensureSherpaOnnxModels(options: { modelsDir: string; modelIds: SherpaOnnxModelId[]; diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-offline-recognizer.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-offline-recognizer.ts index cfde73addf..5137f51810 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-offline-recognizer.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-offline-recognizer.ts @@ -1,6 +1,8 @@ import { existsSync } from "node:fs"; +import path from "node:path"; import type pino from "pino"; +import { engineFiles, type SttEngine } from "./model-catalog.js"; import { loadSherpaOnnxNode } from "./sherpa-onnx-node-loader.js"; function assertFileExists(filePath: string, label: string): void { @@ -9,16 +11,43 @@ function assertFileExists(filePath: string, label: string): void { } } -export interface SherpaOfflineRecognizerModel { - kind: "nemo_transducer"; - encoder: string; - decoder: string; - joiner: string; - tokens: string; +// Each engine family takes its own block under sherpa's modelConfig. +function buildModelConfig(modelDir: string, engine: SttEngine): Record { + const file = (name: string) => path.join(modelDir, name); + switch (engine.family) { + case "transducer": + return { + transducer: { + encoder: file(engine.encoder), + decoder: file(engine.decoder), + joiner: file(engine.joiner), + }, + modelType: "nemo_transducer", + }; + case "whisper": + // An empty language makes Whisper detect the spoken language. + return { + whisper: { + encoder: file(engine.encoder), + decoder: file(engine.decoder), + language: "", + task: "transcribe", + }, + }; + case "sense_voice": + return { + senseVoice: { model: file(engine.model), language: "", useInverseTextNormalization: 1 }, + }; + case "dolphin": + return { dolphin: { model: file(engine.model) } }; + case "nemo_ctc": + return { nemoCtc: { model: file(engine.model) } }; + } } export interface SherpaOfflineRecognizerConfig { - model: SherpaOfflineRecognizerModel; + modelDir: string; + engine: SttEngine; numThreads?: number; provider?: "cpu"; debug?: 0 | 1; @@ -54,10 +83,9 @@ export class SherpaOfflineRecognizerEngine { component: "offline-recognizer", }); - assertFileExists(config.model.encoder, "offline encoder"); - assertFileExists(config.model.decoder, "offline decoder"); - assertFileExists(config.model.joiner, "offline joiner"); - assertFileExists(config.model.tokens, "tokens"); + for (const name of engineFiles(config.engine)) { + assertFileExists(path.join(config.modelDir, name), `${config.engine.family} model file`); + } const sherpa = loadSherpaOnnxNode(); @@ -67,13 +95,8 @@ export class SherpaOfflineRecognizerEngine { featureDim: config.featureDim ?? 80, }, modelConfig: { - transducer: { - encoder: config.model.encoder, - decoder: config.model.decoder, - joiner: config.model.joiner, - }, - tokens: config.model.tokens, - modelType: "nemo_transducer", + ...buildModelConfig(config.modelDir, config.engine), + tokens: path.join(config.modelDir, config.engine.tokens), numThreads: config.numThreads ?? 1, provider: config.provider ?? "cpu", debug: config.debug ?? 0, diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.test.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.test.ts index 06d4a5b41d..0ae51f86ee 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.test.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.test.ts @@ -3,6 +3,7 @@ import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import pino from "pino"; +import { engineFiles, getLocalTtsEngine, type LocalTtsModelId } from "./model-catalog.js"; import { SherpaOnnxTTS } from "./sherpa-tts.js"; const directories: string[] = []; @@ -11,17 +12,28 @@ afterEach(() => { rmSync(directory, { recursive: true, force: true }); }); -function createTts(numThreads?: number, beforeGenerate?: () => Promise) { +function createTts( + numThreads?: number, + beforeGenerate?: () => Promise, + modelId: LocalTtsModelId = "kokoro-en-v0_19", +) { const modelDir = mkdtempSync(join(tmpdir(), "paseo-tts-")); directories.push(modelDir); - for (const name of ["model.onnx", "voices.bin", "tokens.txt"]) - writeFileSync(join(modelDir, name), ""); - mkdirSync(join(modelDir, "espeak-ng-data")); + // Data directories have no extension; everything else is a model file. + for (const name of engineFiles(getLocalTtsEngine(modelId))) { + if (name.includes(".")) writeFileSync(join(modelDir, name), ""); + else mkdirSync(join(modelDir, name)); + } let nativeConfig: unknown; const requests: unknown[] = []; const released: boolean[] = []; const tts = new SherpaOnnxTTS( - { preset: "kokoro-en-v0_19", modelDir, numThreads }, + { + modelId, + modelDir, + engine: getLocalTtsEngine(modelId), + numThreads, + }, pino({ level: "silent" }), () => ({ OfflineTts: class { @@ -138,4 +150,31 @@ describe("SherpaOnnxTTS", () => { expect(nativeConfig).not.toHaveProperty("provider"); }, ); + + it("loads Piper voices through the vits engine", () => { + const { nativeConfig } = createTts(undefined, undefined, "piper-de-thorsten"); + expect(nativeConfig).toMatchObject({ + model: { + vits: { + model: expect.stringMatching(/de_DE-thorsten-medium\.onnx$/), + tokens: expect.stringMatching(/tokens\.txt$/), + dataDir: expect.stringMatching(/espeak-ng-data$/), + }, + }, + }); + expect(nativeConfig).not.toHaveProperty("model.kokoro"); + }); + + it("gives multi-language Kokoro its lexicons, dictionary and number rules", () => { + const { nativeConfig } = createTts(undefined, undefined, "kokoro-int8-multi-lang-v1_1"); + expect(nativeConfig).toMatchObject({ + model: { + kokoro: { + lexicon: expect.stringMatching(/lexicon-us-en\.txt,.*lexicon-zh\.txt$/), + dictDir: expect.stringMatching(/dict$/), + }, + }, + ruleFsts: expect.stringMatching(/date-zh\.fst,.*phone-zh\.fst,.*number-zh\.fst$/), + }); + }); }); diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts index a078f81f51..12c86b8777 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts @@ -1,16 +1,17 @@ import type pino from "pino"; import { Readable } from "node:stream"; import { existsSync } from "node:fs"; +import path from "node:path"; import type { SpeechStreamResult, TextToSpeechProvider } from "../../../speech-provider.js"; import { chunkBuffer, float32ToPcm16le } from "../../../audio.js"; import { loadSherpaOnnxNode } from "./sherpa-onnx-node-loader.js"; - -export type SherpaTtsPreset = "kokoro-en-v0_19"; +import { engineFiles, type LocalTtsModelId, type TtsEngine } from "./model-catalog.js"; export interface SherpaTtsConfig { - preset: SherpaTtsPreset; + modelId: LocalTtsModelId; modelDir: string; + engine: TtsEngine; speakerId?: number; speed?: number; lengthScale?: number; @@ -23,6 +24,53 @@ function assertFileExists(filePath: string, label: string): void { } } +// Each engine family takes its own block under sherpa's TTS model config. +function buildTtsModelConfig( + modelDir: string, + engine: TtsEngine, + lengthScale: number, +): Record { + const file = (name: string) => path.join(modelDir, name); + switch (engine.family) { + case "kokoro": + return { + kokoro: { + model: file(engine.model), + voices: file(engine.voices), + tokens: file(engine.tokens), + dataDir: file(engine.dataDir), + lexicon: (engine.lexicons ?? []).map(file).join(","), + dictDir: engine.dictDir ? file(engine.dictDir) : "", + lengthScale, + }, + }; + case "kitten": + return { + kitten: { + model: file(engine.model), + voices: file(engine.voices), + tokens: file(engine.tokens), + dataDir: file(engine.dataDir), + lengthScale, + }, + }; + case "vits": + return { + vits: { + model: file(engine.model), + tokens: file(engine.tokens), + dataDir: file(engine.dataDir), + lengthScale, + }, + }; + } +} + +function ruleFsts(modelDir: string, engine: TtsEngine): string { + if (engine.family !== "kokoro" || !engine.ruleFsts) return ""; + return engine.ruleFsts.map((name) => path.join(modelDir, name)).join(","); +} + interface SherpaOfflineTtsNative { sampleRate?: number; generateAsync: (args: { @@ -59,39 +107,28 @@ export class SherpaOnnxTTS implements TextToSpeechProvider { throw new Error("sherpa-onnx-node OfflineTts is unavailable"); } - const modelPath = `${config.modelDir}/model.onnx`; - const voicesPath = `${config.modelDir}/voices.bin`; - const tokensPath = `${config.modelDir}/tokens.txt`; - const dataDir = `${config.modelDir}/espeak-ng-data`; - - assertFileExists(modelPath, "TTS model"); - assertFileExists(voicesPath, "TTS voices"); - assertFileExists(tokensPath, "TTS tokens"); - assertFileExists(dataDir, "TTS espeak-ng dataDir"); + for (const name of engineFiles(config.engine)) { + assertFileExists(path.join(config.modelDir, name), `${config.engine.family} TTS file`); + } const modelConfig = { // The native parser reads these under model, despite the upstream JS typedef. numThreads: config.numThreads ?? 2, provider: "cpu", - kokoro: { - model: modelPath, - voices: voicesPath, - tokens: tokensPath, - dataDir, - lengthScale: config.lengthScale ?? 1.0, - }, + ...buildTtsModelConfig(config.modelDir, config.engine, config.lengthScale ?? 1.0), }; const offlineTtsConfig = { model: modelConfig, maxNumSentences: 1, + ruleFsts: ruleFsts(config.modelDir, config.engine), }; this.tts = new ( sherpa as unknown as { OfflineTts: new (config: unknown) => SherpaOfflineTtsNative } ).OfflineTts(offlineTtsConfig); this.logger.info( - { preset: config.preset, modelDir: config.modelDir }, + { modelId: config.modelId, modelDir: config.modelDir }, "Sherpa offline TTS initialized", ); } diff --git a/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts b/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts index ab2eefef55..43ab8af01a 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts @@ -7,6 +7,7 @@ import pino from "pino"; import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./model-downloader.js"; import { createDaemonTestContext } from "../../../../test-utils/index.js"; import { parsePcm16MonoWav, wordSimilarity } from "../../../../test-utils/dictation-e2e.js"; +import { getLocalSttEngine, getLocalTtsEngine } from "./model-catalog.js"; import { SherpaOnnxTTS } from "./sherpa-tts.js"; import { SherpaOfflineRecognizerEngine } from "./sherpa-offline-recognizer.js"; import { SherpaOnnxParakeetSTT } from "./sherpa-parakeet-stt.js"; @@ -192,7 +193,12 @@ downloadTest( const ttsText = "This is a voice note."; const ttsModelDir = getSherpaOnnxModelDir(modelsDir, "kokoro-en-v0_19"); const tts = new SherpaOnnxTTS( - { preset: "kokoro-en-v0_19", modelDir: ttsModelDir, speakerId: 0 }, + { + modelId: "kokoro-en-v0_19", + modelDir: ttsModelDir, + engine: getLocalTtsEngine("kokoro-en-v0_19"), + speakerId: 0, + }, logger, ); const { stream, format: ttsFormat } = await tts.synthesizeSpeech(ttsText); @@ -206,13 +212,8 @@ downloadTest( const sttModelDir = getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v2-int8"); const engine = new SherpaOfflineRecognizerEngine( { - model: { - kind: "nemo_transducer", - encoder: `${sttModelDir}/encoder.int8.onnx`, - decoder: `${sttModelDir}/decoder.int8.onnx`, - joiner: `${sttModelDir}/joiner.int8.onnx`, - tokens: `${sttModelDir}/tokens.txt`, - }, + modelDir: sttModelDir, + engine: getLocalSttEngine("parakeet-tdt-0.6b-v2-int8"), numThreads: 2, debug: 0, }, diff --git a/packages/server/src/server/speech/providers/local/worker-process.ts b/packages/server/src/server/speech/providers/local/worker-process.ts index 957ee6aaa2..2afa3d336a 100644 --- a/packages/server/src/server/speech/providers/local/worker-process.ts +++ b/packages/server/src/server/speech/providers/local/worker-process.ts @@ -3,6 +3,7 @@ import pino from "pino"; import type { StreamingTranscriptionSession } from "../../speech-provider.js"; import type { TurnDetectionSession } from "../../turn-detection-provider.js"; import { getLocalSpeechModelDir, type LocalSttModelId, type LocalTtsModelId } from "./models.js"; +import { getLocalSttEngine, getLocalTtsEngine } from "./sherpa/model-catalog.js"; import { SherpaOfflineRecognizerEngine } from "./sherpa/sherpa-offline-recognizer.js"; import { SherpaOnnxParakeetSTT } from "./sherpa/sherpa-parakeet-stt.js"; import { SherpaParakeetRealtimeTranscriptionSession } from "./sherpa/sherpa-parakeet-realtime-session.js"; @@ -85,13 +86,8 @@ function getSttEngine( const modelDir = getLocalSpeechModelDir(config.modelsDir, modelId); const created = new SherpaOfflineRecognizerEngine( { - model: { - kind: "nemo_transducer", - encoder: `${modelDir}/encoder.int8.onnx`, - decoder: `${modelDir}/decoder.int8.onnx`, - joiner: `${modelDir}/joiner.int8.onnx`, - tokens: `${modelDir}/tokens.txt`, - }, + modelDir, + engine: getLocalSttEngine(modelId), numThreads: 2, debug: 0, }, @@ -125,8 +121,9 @@ function getTtsProvider(config: LocalSpeechWorkerConfig): SherpaOnnxTTS { const modelDir = getLocalSpeechModelDir(config.modelsDir, ttsModelId(config)); const created = new SherpaOnnxTTS( { - preset: ttsModelId(config), + modelId: ttsModelId(config), modelDir, + engine: getLocalTtsEngine(ttsModelId(config)), speakerId: config.voiceTtsSpeakerId, speed: config.voiceTtsSpeed, }, diff --git a/packages/server/src/server/speech/providers/openai/config.test.ts b/packages/server/src/server/speech/providers/openai/config.test.ts index 1f3433a347..ce24ff68ef 100644 --- a/packages/server/src/server/speech/providers/openai/config.test.ts +++ b/packages/server/src/server/speech/providers/openai/config.test.ts @@ -224,6 +224,23 @@ describe("resolveOpenAiSpeechConfig", () => { expect(resolved?.tts?.voice).toBe("Aiden"); }); + test("lets STT_MODEL override the model of every feature on OpenAI", () => { + const persisted = PersistedConfigSchema.parse({ + features: { + dictation: { stt: { provider: "openai", model: "gpt-4o-mini-transcribe" } }, + voiceMode: { stt: { provider: "openai", model: "whisper-1" } }, + }, + }); + const env = { OPENAI_API_KEY: "sk-test", STT_MODEL: "gpt-4o-transcribe" } as NodeJS.ProcessEnv; + + const resolved = resolveOpenAiSpeechConfig({ env, persisted, providers: ALL_OPENAI }); + + expect(resolved?.sttModels).toEqual({ + dictation: "gpt-4o-transcribe", + voiceMode: "gpt-4o-transcribe", + }); + }); + test("accepts OpenAI's own gpt-4o-mini-tts model", () => { const persisted = PersistedConfigSchema.parse({ providers: { openai: { tts: { apiKey: "tts-key" } } }, diff --git a/packages/server/src/server/speech/providers/openai/config.ts b/packages/server/src/server/speech/providers/openai/config.ts index efb25bffaf..4e89e40ba1 100644 --- a/packages/server/src/server/speech/providers/openai/config.ts +++ b/packages/server/src/server/speech/providers/openai/config.ts @@ -7,8 +7,13 @@ import type { TTSConfig } from "./tts.js"; export const DEFAULT_OPENAI_TTS_MODEL = "tts-1"; +/** The features that transcribe through OpenAI; each picks its own model. */ +export type OpenAiSttFeature = "dictation" | "voiceMode"; + export interface OpenAiSpeechProviderConfig { - stt?: Partial & { apiKey?: string }; + /** The transcription endpoint the features share. */ + stt?: Omit, "model"> & { apiKey?: string }; + sttModels?: Partial>; tts?: Partial & { apiKey?: string }; } @@ -37,7 +42,8 @@ const OpenAiEndpointKeysSchema = z.object({ const OpenAiSttOptionsSchema = z.object({ sttConfidenceThreshold: OptionalFiniteNumberSchema, - sttModel: OptionalTrimmedStringSchema, + dictationSttModel: OptionalTrimmedStringSchema, + voiceModeSttModel: OptionalTrimmedStringSchema, }); // Voice and model are free-form so a self-hosted server can use its own ids; the @@ -85,10 +91,14 @@ function buildOpenAiSttInput(params: { env.STT_CONFIDENCE_THRESHOLD, persisted.features?.dictation?.stt?.confidenceThreshold, ]), - sttModel: firstDefined([ + // STT_MODEL is host-wide: it overrides the model of every feature that uses OpenAI. + dictationSttModel: firstDefined([ + env.STT_MODEL, + pickIfOpenAi(providers.dictationStt, persisted.features?.dictation?.stt?.model), + ]), + voiceModeSttModel: firstDefined([ env.STT_MODEL, pickIfOpenAi(providers.voiceStt, persisted.features?.voiceMode?.stt?.model), - pickIfOpenAi(providers.dictationStt, persisted.features?.dictation?.stt?.model), ]), }; } @@ -163,7 +173,7 @@ export function resolveOpenAiSpeechConfig(params: { } return { - ...(keys.sttApiKey ? { stt: buildSttConfig(keys.sttApiKey, keys.sttBaseUrl, input) } : {}), + ...(keys.sttApiKey ? buildSttConfig(keys.sttApiKey, keys.sttBaseUrl, input) : {}), ...(keys.ttsApiKey ? { tts: buildTtsConfig(keys.ttsApiKey, keys.ttsBaseUrl, input) } : {}), }; } @@ -172,15 +182,20 @@ function buildSttConfig( apiKey: string, baseUrl: string | undefined, input: Record, -): OpenAiSpeechProviderConfig["stt"] { +): Pick { const options = OpenAiSttOptionsSchema.parse(input); return { - apiKey, - ...(baseUrl ? { baseUrl } : {}), - ...(options.sttConfidenceThreshold !== undefined - ? { confidenceThreshold: options.sttConfidenceThreshold } - : {}), - ...(options.sttModel ? { model: options.sttModel } : {}), + stt: { + apiKey, + ...(baseUrl ? { baseUrl } : {}), + ...(options.sttConfidenceThreshold !== undefined + ? { confidenceThreshold: options.sttConfidenceThreshold } + : {}), + }, + sttModels: { + ...(options.dictationSttModel ? { dictation: options.dictationSttModel } : {}), + ...(options.voiceModeSttModel ? { voiceMode: options.voiceModeSttModel } : {}), + }, }; } diff --git a/packages/server/src/server/speech/providers/openai/runtime.test.ts b/packages/server/src/server/speech/providers/openai/runtime.test.ts index c947d86600..34aa35d758 100644 --- a/packages/server/src/server/speech/providers/openai/runtime.test.ts +++ b/packages/server/src/server/speech/providers/openai/runtime.test.ts @@ -1,6 +1,11 @@ +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; import pino from "pino"; -import { describe, expect, test } from "vitest"; +import { afterEach, describe, expect, test } from "vitest"; +import { PersistedConfigSchema } from "../../../persisted-config.js"; +import type { SpeechToTextProvider } from "../../speech-provider.js"; +import { resolveOpenAiSpeechConfig } from "./config.js"; import { initializeOpenAiSpeechServices } from "./runtime.js"; import { OpenAISTT } from "./stt.js"; import { OpenAITTS } from "./tts.js"; @@ -32,3 +37,88 @@ describe("initializeOpenAiSpeechServices", () => { expect(services.ttsService).toBeInstanceOf(OpenAITTS); }); }); + +// An OpenAI-compatible transcription endpoint that records the model each request asks for. +async function startTranscriptionServer(): Promise<{ + server: Server; + baseUrl: string; + models: string[]; +}> { + const models: string[] = []; + const server = createServer((request, response) => { + let body = ""; + request.on("data", (chunk: Buffer) => { + body += chunk.toString("latin1"); + }); + request.on("end", () => { + const model = /name="model"\r\n\r\n([^\r]*)/.exec(body)?.[1]; + models.push(model ?? ""); + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ text: "hello" })); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as AddressInfo; + return { server, baseUrl: `http://127.0.0.1:${port}/v1`, models }; +} + +async function transcribeOnce(stt: SpeechToTextProvider | null): Promise { + if (!stt) throw new Error("STT service was not created"); + const session = stt.createSession({ logger: pino({ level: "silent" }), language: "en" }); + const transcript = new Promise((resolve, reject) => { + session.on("transcript", (event) => { + if (event.isFinal) resolve(); + }); + session.on("error", (error) => + reject(error instanceof Error ? error : new Error(String(error))), + ); + }); + await session.connect(); + session.appendPcm16(Buffer.alloc(3200)); + session.commit(); + await transcript; + session.close(); +} + +describe("OpenAI speech-to-text models", () => { + let server: Server | null = null; + + afterEach(async () => { + await new Promise((resolve) => (server ? server.close(() => resolve()) : resolve())); + server = null; + }); + + test("dictation and voice mode each transcribe with the model chosen for them", async () => { + const endpoint = await startTranscriptionServer(); + server = endpoint.server; + const persisted = PersistedConfigSchema.parse({ + features: { + dictation: { stt: { provider: "openai", model: "gpt-4o-mini-transcribe" } }, + voiceMode: { enabled: true, stt: { provider: "openai", model: "whisper-1" } }, + }, + providers: { openai: { apiKey: "sk-test", baseUrl: endpoint.baseUrl } }, + }); + const providers = { + dictationStt: { provider: "openai", explicit: true }, + voiceTurnDetection: { provider: "local", explicit: false, enabled: false }, + voiceStt: { provider: "openai", explicit: true }, + voiceTts: { provider: "local", explicit: false }, + } as const; + + const services = initializeOpenAiSpeechServices({ + providers, + openaiConfig: resolveOpenAiSpeechConfig({ env: {}, persisted, providers }), + existing: { + turnDetectionService: null, + sttService: null, + ttsService: null, + dictationSttService: null, + }, + logger: pino({ level: "silent" }), + }); + await transcribeOnce(services.dictationSttService); + await transcribeOnce(services.sttService); + + expect(endpoint.models).toEqual(["gpt-4o-mini-transcribe", "whisper-1"]); + }); +}); diff --git a/packages/server/src/server/speech/providers/openai/runtime.ts b/packages/server/src/server/speech/providers/openai/runtime.ts index a1231ddbe6..4e967e9cb4 100644 --- a/packages/server/src/server/speech/providers/openai/runtime.ts +++ b/packages/server/src/server/speech/providers/openai/runtime.ts @@ -3,7 +3,11 @@ import type { Logger } from "pino"; import type { SpeechToTextProvider, TextToSpeechProvider } from "../../speech-provider.js"; import type { RequestedSpeechProviders } from "../../speech-types.js"; import type { TurnDetectionProvider } from "../../turn-detection-provider.js"; -import { DEFAULT_OPENAI_TTS_MODEL, type OpenAiSpeechProviderConfig } from "./config.js"; +import { + DEFAULT_OPENAI_TTS_MODEL, + type OpenAiSpeechProviderConfig, + type OpenAiSttFeature, +} from "./config.js"; import { OpenAISTT } from "./stt.js"; import { OpenAITTS } from "./tts.js"; @@ -94,13 +98,18 @@ export function validateOpenAiCredentialRequirements(params: { } } -function createOpenAiStt( - apiKey: string, - openaiConfig: OpenAiSpeechProviderConfig | undefined, - logger: Logger, -): SpeechToTextProvider { - const { apiKey: _sttApiKey, ...sttConfig } = openaiConfig?.stt ?? {}; - return new OpenAISTT({ apiKey, ...sttConfig }, logger); +function createOpenAiStt(params: { + apiKey: string; + openaiConfig: OpenAiSpeechProviderConfig | undefined; + feature: OpenAiSttFeature; + logger: Logger; +}): SpeechToTextProvider { + const { apiKey: _sttApiKey, ...sttConfig } = params.openaiConfig?.stt ?? {}; + const model = params.openaiConfig?.sttModels?.[params.feature]; + return new OpenAISTT( + { apiKey: params.apiKey, ...sttConfig, ...(model ? { model } : {}) }, + params.logger, + ); } function createOpenAiTts( @@ -154,7 +163,12 @@ export function initializeOpenAiSpeechServices(params: { logger.info("OpenAI speech provider initialized"); if (needsOpenAiStt && openAiCredentials.openaiSttApiKey) { - sttService = createOpenAiStt(openAiCredentials.openaiSttApiKey, openaiConfig, logger); + sttService = createOpenAiStt({ + apiKey: openAiCredentials.openaiSttApiKey, + openaiConfig, + feature: "voiceMode", + logger, + }); } if (needsOpenAiTts && openAiCredentials.openaiTtsApiKey) { @@ -162,11 +176,12 @@ export function initializeOpenAiSpeechServices(params: { } if (needsOpenAiDictation && openAiCredentials.openaiDictationApiKey) { - dictationSttService = createOpenAiStt( - openAiCredentials.openaiDictationApiKey, + dictationSttService = createOpenAiStt({ + apiKey: openAiCredentials.openaiDictationApiKey, openaiConfig, + feature: "dictation", logger, - ); + }); } } else if (needsAnyOpenAi) { // validateOpenAiCredentialRequirements already warned about missing credentials diff --git a/packages/server/src/server/speech/speech-config-resolver.test.ts b/packages/server/src/server/speech/speech-config-resolver.test.ts index 0ee9df888a..1300dadf7e 100644 --- a/packages/server/src/server/speech/speech-config-resolver.test.ts +++ b/packages/server/src/server/speech/speech-config-resolver.test.ts @@ -6,7 +6,7 @@ import { PersistedConfigSchema } from "../persisted-config.js"; import { resolveSpeechConfig } from "./speech-config-resolver.js"; describe("resolveSpeechConfig", () => { - test("resolves local-first defaults without env overrides", () => { + test("resolves local-first defaults with voice mode off until turned on", () => { const paseoHome = "/tmp/paseo-home"; const persisted = PersistedConfigSchema.parse({}); const env = {} as NodeJS.ProcessEnv; @@ -26,17 +26,17 @@ describe("resolveSpeechConfig", () => { expect(result.speech.providers.voiceTurnDetection).toEqual({ provider: "local", explicit: false, - enabled: true, + enabled: false, }); expect(result.speech.providers.voiceStt).toEqual({ provider: "local", explicit: false, - enabled: true, + enabled: false, }); expect(result.speech.providers.voiceTts).toEqual({ provider: "local", explicit: false, - enabled: true, + enabled: false, }); expect(result.speech.local).toEqual({ modelsDir: path.join(paseoHome, "models", "local-speech"), @@ -61,6 +61,7 @@ describe("resolveSpeechConfig", () => { const persisted = PersistedConfigSchema.parse({ features: { voiceMode: { + enabled: true, turnDetection: { provider: "local" }, stt: { provider: "openai", model: "gpt-4o-transcribe" }, }, @@ -131,7 +132,7 @@ describe("resolveSpeechConfig", () => { }); expect(result.openai?.stt?.apiKey).toBe("persisted-key"); expect(result.openai?.tts?.apiKey).toBe("persisted-key"); - expect(result.openai?.stt?.model).toBe("gpt-4o-transcribe"); + expect(result.openai?.sttModels).toEqual({ voiceMode: "gpt-4o-transcribe" }); }); test("resolves STT language from env, settings, and voice-to-dictation fallback", () => { diff --git a/packages/server/src/server/speech/speech-config-resolver.ts b/packages/server/src/server/speech/speech-config-resolver.ts index 95c78707a5..ae0c299aa1 100644 --- a/packages/server/src/server/speech/speech-config-resolver.ts +++ b/packages/server/src/server/speech/speech-config-resolver.ts @@ -43,8 +43,13 @@ const RequestedSpeechProvidersSchema = z.object({ voiceTts: OptionalSpeechProviderSchema.default("local"), }); -function resolveOptionalBooleanFlag(value: unknown): boolean { - return OptionalBooleanFlagSchema.parse(value) ?? true; +// Dictation is on unless turned off. Voice mode is opt-in: it downloads its own models and +// runs a hidden agent, so a host only pays for it once someone turns it on. +export const DICTATION_ENABLED_BY_DEFAULT = true; +export const VOICE_MODE_ENABLED_BY_DEFAULT = false; + +function resolveOptionalBooleanFlag(value: unknown, fallback: boolean): boolean { + return OptionalBooleanFlagSchema.parse(value) ?? fallback; } interface FeatureProviderInputs { @@ -70,6 +75,7 @@ function buildFeatureProviderInputs(params: { params.env.PASEO_VOICE_MODE_ENABLED, params.persisted.features?.voiceMode?.enabled, ]), + VOICE_MODE_ENABLED_BY_DEFAULT, ); return { dictationStt: { @@ -82,6 +88,7 @@ function buildFeatureProviderInputs(params: { params.env.PASEO_DICTATION_ENABLED, params.persisted.features?.dictation?.enabled, ]), + DICTATION_ENABLED_BY_DEFAULT, ), }, voiceTurnDetection: { @@ -173,6 +180,7 @@ export function resolveSpeechConfig(params: { openai, speech: { providers, + localModelsDir: local.modelsDir, sttLanguages: local.sttLanguages, ...(local.local ? { local: local.local } : {}), }, diff --git a/packages/server/src/server/speech/speech-runtime.test.ts b/packages/server/src/server/speech/speech-runtime.test.ts index c9c21b3e89..cd3580e64a 100644 --- a/packages/server/src/server/speech/speech-runtime.test.ts +++ b/packages/server/src/server/speech/speech-runtime.test.ts @@ -2,13 +2,14 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import pino from "pino"; import type { PaseoSpeechConfig } from "../bootstrap.js"; +import type { LocalSpeechModelId } from "./providers/local/models.js"; import type { InitializedLocalSpeech } from "./providers/local/runtime.js"; import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech-provider.js"; import type { TurnDetectionProvider } from "./turn-detection-provider.js"; -import { createSpeechService } from "./speech-runtime.js"; +import { createSpeechService, LocalSpeechModelInUseError } from "./speech-runtime.js"; -const { ensureLocalSpeechModelsMock, initializeLocalSpeechServicesMock } = vi.hoisted(() => ({ - ensureLocalSpeechModelsMock: vi.fn(async () => ({})), +const { ensureSherpaOnnxModelMock, initializeLocalSpeechServicesMock } = vi.hoisted(() => ({ + ensureSherpaOnnxModelMock: vi.fn(async (_options: { signal?: AbortSignal }) => ""), initializeLocalSpeechServicesMock: vi.fn<(args: unknown) => Promise>(), })); @@ -34,9 +35,10 @@ vi.mock("./providers/openai/runtime.js", () => ({ validateOpenAiCredentialRequirements: () => {}, })); -vi.mock("./providers/local/models.js", () => ({ - ensureLocalSpeechModels: ensureLocalSpeechModelsMock, - listMissingLocalSpeechModels: vi.fn(async ({ modelIds }: { modelIds: string[] }) => modelIds), +vi.mock("./providers/local/sherpa/model-downloader.js", () => ({ + ensureSherpaOnnxModel: ensureSherpaOnnxModelMock, + isSherpaOnnxModelInstalled: vi.fn(async () => false), + deleteSherpaOnnxModel: vi.fn(async () => {}), })); function createStubStt(id: string): SpeechToTextProvider { @@ -75,6 +77,7 @@ function rejectWhenAborted(signal?: AbortSignal): Promise { function createSpeechConfig(providers: PaseoSpeechConfig["providers"]): PaseoSpeechConfig { return { providers, + localModelsDir: "/tmp/missing-local-speech-models", sttLanguages: { dictation: "en", voice: "en", @@ -84,8 +87,8 @@ function createSpeechConfig(providers: PaseoSpeechConfig["providers"]): PaseoSpe describe("createSpeechService readiness", () => { beforeEach(() => { - ensureLocalSpeechModelsMock.mockReset(); - ensureLocalSpeechModelsMock.mockResolvedValue({}); + ensureSherpaOnnxModelMock.mockReset(); + ensureSherpaOnnxModelMock.mockResolvedValue(""); initializeLocalSpeechServicesMock.mockReset(); }); @@ -169,7 +172,7 @@ describe("createSpeechService readiness", () => { it("aborts and joins an in-flight model download when stopped", async () => { let downloadSignal: AbortSignal | undefined; - ensureLocalSpeechModelsMock.mockImplementation(({ signal }: { signal?: AbortSignal }) => { + ensureSherpaOnnxModelMock.mockImplementation(({ signal }: { signal?: AbortSignal }) => { downloadSignal = signal; return rejectWhenAborted(signal); }); @@ -201,10 +204,104 @@ describe("createSpeechService readiness", () => { }); runtime.start(); await runtime.ready; - expect(ensureLocalSpeechModelsMock).toHaveBeenCalledOnce(); + expect(ensureSherpaOnnxModelMock).toHaveBeenCalledOnce(); await runtime.stop(); expect(downloadSignal?.aborted).toBe(true); }); }); + +describe("createSpeechService settings changes", () => { + const dictationOnly = createSpeechConfig({ + dictationStt: { provider: "local", enabled: true, explicit: true }, + voiceTurnDetection: { provider: "local", enabled: false, explicit: true }, + voiceStt: { provider: "local", enabled: false, explicit: true }, + voiceTts: { provider: "local", enabled: false, explicit: true }, + }); + const nothingEnabled = createSpeechConfig({ + dictationStt: { provider: "local", enabled: false, explicit: true }, + voiceTurnDetection: { provider: "local", enabled: false, explicit: true }, + voiceStt: { provider: "local", enabled: false, explicit: true }, + voiceTts: { provider: "local", enabled: false, explicit: true }, + }); + + function localSpeech(input: { + dictationStt: SpeechToTextProvider | null; + defaultModelIds: LocalSpeechModelId[]; + }): InitializedLocalSpeech { + return { + turnDetectionService: null, + sttService: null, + ttsService: null, + dictationSttService: input.dictationStt, + localVoiceTtsProvider: null, + localModelConfig: { + modelsDir: "/tmp/missing-local-speech-models", + defaultModelIds: input.defaultModelIds, + }, + availability: { configured: true, modelsDir: "/tmp/missing-local-speech-models" }, + cleanup: () => {}, + }; + } + + beforeEach(() => { + ensureSherpaOnnxModelMock.mockReset(); + ensureSherpaOnnxModelMock.mockResolvedValue(""); + initializeLocalSpeechServicesMock.mockReset(); + }); + + it("applies new providers without a restart", async () => { + initializeLocalSpeechServicesMock.mockResolvedValue( + localSpeech({ dictationStt: createStubStt("dictation-local"), defaultModelIds: [] }), + ); + const runtime = createSpeechService({ + logger: pino({ level: "silent" }), + speechConfig: dictationOnly, + }); + runtime.start(); + await runtime.ready; + expect(runtime.getReadiness().dictation.available).toBe(true); + + const disabled = new Promise((resolve) => { + runtime.onReadinessChange((snapshot) => { + if (snapshot.dictation.reasonCode === "disabled") resolve(); + }); + }); + initializeLocalSpeechServicesMock.mockResolvedValue( + localSpeech({ dictationStt: null, defaultModelIds: [] }), + ); + runtime.reconfigure({ speechConfig: nothingEnabled }); + await disabled; + + expect(runtime.resolveDictationStt()).toBeNull(); + await runtime.stop(); + }); + + it("refuses to delete a model an enabled feature uses", async () => { + initializeLocalSpeechServicesMock.mockResolvedValue( + localSpeech({ + dictationStt: createStubStt("dictation-local"), + defaultModelIds: ["parakeet-tdt-0.6b-v2-int8"], + }), + ); + ensureSherpaOnnxModelMock.mockImplementation(({ signal }) => rejectWhenAborted(signal)); + const runtime = createSpeechService({ + logger: pino({ level: "silent" }), + speechConfig: dictationOnly, + }); + runtime.start(); + await runtime.ready; + + const models = await runtime.listLocalModels(); + expect(models.filter((model) => model.inUse).map((model) => model.spec.id)).toEqual([ + "parakeet-tdt-0.6b-v2-int8", + ]); + await expect(runtime.deleteLocalModel("parakeet-tdt-0.6b-v2-int8")).rejects.toBeInstanceOf( + LocalSpeechModelInUseError, + ); + await expect(runtime.deleteLocalModel("kokoro-int8-en-v0_19")).resolves.toBeUndefined(); + + await runtime.stop(); + }); +}); diff --git a/packages/server/src/server/speech/speech-runtime.ts b/packages/server/src/server/speech/speech-runtime.ts index 932f87c077..270b13b03c 100644 --- a/packages/server/src/server/speech/speech-runtime.ts +++ b/packages/server/src/server/speech/speech-runtime.ts @@ -2,7 +2,10 @@ import type { Logger } from "pino"; import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js"; import type { LocalSpeechModelId } from "./providers/local/config.js"; -import { ensureLocalSpeechModels, listMissingLocalSpeechModels } from "./providers/local/models.js"; +import { + LocalSpeechModelLibrary, + type LocalSpeechModelEntry, +} from "./providers/local/model-library.js"; import { initializeLocalSpeechServices } from "./providers/local/runtime.js"; import { getOpenAiSpeechAvailability, @@ -92,7 +95,7 @@ function buildRealtimeVoiceReadiness(params: { enabled: false, available: false, reasonCode: "disabled", - message: "Realtime voice is disabled in daemon config.", + message: "Voice mode is off. Turn it on in Settings > Voice.", retryable: false, missingModelIds: [], }; @@ -147,7 +150,7 @@ function buildDictationReadiness(params: { enabled: false, available: false, reasonCode: "disabled", - message: "Dictation is disabled in daemon config.", + message: "Dictation is off. Turn it on in Settings > Voice.", retryable: false, missingModelIds: [], }; @@ -292,6 +295,23 @@ function resolveEffectiveProviderIds(params: { }; } +export interface LocalSpeechModelStatus extends LocalSpeechModelEntry { + // Selected by an enabled feature, so the runtime keeps it installed. + inUse: boolean; +} + +export class LocalSpeechModelInUseError extends Error { + constructor(public readonly modelId: LocalSpeechModelId) { + super(`${modelId} is selected for dictation or voice mode. Choose another model first.`); + this.name = "LocalSpeechModelInUseError"; + } +} + +export interface SpeechServiceConfig { + openaiConfig?: PaseoOpenAIConfig; + speechConfig?: PaseoSpeechConfig; +} + export interface SpeechService { resolveStt: () => SpeechToTextProvider | null; resolveSttLanguage: () => string; @@ -301,37 +321,49 @@ export interface SpeechService { resolveDictationSttLanguage: () => string; getReadiness: () => SpeechReadinessSnapshot; onReadinessChange: (listener: (snapshot: SpeechReadinessSnapshot) => void) => () => void; + // Applies new provider/model settings without restarting the daemon. + reconfigure: (config: SpeechServiceConfig) => void; + listLocalModels: () => Promise; + downloadLocalModel: (modelId: LocalSpeechModelId) => void; + deleteLocalModel: (modelId: LocalSpeechModelId) => Promise; start: () => void; stop: () => Promise; ready: Promise; } -export function createSpeechService(params: { - logger: Logger; - openaiConfig?: PaseoOpenAIConfig; - speechConfig?: PaseoSpeechConfig; -}): SpeechService { +export function createSpeechService( + params: SpeechServiceConfig & { + logger: Logger; + }, +): SpeechService { const logger = params.logger.child({ module: "speech-runtime" }); - const speechConfig = params.speechConfig ?? null; - const openaiConfig = params.openaiConfig; - const providers = resolveRequestedSpeechProviders(speechConfig); - const requestedProviders = describeRequestedProviders(providers); + let speechConfig = params.speechConfig ?? null; + let openaiConfig = params.openaiConfig; + let providers = resolveRequestedSpeechProviders(speechConfig); + let requestedProviders = describeRequestedProviders(providers); + // The models directory is fixed for the daemon's lifetime; settings changes only pick + // which of its models are used. + const modelLibrary = speechConfig + ? new LocalSpeechModelLibrary(speechConfig.localModelsDir, logger) + : null; - validateOpenAiCredentialRequirements({ - providers, - openaiConfig, - logger, - }); - - logger.info( - { - requestedProviders, - availability: { - openai: getOpenAiSpeechAvailability(openaiConfig), + const logReconciliationStart = (): void => { + validateOpenAiCredentialRequirements({ + providers, + openaiConfig, + logger, + }); + logger.info( + { + requestedProviders, + availability: { + openai: getOpenAiSpeechAvailability(openaiConfig), + }, }, - }, - "Speech provider reconciliation started", - ); + "Speech provider reconciliation started", + ); + }; + logReconciliationStart(); let sttService: SpeechToTextProvider | null = null; let ttsService: TextToSpeechProvider | null = null; @@ -346,12 +378,11 @@ export function createSpeechService(params: { let missingLocalModelIds: LocalSpeechModelId[] = []; let backgroundDownloadInProgress = false; - let backgroundDownloadAbortController: AbortController | null = null; let backgroundDownloadPromise: Promise | null = null; let backgroundDownloadError: string | null = null; let stopped = false; let monitorTimeout: ReturnType | null = null; - let reconcileInFlight: Promise | null = null; + let reconcileQueue: Promise = Promise.resolve(); const readinessListeners = new Set<(snapshot: SpeechReadinessSnapshot) => void>(); let lastReadinessFingerprint: string | null = null; let lastPublishedReadinessSnapshot: SpeechReadinessSnapshot | null = null; @@ -445,12 +476,10 @@ export function createSpeechService(params: { }; const refreshMissingLocalModels = async (): Promise => { - missingLocalModelIds = localModelConfig - ? await listMissingLocalSpeechModels({ - modelsDir: localModelConfig.modelsDir, - modelIds: localModelConfig.defaultModelIds, - }) - : []; + missingLocalModelIds = + localModelConfig && modelLibrary + ? await modelLibrary.listMissing(localModelConfig.defaultModelIds) + : []; }; const reconcileServices = async (): Promise => { @@ -520,16 +549,12 @@ export function createSpeechService(params: { } }; + // Reconciles run one after another, and each starts after it was requested, so a + // reconfigure that lands mid-reconcile is applied instead of joining the stale run. const runReconcile = async (): Promise => { - if (reconcileInFlight) { - await reconcileInFlight; - publishReadinessIfChanged(); - return; - } - reconcileInFlight = reconcileServices().finally(() => { - reconcileInFlight = null; - }); - await reconcileInFlight; + const next = reconcileQueue.catch(() => undefined).then(reconcileServices); + reconcileQueue = next; + await next; publishReadinessIfChanged(); }; @@ -544,12 +569,12 @@ export function createSpeechService(params: { }; const startBackgroundDownload = (): void => { - if (stopped || backgroundDownloadInProgress) { + if (stopped || backgroundDownloadInProgress || !modelLibrary) { return; } - const modelsDir = localModelConfig?.modelsDir ?? null; + const modelsDir = modelLibrary.getModelsDir(); const modelIds = [...missingLocalModelIds]; - if (!modelsDir || modelIds.length === 0) { + if (modelIds.length === 0) { return; } @@ -565,16 +590,9 @@ export function createSpeechService(params: { "Starting background download for missing local speech models", ); - const abortController = new AbortController(); - backgroundDownloadAbortController = abortController; backgroundDownloadPromise = (async () => { try { - await ensureLocalSpeechModels({ - modelsDir, - modelIds, - logger, - signal: abortController.signal, - }); + await Promise.all(modelIds.map((modelId) => modelLibrary.download(modelId))); await runReconcile(); backgroundDownloadError = null; } catch (error) { @@ -588,7 +606,6 @@ export function createSpeechService(params: { "Background local speech model download failed", ); } finally { - backgroundDownloadAbortController = null; backgroundDownloadInProgress = false; await refreshMissingLocalModels().catch((error) => { logger.warn({ err: error }, "Failed to refresh local speech model status after download"); @@ -630,6 +647,62 @@ export function createSpeechService(params: { } }; + const reconcileAndFetchMissingModels = async (): Promise => { + await runReconcile(); + const snapshot = computeReadinessSnapshot(); + if (snapshot.voiceFeature.enabled && !snapshot.voiceFeature.available) { + if (missingLocalModelIds.length > 0) { + startBackgroundDownload(); + } + scheduleMonitor(); + } + }; + + const reconfigure = (next: SpeechServiceConfig): void => { + speechConfig = next.speechConfig ?? null; + openaiConfig = next.openaiConfig; + providers = resolveRequestedSpeechProviders(speechConfig); + requestedProviders = describeRequestedProviders(providers); + // A failure for the previous selection must not block downloading the new one. + backgroundDownloadError = null; + logReconciliationStart(); + if (!started || stopped) { + return; + } + void reconcileAndFetchMissingModels().catch((error) => { + logger.error({ err: error }, "Speech runtime failed to apply new configuration"); + }); + }; + + const requireModelLibrary = (): LocalSpeechModelLibrary => { + if (!modelLibrary) { + throw new Error("Local speech models are unavailable on this daemon"); + } + return modelLibrary; + }; + + const listLocalModels = async (): Promise => { + const inUse = new Set(localModelConfig?.defaultModelIds ?? []); + const entries = await requireModelLibrary().list(); + return entries.map(({ spec, state }) => ({ spec, state, inUse: inUse.has(spec.id) })); + }; + + const downloadLocalModel = (modelId: LocalSpeechModelId): void => { + void requireModelLibrary() + .download(modelId) + .then(() => runMonitorTick()) + .catch((error) => { + logger.warn({ err: error, modelId }, "Local speech model download failed"); + }); + }; + + const deleteLocalModel = async (modelId: LocalSpeechModelId): Promise => { + if (localModelConfig?.defaultModelIds.includes(modelId)) { + throw new LocalSpeechModelInUseError(modelId); + } + await requireModelLibrary().delete(modelId); + }; + const start = (): void => { if (started || stopped) { return; @@ -637,14 +710,7 @@ export function createSpeechService(params: { started = true; void (async () => { try { - await runReconcile(); - const snapshot = computeReadinessSnapshot(); - if (snapshot.voiceFeature.enabled && !snapshot.voiceFeature.available) { - if (missingLocalModelIds.length > 0) { - startBackgroundDownload(); - } - scheduleMonitor(); - } + await reconcileAndFetchMissingModels(); if (!readySettled) { readySettled = true; resolveReady(); @@ -666,7 +732,7 @@ export function createSpeechService(params: { monitorTimeout = null; } localCleanup(); - backgroundDownloadAbortController?.abort(); + await modelLibrary?.stop(); await backgroundDownloadPromise?.catch(() => undefined); backgroundDownloadPromise = null; }; @@ -680,6 +746,10 @@ export function createSpeechService(params: { resolveDictationSttLanguage: () => speechConfig?.sttLanguages?.dictation ?? "en", getReadiness: () => lastPublishedReadinessSnapshot ?? computeReadinessSnapshot(), onReadinessChange: subscribeSpeechReadiness, + reconfigure, + listLocalModels, + downloadLocalModel, + deleteLocalModel, start, stop, ready, diff --git a/packages/server/src/server/speech/speech-settings.ts b/packages/server/src/server/speech/speech-settings.ts new file mode 100644 index 0000000000..a04a8bed59 --- /dev/null +++ b/packages/server/src/server/speech/speech-settings.ts @@ -0,0 +1,211 @@ +import type { SpeechSettings, SpeechSettingsPatch } from "@getpaseo/protocol/speech/rpc-schemas"; + +import type { PersistedConfig } from "../persisted-config.js"; +import { + DICTATION_ENABLED_BY_DEFAULT, + VOICE_MODE_ENABLED_BY_DEFAULT, + resolveSpeechConfig, +} from "./speech-config-resolver.js"; +import type { SpeechService } from "./speech-runtime.js"; +import { LocalSttModelIdSchema, LocalTtsModelIdSchema } from "./providers/local/models.js"; +import { SpeechProviderIdSchema, type SpeechProviderId } from "./speech-types.js"; + +// The `speech` section of the mutable daemon config is a view over three persisted +// blocks: features.dictation, features.voiceMode and providers.openai. This module owns +// both directions of that mapping. + +type PersistedFeatures = NonNullable; +type PersistedDictation = NonNullable; +type PersistedVoiceMode = NonNullable; +type PersistedStt = NonNullable; +type PersistedTts = NonNullable; +type PersistedOpenAi = NonNullable["openai"]>; +type SttPatch = NonNullable["stt"]>; +type TtsPatch = NonNullable["tts"]>; + +const DEFAULT_SPEECH_PROVIDER = "local"; + +// The daemon config store derives the speech view through this owner (the OpenAI key is +// write-only) and calls apply when the persisted speech blocks change. +export interface SpeechSettingsOwner { + read(persisted: PersistedConfig): SpeechSettings; + apply(persisted: PersistedConfig): void; +} + +export function createSpeechSettingsOwner(params: { + paseoHome: string; + env: NodeJS.ProcessEnv; + speechService: Pick; +}): SpeechSettingsOwner { + const { paseoHome, env, speechService } = params; + return { + read: (persisted) => readSpeechSettings(persisted, env), + apply: (persisted) => { + const resolved = resolveSpeechConfig({ paseoHome, env, persisted }); + speechService.reconfigure({ openaiConfig: resolved.openai, speechConfig: resolved.speech }); + }, + }; +} + +export function readSpeechSettings( + persisted: PersistedConfig, + env: NodeJS.ProcessEnv, +): SpeechSettings { + const dictation = persisted.features?.dictation; + const voiceMode = persisted.features?.voiceMode; + return { + dictation: { + enabled: dictation?.enabled ?? DICTATION_ENABLED_BY_DEFAULT, + stt: readStt(dictation?.stt), + }, + voiceMode: { + enabled: voiceMode?.enabled ?? VOICE_MODE_ENABLED_BY_DEFAULT, + stt: readStt(voiceMode?.stt), + tts: readTts(voiceMode?.tts), + }, + openai: readOpenAi(persisted.providers?.openai, env), + }; +} + +function readTts(tts: PersistedTts | undefined): SpeechSettings["voiceMode"]["tts"] { + return { + provider: tts?.provider ?? DEFAULT_SPEECH_PROVIDER, + ...(tts?.model ? { model: tts.model } : {}), + ...(tts?.voice ? { voice: tts.voice } : {}), + }; +} + +function readOpenAi( + openai: PersistedOpenAi | undefined, + env: NodeJS.ProcessEnv, +): SpeechSettings["openai"] { + const keys = [ + openai?.apiKey, + openai?.stt?.apiKey, + openai?.tts?.apiKey, + env.OPENAI_API_KEY, + env.OPENAI_STT_API_KEY, + env.OPENAI_TTS_API_KEY, + ]; + return { + apiKeyConfigured: keys.some((key) => Boolean(key?.trim())), + ...(openai?.baseUrl ? { baseUrl: openai.baseUrl } : {}), + }; +} + +function readStt(stt: PersistedStt | undefined): SpeechSettings["dictation"]["stt"] { + return { + provider: stt?.provider ?? DEFAULT_SPEECH_PROVIDER, + ...(stt?.model ? { model: stt.model } : {}), + ...(stt?.language ? { language: stt.language } : {}), + }; +} + +export function mergeSpeechSettingsPatch( + persisted: PersistedConfig, + patch: SpeechSettingsPatch, +): PersistedConfig { + const features: PersistedFeatures = { ...persisted.features }; + if (patch.dictation) { + const current = features.dictation; + const stt = patch.dictation.stt ? mergeStt(current?.stt, patch.dictation.stt) : current?.stt; + features.dictation = withoutUndefined({ + ...current, + enabled: patch.dictation.enabled ?? current?.enabled, + stt, + }); + } + if (patch.voiceMode) { + const current = features.voiceMode; + features.voiceMode = withoutUndefined({ + ...current, + enabled: patch.voiceMode.enabled ?? current?.enabled, + stt: patch.voiceMode.stt ? mergeStt(current?.stt, patch.voiceMode.stt) : current?.stt, + tts: patch.voiceMode.tts ? mergeTts(current?.tts, patch.voiceMode.tts) : current?.tts, + }); + } + + const providers = { ...persisted.providers }; + if (patch.openai) { + providers.openai = mergeOpenAi(providers.openai, patch.openai); + } + + return { ...persisted, features, providers }; +} + +function mergeStt(current: PersistedStt | undefined, patch: SttPatch): PersistedStt { + const provider = parseProvider(patch.provider) ?? current?.provider; + const providerChanged = patch.provider !== undefined && patch.provider !== current?.provider; + const model = resolvePatchedValue(providerChanged ? undefined : current?.model, patch.model); + assertLocalModelKind(provider, model, "stt"); + return withoutUndefined({ + ...current, + provider, + model, + language: resolvePatchedValue(current?.language, patch.language), + }); +} + +function mergeTts(current: PersistedTts | undefined, patch: TtsPatch): PersistedTts { + const provider = parseProvider(patch.provider) ?? current?.provider; + const providerChanged = patch.provider !== undefined && patch.provider !== current?.provider; + const model = resolvePatchedValue(providerChanged ? undefined : current?.model, patch.model); + assertLocalModelKind(provider, model, "tts"); + return withoutUndefined({ + ...current, + provider, + model, + voice: resolvePatchedValue(providerChanged ? undefined : current?.voice, patch.voice), + }); +} + +function mergeOpenAi( + current: PersistedOpenAi | undefined, + patch: NonNullable, +): PersistedOpenAi | undefined { + const next = withoutUndefined({ + ...current, + apiKey: resolvePatchedValue(current?.apiKey, trimToNull(patch.apiKey)), + baseUrl: resolvePatchedValue(current?.baseUrl, trimToNull(patch.baseUrl)), + }); + return Object.keys(next).length > 0 ? next : undefined; +} + +// A local model id that is not in the catalog would stop the daemon from resolving its +// speech config at the next start, so reject it before it is written. +function assertLocalModelKind( + provider: string | undefined, + model: string | undefined, + kind: "stt" | "tts", +): void { + if ((provider ?? DEFAULT_SPEECH_PROVIDER) !== "local" || model === undefined) { + return; + } + const schema = kind === "stt" ? LocalSttModelIdSchema : LocalTtsModelIdSchema; + if (!schema.safeParse(model).success) { + throw new Error(`Unknown local ${kind === "stt" ? "speech-to-text" : "voice"} model: ${model}`); + } +} + +function parseProvider(provider: string | undefined): SpeechProviderId | undefined { + return provider === undefined ? undefined : SpeechProviderIdSchema.parse(provider); +} + +// undefined keeps the current value; null clears it. +function resolvePatchedValue( + current: string | undefined, + patch: string | null | undefined, +): string | undefined { + if (patch === undefined) return current; + return patch ?? undefined; +} + +function trimToNull(value: string | null | undefined): string | null | undefined { + if (typeof value !== "string") return value; + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : null; +} + +function withoutUndefined(value: T): T { + return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined)) as T; +} diff --git a/packages/server/src/server/test-utils/paseo-daemon.ts b/packages/server/src/server/test-utils/paseo-daemon.ts index 91b125328c..fef9d76014 100644 --- a/packages/server/src/server/test-utils/paseo-daemon.ts +++ b/packages/server/src/server/test-utils/paseo-daemon.ts @@ -38,7 +38,8 @@ interface TestPaseoDaemonOptions { staticDir?: string; cleanup?: boolean; openai?: PaseoOpenAIConfig; - speech?: PaseoSpeechConfig; + // localModelsDir defaults to the local provider's modelsDir, then to a dir under paseoHome. + speech?: Omit & { localModelsDir?: string }; voiceLlmProvider?: PaseoDaemonConfig["voiceLlmProvider"]; voiceLlmProviderExplicit?: boolean; voiceLlmModel?: string | null; @@ -207,7 +208,15 @@ async function prepareTestDaemonConfig( webUi: options.webUi, trustedProxies: options.trustedProxies, openai: options.openai, - speech: options.speech, + speech: options.speech + ? { + ...options.speech, + localModelsDir: + options.speech.localModelsDir ?? + options.speech.local?.modelsDir ?? + path.join(paseoHome, "models", "local-speech"), + } + : undefined, voiceLlmProvider: options.voiceLlmProvider ?? null, voiceLlmProviderExplicit: options.voiceLlmProviderExplicit ?? false, voiceLlmModel: options.voiceLlmModel ?? null, diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index e833a07ce3..65111db5f3 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -1500,6 +1500,7 @@ export class VoiceAssistantWebSocketServer { stt: () => this.speech?.resolveStt() ?? null, sttLanguage: this.speech?.resolveSttLanguage() ?? "en", tts: () => this.speech?.resolveTts() ?? null, + speechModels: this.speech, terminalManager: this.terminalManager, providerSnapshotManager: this.providerSnapshotManager, hubExecutionAgents: options.hubExecutionAgents, @@ -1831,6 +1832,8 @@ export class VoiceAssistantWebSocketServer { ...(this.advertiseDaemonStatusRpc ? { daemonStatusRpc: true } : {}), // COMPAT(daemonConfigReload): added in v0.4.0, remove gate after 2027-02-14. daemonConfigReload: true, + // COMPAT(speechSettings): added in v0.12.0, remove gate after 2027-04-18. + speechSettings: true, // COMPAT(relayConfig): added in v0.2.6, remove gate after 2027-01-31. ...(this.advertiseRelayConfig ? { relayConfig: true } : {}), // COMPAT(pushTokenRevocation): added in v0.3.2, remove gate after 2027-02-10. diff --git a/public-docs/voice.md b/public-docs/voice.md index 79022c15ad..00fe21ae62 100644 --- a/public-docs/voice.md +++ b/public-docs/voice.md @@ -27,16 +27,13 @@ This keeps credentials and execution in your environment and avoids introducing Local speech defaults to model IDs `parakeet-tdt-0.6b-v2-int8` (STT) and `kokoro-en-v0_19` (TTS, speaker 0 / voice 00). -Missing models are downloaded at daemon startup into `$PASEO_HOME/models/local-speech`. Downloads happen only for missing files. +Dictation is on by default. Voice mode is off until you turn it on in **Settings > Voice** or set `features.voiceMode.enabled: true`. + +Missing models for enabled features are downloaded at daemon startup into `$PASEO_HOME/models/local-speech`. Downloads happen only for missing files. **Settings > Voice** lists every local model with its size and languages, and lets you download or delete each one. ### Local STT models and language support -| Model ID | Languages | -| --------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `parakeet-tdt-0.6b-v2-int8` | English only (default). Includes punctuation and capitalization. | -| `parakeet-tdt-0.6b-v3-int8` | 25 European languages, auto-detected: Bulgarian, Croatian, Czech, Danish, Dutch, English, Estonian, Finnish, French, German, Greek, Hungarian, Italian, Latvian, Lithuanian, Maltese, Polish, Portuguese, Romanian, Russian, Slovak, Slovenian, Spanish, Swedish, Ukrainian. | - -**To use a non-English language, switch the local STT model to `parakeet-tdt-0.6b-v3-int8`.** v3 detects the spoken language automatically — there is no per-language setting for it. The `language` field below does **not** steer the local Parakeet model (v2 is English-only, v3 auto-detects); it only applies to the OpenAI STT provider. +**Settings > Voice** lists every local STT model with the languages it supports. The default, `parakeet-tdt-0.6b-v2-int8`, is English-only. For other languages, pick a multilingual model such as `parakeet-tdt-0.6b-v3-int8` (25 European languages) or `whisper-base` (99 languages). Local models detect the spoken language themselves; the `language` field below only applies to the OpenAI STT provider. ```json { @@ -59,7 +56,7 @@ Missing models are downloaded at daemon startup into `$PASEO_HOME/models/local-s } ``` -For multilingual local dictation, set the model to v3 — it auto-detects the language, so no `language` field is needed: +For multilingual local dictation, set a multilingual model. No `language` field is needed: ```json { @@ -72,7 +69,7 @@ For multilingual local dictation, set the model to v3 — it auto-detects the la } ``` -The `language` field applies only to the OpenAI STT provider: set `features.dictation.stt.language` for dictation and `features.voiceMode.stt.language` for voice mode. If voice language is omitted, Paseo uses the dictation language before falling back to `en`. It has no effect on the local Parakeet models. +The `language` field applies only to the OpenAI STT provider: set `features.dictation.stt.language` for dictation and `features.voiceMode.stt.language` for voice mode. If voice language is omitted, Paseo uses the dictation language before falling back to `en`. It has no effect on local models. ## OpenAI Voice Option