From 9bd26a46e1d38ef8091d67131dd685eadf1df563 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 6 Feb 2026 23:49:40 +0700 Subject: [PATCH] Update files --- .../server/scripts/download-speech-models.ts | 22 +++++---- packages/server/scripts/list-speech-models.ts | 4 +- .../server/src/server/bootstrap.smoke.test.ts | 10 ---- packages/server/src/server/bootstrap.ts | 7 --- packages/server/src/server/config.ts | 26 +++++----- .../src/server/daemon-client.e2e.test.ts | 25 ++++------ packages/server/src/server/session.ts | 28 +++++------ .../server/speech/providers/local/config.ts | 36 ++++++++------ .../server/speech/providers/local/models.ts | 48 +++++++++++++++++++ .../server/speech/providers/local/runtime.ts | 30 ++++++------ .../providers/local/sherpa/model-catalog.ts | 2 +- .../local/sherpa/model-downloader.ts | 4 +- .../local/sherpa/sherpa-offline-recognizer.ts | 2 +- .../local/sherpa/sherpa-online-recognizer.ts | 2 +- .../local/sherpa/sherpa-parakeet-stt.ts | 2 +- .../providers/local/sherpa/sherpa-stt.ts | 2 +- .../providers/local/sherpa/sherpa-tts.ts | 2 +- .../local/sherpa/speech-download.e2e.test.ts | 24 +++++----- .../speech/speech-config-resolver.test.ts | 34 ++++++++----- .../server/speech/speech-config-resolver.ts | 11 ----- .../src/server/voice-local-agent.e2e.test.ts | 5 -- .../server/src/server/websocket-server.ts | 6 +-- packages/website/src/routes/index.tsx | 4 +- 23 files changed, 181 insertions(+), 155 deletions(-) create mode 100644 packages/server/src/server/speech/providers/local/models.ts diff --git a/packages/server/scripts/download-speech-models.ts b/packages/server/scripts/download-speech-models.ts index a531c5942..fe2b91667 100644 --- a/packages/server/scripts/download-speech-models.ts +++ b/packages/server/scripts/download-speech-models.ts @@ -1,12 +1,16 @@ import { resolvePaseoHome } from "../src/server/paseo-home.js"; import { createRootLogger } from "../src/server/logger.js"; -import { ensureSherpaOnnxModels } from "../src/server/speech/providers/local/sherpa/model-downloader.js"; -import type { SherpaOnnxModelId } from "../src/server/speech/providers/local/sherpa/model-catalog.js"; +import { + DEFAULT_LOCAL_STT_MODEL, + DEFAULT_LOCAL_TTS_MODEL, + ensureLocalSpeechModels, + type LocalSpeechModelId, +} from "../src/server/speech/providers/local/models.js"; -function parseArgs(argv: string[]): { modelsDir: string; modelIds: SherpaOnnxModelId[] } { +function parseArgs(argv: string[]): { modelsDir: string; modelIds: LocalSpeechModelId[] } { const home = resolvePaseoHome(); - let modelsDir = process.env.PASEO_SHERPA_ONNX_MODELS_DIR || `${home}/models/sherpa-onnx`; - const modelIds: SherpaOnnxModelId[] = []; + let modelsDir = process.env.PASEO_LOCAL_MODELS_DIR || `${home}/models/local-speech`; + const modelIds: LocalSpeechModelId[] = []; for (let i = 0; i < argv.length; i++) { const arg = argv[i]; @@ -16,7 +20,7 @@ function parseArgs(argv: string[]): { modelsDir: string; modelIds: SherpaOnnxMod continue; } if (arg === "--model") { - const id = argv[i + 1] as SherpaOnnxModelId | undefined; + const id = argv[i + 1] as LocalSpeechModelId | undefined; if (!id) { throw new Error("--model requires a value"); } @@ -27,8 +31,8 @@ function parseArgs(argv: string[]): { modelsDir: string; modelIds: SherpaOnnxMod } if (modelIds.length === 0) { - const stt = (process.env.PASEO_SHERPA_STT_PRESET || "zipformer-bilingual-zh-en-2023-02-20") as SherpaOnnxModelId; - const tts = (process.env.PASEO_SHERPA_TTS_PRESET || "pocket-tts-onnx-int8") as SherpaOnnxModelId; + const stt = (process.env.PASEO_LOCAL_STT_MODEL || DEFAULT_LOCAL_STT_MODEL) as LocalSpeechModelId; + const tts = (process.env.PASEO_LOCAL_TTS_MODEL || DEFAULT_LOCAL_TTS_MODEL) as LocalSpeechModelId; modelIds.push(stt, tts); } @@ -38,5 +42,5 @@ function parseArgs(argv: string[]): { modelsDir: string; modelIds: SherpaOnnxMod const logger = createRootLogger({ level: "info", format: "pretty" }); const { modelsDir, modelIds } = parseArgs(process.argv.slice(2)); -await ensureSherpaOnnxModels({ modelsDir, modelIds, autoDownload: true, logger }); +await ensureLocalSpeechModels({ modelsDir, modelIds, autoDownload: true, logger }); logger.info({ modelsDir, modelIds }, "Done downloading speech models"); diff --git a/packages/server/scripts/list-speech-models.ts b/packages/server/scripts/list-speech-models.ts index 8462dfc27..dc014bb5e 100644 --- a/packages/server/scripts/list-speech-models.ts +++ b/packages/server/scripts/list-speech-models.ts @@ -1,6 +1,6 @@ -import { listSherpaOnnxModels } from "../src/server/speech/providers/local/sherpa/model-catalog.js"; +import { listLocalSpeechModels } from "../src/server/speech/providers/local/models.js"; -const models = listSherpaOnnxModels() +const models = listLocalSpeechModels() .slice() .sort((a, b) => a.kind.localeCompare(b.kind) || a.id.localeCompare(b.id)); diff --git a/packages/server/src/server/bootstrap.smoke.test.ts b/packages/server/src/server/bootstrap.smoke.test.ts index 5f6d0c8a0..404f01780 100644 --- a/packages/server/src/server/bootstrap.smoke.test.ts +++ b/packages/server/src/server/bootstrap.smoke.test.ts @@ -18,11 +18,6 @@ describe("paseo daemon bootstrap", () => { voiceStt: { provider: "openai", explicit: true }, voiceTts: { provider: "openai", explicit: true }, }, - localModels: { - dictationStt: "parakeet-tdt-0.6b-v3-int8", - voiceStt: "parakeet-tdt-0.6b-v3-int8", - voiceTts: "pocket-tts-onnx-int8", - }, }, }); try { @@ -68,11 +63,6 @@ describe("paseo daemon bootstrap", () => { voiceStt: { provider: "openai", explicit: true }, voiceTts: { provider: "openai", explicit: true }, }, - localModels: { - dictationStt: "parakeet-tdt-0.6b-v3-int8", - voiceStt: "parakeet-tdt-0.6b-v3-int8", - voiceTts: "pocket-tts-onnx-int8", - }, }, }; diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 754916a82..11b5f2705 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -108,13 +108,6 @@ export type PaseoLocalSpeechConfig = LocalSpeechProviderConfig; export type PaseoSpeechConfig = { providers: RequestedSpeechProviders; local?: PaseoLocalSpeechConfig; - localModels: { - dictationStt: string; - voiceStt: string; - voiceTts: string; - voiceTtsSpeakerId?: number; - voiceTtsSpeed?: number; - }; }; export type PaseoDaemonConfig = { diff --git a/packages/server/src/server/config.ts b/packages/server/src/server/config.ts index 1b3283c2d..e78863184 100644 --- a/packages/server/src/server/config.ts +++ b/packages/server/src/server/config.ts @@ -1,9 +1,10 @@ import path from "node:path"; +import { z } from "zod"; import type { PaseoDaemonConfig } from "./bootstrap.js"; import { loadPersistedConfig } from "./persisted-config.js"; import type { AgentProvider } from "./agent/agent-sdk-types.js"; -import { AGENT_PROVIDER_IDS } from "./agent/provider-manifest.js"; +import { AgentProviderSchema } from "./agent/provider-manifest.js"; import { resolveSpeechConfig } from "./speech/speech-config-resolver.js"; import { mergeAllowedHosts, @@ -26,17 +27,14 @@ export type CliConfigOverrides = Partial<{ allowedHosts: AllowedHostsConfig; }>; -function parseVoiceLlmProviderId(value: unknown): AgentProvider | null { - if (typeof value !== "string") { - return null; - } - const normalized = value.trim().toLowerCase(); - if (!normalized) { - return null; - } - return (AGENT_PROVIDER_IDS as readonly string[]).includes(normalized) - ? (normalized as AgentProvider) - : null; +const OptionalVoiceLlmProviderSchema = z + .union([z.string(), z.null(), z.undefined()]) + .transform((value): string | null => (typeof value === "string" ? value.trim().toLowerCase() : null)) + .pipe(z.union([AgentProviderSchema, z.null()])); + +function parseOptionalVoiceLlmProvider(value: unknown): AgentProvider | null { + const parsed = OptionalVoiceLlmProviderSchema.safeParse(value); + return parsed.success ? parsed.data : null; } export function loadConfig( @@ -97,8 +95,8 @@ export function loadConfig( persisted, }); - const envVoiceLlmProvider = parseVoiceLlmProviderId(env.PASEO_VOICE_LLM_PROVIDER); - const persistedVoiceLlmProvider = parseVoiceLlmProviderId( + const envVoiceLlmProvider = parseOptionalVoiceLlmProvider(env.PASEO_VOICE_LLM_PROVIDER); + const persistedVoiceLlmProvider = parseOptionalVoiceLlmProvider( persisted.features?.voiceMode?.llm?.provider ); const voiceLlmProvider = envVoiceLlmProvider ?? persistedVoiceLlmProvider ?? null; diff --git a/packages/server/src/server/daemon-client.e2e.test.ts b/packages/server/src/server/daemon-client.e2e.test.ts index 807443051..3eaea436a 100644 --- a/packages/server/src/server/daemon-client.e2e.test.ts +++ b/packages/server/src/server/daemon-client.e2e.test.ts @@ -101,11 +101,6 @@ describe("daemon client E2E", () => { voiceStt: { provider: "openai" as const, explicit: true }, voiceTts: { provider: "openai" as const, explicit: true }, }, - localModels: { - dictationStt: "parakeet-tdt-0.6b-v3-int8", - voiceStt: "parakeet-tdt-0.6b-v3-int8", - voiceTts: "pocket-tts-onnx-int8", - }, } : hasLocalSpeech ? { @@ -116,16 +111,16 @@ describe("daemon client E2E", () => { }, local: { modelsDir: localModelsDir, - }, - localModels: { - dictationStt: - process.env.PASEO_DICTATION_LOCAL_STT_MODEL ?? - "zipformer-bilingual-zh-en-2023-02-20", - voiceStt: - process.env.PASEO_VOICE_LOCAL_STT_MODEL ?? - "zipformer-bilingual-zh-en-2023-02-20", - voiceTts: - process.env.PASEO_VOICE_LOCAL_TTS_MODEL ?? "kitten-nano-en-v0_1-fp16", + models: { + dictationStt: + process.env.PASEO_DICTATION_LOCAL_STT_MODEL ?? + "zipformer-bilingual-zh-en-2023-02-20", + voiceStt: + process.env.PASEO_VOICE_LOCAL_STT_MODEL ?? + "zipformer-bilingual-zh-en-2023-02-20", + voiceTts: + process.env.PASEO_VOICE_LOCAL_TTS_MODEL ?? "kitten-nano-en-v0_1-fp16", + }, }, } : undefined; diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index d9001b0b8..75b689210 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -103,13 +103,11 @@ import { import { getProjectIcon } from "../utils/project-icon.js"; import { expandTilde } from "../utils/path.js"; import { - ensureSherpaOnnxModels, - getSherpaOnnxModelDir, -} from "./speech/providers/local/sherpa/model-downloader.js"; -import { - listSherpaOnnxModels, - type SherpaOnnxModelId, -} from "./speech/providers/local/sherpa/model-catalog.js"; + ensureLocalSpeechModels, + getLocalSpeechModelDir, + listLocalSpeechModels, + type LocalSpeechModelId, +} from "./speech/providers/local/models.js"; import type pino from "pino"; const execAsync = promisify(exec); @@ -222,7 +220,7 @@ export type SessionOptions = { stt?: SpeechToTextProvider | null; localModels?: { modelsDir: string; - defaultModelIds: SherpaOnnxModelId[]; + defaultModelIds: LocalSpeechModelId[]; }; }; }; @@ -371,7 +369,7 @@ export class Session { private readonly voiceLlmModel: string | null; private readonly voiceAgentMcpStdio: VoiceMcpStdioConfig | null; private readonly localSpeechModelsDir: string; - private readonly defaultLocalSpeechModelIds: SherpaOnnxModelId[]; + private readonly defaultLocalSpeechModelIds: LocalSpeechModelId[]; private readonly registerVoiceSpeakHandler?: ( agentId: string, handler: VoiceSpeakHandler @@ -1894,8 +1892,8 @@ export class Session { const modelsDir = this.localSpeechModelsDir; const models = await Promise.all( - listSherpaOnnxModels().map(async (model) => { - const modelDir = getSherpaOnnxModelDir(modelsDir, model.id); + listLocalSpeechModels().map(async (model) => { + const modelDir = getLocalSpeechModelDir(modelsDir, model.id); const missingFiles: string[] = []; for (const rel of model.requiredFiles) { const filePath = join(modelDir, rel); @@ -1943,8 +1941,8 @@ export class Session { ? msg.modelIds : this.defaultLocalSpeechModelIds; - const allModelIds = new Set(listSherpaOnnxModels().map((m) => m.id)); - const invalid = modelIdsRaw.filter((id) => !allModelIds.has(id as SherpaOnnxModelId)); + const allModelIds = new Set(listLocalSpeechModels().map((m) => m.id)); + const invalid = modelIdsRaw.filter((id) => !allModelIds.has(id as LocalSpeechModelId)); if (invalid.length > 0) { this.emit({ type: "speech_models_download_response", @@ -1958,9 +1956,9 @@ export class Session { return; } - const modelIds = modelIdsRaw as SherpaOnnxModelId[]; + const modelIds = modelIdsRaw as LocalSpeechModelId[]; try { - await ensureSherpaOnnxModels({ + await ensureLocalSpeechModels({ modelsDir, modelIds, autoDownload: true, diff --git a/packages/server/src/server/speech/providers/local/config.ts b/packages/server/src/server/speech/providers/local/config.ts index 1919d961d..5399111ce 100644 --- a/packages/server/src/server/speech/providers/local/config.ts +++ b/packages/server/src/server/speech/providers/local/config.ts @@ -12,20 +12,24 @@ import { type LocalSpeechModelId, type LocalSttModelId, type LocalTtsModelId, -} from "./sherpa/model-catalog.js"; +} from "./models.js"; + +export type LocalSpeechModelConfig = { + dictationStt: LocalSttModelId; + voiceStt: LocalSttModelId; + voiceTts: LocalTtsModelId; + voiceTtsSpeakerId?: number; + voiceTtsSpeed?: number; +}; export type LocalSpeechProviderConfig = { modelsDir: string; autoDownload?: boolean; + models: LocalSpeechModelConfig; }; export type ResolvedLocalSpeechConfig = { local: LocalSpeechProviderConfig | undefined; - dictationLocalSttModel: LocalSttModelId; - voiceLocalSttModel: LocalSttModelId; - voiceLocalTtsModel: LocalTtsModelId; - voiceLocalTtsSpeakerId?: number; - voiceLocalTtsSpeed?: number; }; export type { LocalSpeechModelId, LocalSttModelId, LocalTtsModelId }; @@ -146,16 +150,18 @@ export function resolveLocalSpeechConfig(params: { ? { modelsDir: parsed.modelsDir, autoDownload: parsed.autoDownload, + models: { + dictationStt: parsed.dictationLocalSttModel, + voiceStt: parsed.voiceLocalSttModel, + voiceTts: parsed.voiceLocalTtsModel, + ...(parsed.voiceLocalTtsSpeakerId !== undefined + ? { voiceTtsSpeakerId: parsed.voiceLocalTtsSpeakerId } + : {}), + ...(parsed.voiceLocalTtsSpeed !== undefined + ? { voiceTtsSpeed: parsed.voiceLocalTtsSpeed } + : {}), + }, } : undefined, - dictationLocalSttModel: parsed.dictationLocalSttModel, - voiceLocalSttModel: parsed.voiceLocalSttModel, - voiceLocalTtsModel: parsed.voiceLocalTtsModel, - ...(parsed.voiceLocalTtsSpeakerId !== undefined - ? { voiceLocalTtsSpeakerId: parsed.voiceLocalTtsSpeakerId } - : {}), - ...(parsed.voiceLocalTtsSpeed !== undefined - ? { voiceLocalTtsSpeed: parsed.voiceLocalTtsSpeed } - : {}), }; } diff --git a/packages/server/src/server/speech/providers/local/models.ts b/packages/server/src/server/speech/providers/local/models.ts new file mode 100644 index 000000000..ee2dee911 --- /dev/null +++ b/packages/server/src/server/speech/providers/local/models.ts @@ -0,0 +1,48 @@ +import { + ensureSherpaOnnxModels, + getSherpaOnnxModelDir, +} from "./sherpa/model-downloader.js"; +import { + DEFAULT_LOCAL_STT_MODEL, + DEFAULT_LOCAL_TTS_MODEL, + LocalSttModelIdSchema, + LocalTtsModelIdSchema, + listSherpaOnnxModels, + type LocalSpeechModelId, + type LocalSttModelId, + type LocalTtsModelId, +} from "./sherpa/model-catalog.js"; + +export { + DEFAULT_LOCAL_STT_MODEL, + DEFAULT_LOCAL_TTS_MODEL, + LocalSttModelIdSchema, + LocalTtsModelIdSchema, + type LocalSpeechModelId, + type LocalSttModelId, + type LocalTtsModelId, +}; + +export type LocalSpeechModelSpec = ReturnType[number]; + +export function listLocalSpeechModels(): LocalSpeechModelSpec[] { + return listSherpaOnnxModels(); +} + +export function getLocalSpeechModelDir(modelsDir: string, modelId: LocalSpeechModelId): string { + return getSherpaOnnxModelDir(modelsDir, modelId); +} + +export async function ensureLocalSpeechModels(options: { + modelsDir: string; + modelIds: LocalSpeechModelId[]; + autoDownload?: boolean; + logger: import("pino").Logger; +}): Promise> { + return ensureSherpaOnnxModels({ + modelsDir: options.modelsDir, + modelIds: options.modelIds, + autoDownload: options.autoDownload ?? true, + logger: options.logger, + }); +} diff --git a/packages/server/src/server/speech/providers/local/runtime.ts b/packages/server/src/server/speech/providers/local/runtime.ts index f72c23b52..13bd4ef9d 100644 --- a/packages/server/src/server/speech/providers/local/runtime.ts +++ b/packages/server/src/server/speech/providers/local/runtime.ts @@ -5,10 +5,8 @@ import type { SpeechToTextProvider, TextToSpeechProvider } from "../../speech-pr import type { RequestedSpeechProviders } from "../../speech-types.js"; import { PocketTtsOnnxTTS } from "./pocket/pocket-tts-onnx.js"; import { - ensureSherpaOnnxModels, - getSherpaOnnxModelDir, -} from "./sherpa/model-downloader.js"; -import { + ensureLocalSpeechModels, + getLocalSpeechModelDir, DEFAULT_LOCAL_STT_MODEL, DEFAULT_LOCAL_TTS_MODEL, LocalSttModelIdSchema, @@ -16,7 +14,7 @@ import { type LocalSpeechModelId, type LocalSttModelId, type LocalTtsModelId, -} from "./sherpa/model-catalog.js"; +} from "./models.js"; import { SherpaOfflineRecognizerEngine } from "./sherpa/sherpa-offline-recognizer.js"; import { SherpaOnlineRecognizerEngine } from "./sherpa/sherpa-online-recognizer.js"; import { SherpaOnnxParakeetSTT } from "./sherpa/sherpa-parakeet-stt.js"; @@ -63,13 +61,13 @@ function resolveConfiguredLocalModels( ): ResolvedLocalModels { return { dictationLocalSttModel: LocalSttModelIdSchema.parse( - speechConfig?.localModels.dictationStt ?? DEFAULT_LOCAL_STT_MODEL + speechConfig?.local?.models.dictationStt ?? DEFAULT_LOCAL_STT_MODEL ), voiceLocalSttModel: LocalSttModelIdSchema.parse( - speechConfig?.localModels.voiceStt ?? DEFAULT_LOCAL_STT_MODEL + speechConfig?.local?.models.voiceStt ?? DEFAULT_LOCAL_STT_MODEL ), voiceLocalTtsModel: LocalTtsModelIdSchema.parse( - speechConfig?.localModels.voiceTts ?? DEFAULT_LOCAL_TTS_MODEL + speechConfig?.local?.models.voiceTts ?? DEFAULT_LOCAL_TTS_MODEL ), }; } @@ -110,7 +108,7 @@ async function createLocalSttEngine(params: { const { modelId, modelsDir, logger } = params; if (modelId === "parakeet-tdt-0.6b-v3-int8") { - const modelDir = getSherpaOnnxModelDir(modelsDir, modelId); + const modelDir = getLocalSpeechModelDir(modelsDir, modelId); return { kind: "offline", engine: new SherpaOfflineRecognizerEngine( @@ -131,7 +129,7 @@ async function createLocalSttEngine(params: { } if (modelId === "paraformer-bilingual-zh-en") { - const modelDir = getSherpaOnnxModelDir(modelsDir, modelId); + const modelDir = getLocalSpeechModelDir(modelsDir, modelId); return { kind: "online", engine: new SherpaOnlineRecognizerEngine( @@ -151,7 +149,7 @@ async function createLocalSttEngine(params: { } if (modelId === "zipformer-bilingual-zh-en-2023-02-20") { - const modelDir = getSherpaOnnxModelDir(modelsDir, modelId); + const modelDir = getLocalSpeechModelDir(modelsDir, modelId); return { kind: "online", engine: new SherpaOnlineRecognizerEngine( @@ -204,7 +202,7 @@ export async function initializeLocalSpeechServices(params: { }, "Ensuring local speech models" ); - await ensureSherpaOnnxModels({ + await ensureLocalSpeechModels({ modelsDir: localConfig.modelsDir, modelIds: requiredLocalModelIds, autoDownload: localConfig.autoDownload ?? true, @@ -308,7 +306,7 @@ export async function initializeLocalSpeechServices(params: { } else { try { if (localModels.voiceLocalTtsModel === "pocket-tts-onnx-int8") { - const modelDir = getSherpaOnnxModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel); + const modelDir = getLocalSpeechModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel); localVoiceTtsProvider = await PocketTtsOnnxTTS.create( { modelDir, @@ -318,13 +316,13 @@ export async function initializeLocalSpeechServices(params: { logger ); } else { - const modelDir = getSherpaOnnxModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel); + const modelDir = getLocalSpeechModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel); localVoiceTtsProvider = new SherpaOnnxTTS( { preset: localModels.voiceLocalTtsModel, modelDir, - speakerId: speechConfig?.localModels.voiceTtsSpeakerId, - speed: speechConfig?.localModels.voiceTtsSpeed, + speakerId: speechConfig?.local?.models.voiceTtsSpeakerId, + speed: speechConfig?.local?.models.voiceTtsSpeed, }, logger ); diff --git a/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts b/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts index 2390d441b..084c89b9d 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts @@ -209,7 +209,7 @@ export function listSherpaOnnxModels(): SherpaOnnxModelSpec[] { export function getSherpaOnnxModelSpec(id: SherpaOnnxModelId): SherpaOnnxModelSpec { const spec = SHERPA_ONNX_MODEL_CATALOG[id]; if (!spec) { - throw new Error(`Unknown sherpa-onnx model id: ${id}`); + throw new Error(`Unknown local speech model id: ${id}`); } return { id, diff --git a/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts b/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts index 68bbe83a6..66298751d 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts @@ -96,7 +96,7 @@ async function isNonEmptyFile(filePath: string): Promise { export async function ensureSherpaOnnxModel(options: EnsureSherpaOnnxModelOptions): Promise { const logger = options.logger.child({ module: "speech", - provider: "sherpa-onnx", + provider: "local", component: "model-downloader", modelId: options.modelId, }); @@ -109,7 +109,7 @@ export async function ensureSherpaOnnxModel(options: EnsureSherpaOnnxModelOption if (!options.autoDownload) { throw new Error( - `Missing sherpa-onnx model files for ${options.modelId} in ${modelDir}. ` + + `Missing local speech model files for ${options.modelId} in ${modelDir}. ` + `Set PASEO_LOCAL_AUTO_DOWNLOAD=1 to auto-download.` ); } diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-offline-recognizer.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-offline-recognizer.ts index 810b8d165..4cb40416c 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-offline-recognizer.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-offline-recognizer.ts @@ -34,7 +34,7 @@ export class SherpaOfflineRecognizerEngine { private readonly logger: pino.Logger; constructor(config: SherpaOfflineRecognizerConfig, logger: pino.Logger) { - this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "offline-recognizer" }); + this.logger = logger.child({ module: "speech", provider: "local", component: "offline-recognizer" }); assertFileExists(config.model.encoder, "offline encoder"); assertFileExists(config.model.decoder, "offline decoder"); diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-online-recognizer.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-online-recognizer.ts index c7c8c3b1b..4148ca860 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-online-recognizer.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-online-recognizer.ts @@ -46,7 +46,7 @@ export class SherpaOnlineRecognizerEngine { private readonly logger: pino.Logger; constructor(config: SherpaOnlineRecognizerConfig, logger: pino.Logger) { - this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "online-recognizer" }); + this.logger = logger.child({ module: "speech", provider: "local", component: "online-recognizer" }); const { model } = config; if (model.kind === "transducer") { diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-stt.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-stt.ts index 3b23fa8e1..8bc8bb52c 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-stt.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-stt.ts @@ -25,7 +25,7 @@ export class SherpaOnnxParakeetSTT implements SpeechToTextProvider { constructor(config: SherpaParakeetSttConfig, logger: pino.Logger) { this.engine = config.engine; this.silencePeakThreshold = config.silencePeakThreshold ?? 300; - this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "parakeet-stt" }); + this.logger = logger.child({ module: "speech", provider: "local", component: "parakeet-stt" }); } public createSession(params: { diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-stt.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-stt.ts index ff308f51d..05dbf1d2a 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-stt.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-stt.ts @@ -28,7 +28,7 @@ export class SherpaOnnxSTT implements SpeechToTextProvider { this.engine = config.engine; this.silencePeakThreshold = config.silencePeakThreshold ?? 300; this.tailPaddingMs = config.tailPaddingMs ?? 500; - this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "stt" }); + this.logger = logger.child({ module: "speech", provider: "local", component: "stt" }); } public createSession(params: { diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts index 99d9dab6a..8c05461b8 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts @@ -33,7 +33,7 @@ export class SherpaOnnxTTS implements TextToSpeechProvider { if (config.preset !== "kokoro-en-v0_19" && config.preset !== "kitten-nano-en-v0_1-fp16") { throw new Error(`Unsupported Sherpa TTS preset: ${config.preset}`); } - this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "tts" }); + this.logger = logger.child({ module: "speech", provider: "local", component: "tts" }); this.speakerId = config.speakerId ?? 0; this.speed = config.speed ?? 1.0; diff --git a/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts b/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts index 6fc8f8324..8af66a672 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts @@ -123,18 +123,18 @@ describe("speech models (download E2E)", () => { local: { modelsDir, autoDownload: false, - }, - localModels: { - dictationStt: - set === "parakeet-pocket" - ? "parakeet-tdt-0.6b-v3-int8" - : "zipformer-bilingual-zh-en-2023-02-20", - voiceStt: - set === "parakeet-pocket" - ? "parakeet-tdt-0.6b-v3-int8" - : "zipformer-bilingual-zh-en-2023-02-20", - voiceTts: - set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16", + models: { + dictationStt: + set === "parakeet-pocket" + ? "parakeet-tdt-0.6b-v3-int8" + : "zipformer-bilingual-zh-en-2023-02-20", + voiceStt: + set === "parakeet-pocket" + ? "parakeet-tdt-0.6b-v3-int8" + : "zipformer-bilingual-zh-en-2023-02-20", + voiceTts: + set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16", + }, }, }, }); diff --git a/packages/server/src/server/speech/speech-config-resolver.test.ts b/packages/server/src/server/speech/speech-config-resolver.test.ts index 211b6265f..99c5a955f 100644 --- a/packages/server/src/server/speech/speech-config-resolver.test.ts +++ b/packages/server/src/server/speech/speech-config-resolver.test.ts @@ -33,10 +33,15 @@ describe("resolveSpeechConfig", () => { expect(result.speech.local).toEqual({ modelsDir: path.join(paseoHome, "models", "local-speech"), autoDownload: true, + models: { + dictationStt: "parakeet-tdt-0.6b-v3-int8", + voiceStt: "parakeet-tdt-0.6b-v3-int8", + voiceTts: "pocket-tts-onnx-int8", + }, }); - expect(result.speech.localModels.dictationStt).toBe("parakeet-tdt-0.6b-v3-int8"); - expect(result.speech.localModels.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8"); - expect(result.speech.localModels.voiceTts).toBe("pocket-tts-onnx-int8"); + expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v3-int8"); + expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8"); + expect(result.speech.local?.models.voiceTts).toBe("pocket-tts-onnx-int8"); }); test("resolves feature-scoped local model env vars", () => { @@ -73,6 +78,13 @@ describe("resolveSpeechConfig", () => { expect(result.speech.local).toEqual({ modelsDir: "/tmp/models", autoDownload: false, + models: { + dictationStt: "zipformer-bilingual-zh-en-2023-02-20", + voiceStt: "parakeet-tdt-0.6b-v3-int8", + voiceTts: "kitten-nano-en-v0_1-fp16", + voiceTtsSpeakerId: 5, + voiceTtsSpeed: 1.35, + }, }); expect(result.speech.providers.dictationStt).toEqual({ provider: "local", @@ -86,11 +98,11 @@ describe("resolveSpeechConfig", () => { provider: "local", explicit: true, }); - expect(result.speech.localModels.dictationStt).toBe("zipformer-bilingual-zh-en-2023-02-20"); - expect(result.speech.localModels.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8"); - expect(result.speech.localModels.voiceTts).toBe("kitten-nano-en-v0_1-fp16"); - expect(result.speech.localModels.voiceTtsSpeakerId).toBe(5); - expect(result.speech.localModels.voiceTtsSpeed).toBe(1.35); + expect(result.speech.local?.models.dictationStt).toBe("zipformer-bilingual-zh-en-2023-02-20"); + expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8"); + expect(result.speech.local?.models.voiceTts).toBe("kitten-nano-en-v0_1-fp16"); + expect(result.speech.local?.models.voiceTtsSpeakerId).toBe(5); + expect(result.speech.local?.models.voiceTtsSpeed).toBe(1.35); expect(result.openai?.apiKey).toBe("env-key"); expect(result.openai?.stt?.model).toBe("gpt-4o-transcribe"); }); @@ -108,8 +120,8 @@ describe("resolveSpeechConfig", () => { persisted, }); - expect(result.speech.localModels.dictationStt).toBe("parakeet-tdt-0.6b-v3-int8"); - expect(result.speech.localModels.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8"); - expect(result.speech.localModels.voiceTts).toBe("pocket-tts-onnx-int8"); + expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v3-int8"); + expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8"); + expect(result.speech.local?.models.voiceTts).toBe("pocket-tts-onnx-int8"); }); }); diff --git a/packages/server/src/server/speech/speech-config-resolver.ts b/packages/server/src/server/speech/speech-config-resolver.ts index 1389e8738..0d374954a 100644 --- a/packages/server/src/server/speech/speech-config-resolver.ts +++ b/packages/server/src/server/speech/speech-config-resolver.ts @@ -100,17 +100,6 @@ export function resolveSpeechConfig(params: { ...(local.local ? { local: local.local } : {}), - localModels: { - dictationStt: local.dictationLocalSttModel, - voiceStt: local.voiceLocalSttModel, - voiceTts: local.voiceLocalTtsModel, - ...(local.voiceLocalTtsSpeakerId !== undefined - ? { voiceTtsSpeakerId: local.voiceLocalTtsSpeakerId } - : {}), - ...(local.voiceLocalTtsSpeed !== undefined - ? { voiceTtsSpeed: local.voiceLocalTtsSpeed } - : {}), - }, }, }; } diff --git a/packages/server/src/server/voice-local-agent.e2e.test.ts b/packages/server/src/server/voice-local-agent.e2e.test.ts index 69499b55c..95e48ecbd 100644 --- a/packages/server/src/server/voice-local-agent.e2e.test.ts +++ b/packages/server/src/server/voice-local-agent.e2e.test.ts @@ -53,11 +53,6 @@ function waitForSignal( voiceStt: { provider: "openai", explicit: true }, voiceTts: { provider: "openai", explicit: true }, }, - localModels: { - dictationStt: "parakeet-tdt-0.6b-v3-int8", - voiceStt: "parakeet-tdt-0.6b-v3-int8", - voiceTts: "pocket-tts-onnx-int8", - }, }, voiceLlmProvider: "codex", voiceLlmProviderExplicit: true, diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index cc2d9a267..610420c1a 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -20,7 +20,7 @@ import type { AgentProvider } from "./agent/agent-sdk-types.js"; import { PushTokenStore } from "./push/token-store.js"; import { PushService } from "./push/push-service.js"; import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js"; -import type { SherpaOnnxModelId } from "./speech/providers/local/sherpa/model-catalog.js"; +import type { LocalSpeechModelId } from "./speech/providers/local/models.js"; import type { VoiceCallerContext, VoiceMcpStdioConfig, @@ -79,7 +79,7 @@ export class VoiceAssistantWebSocketServer { stt?: SpeechToTextProvider | null; localModels?: { modelsDir: string; - defaultModelIds: SherpaOnnxModelId[]; + defaultModelIds: LocalSpeechModelId[]; }; } | null; private readonly voice: { @@ -119,7 +119,7 @@ export class VoiceAssistantWebSocketServer { stt?: SpeechToTextProvider | null; localModels?: { modelsDir: string; - defaultModelIds: SherpaOnnxModelId[]; + defaultModelIds: LocalSpeechModelId[]; }; } ) { diff --git a/packages/website/src/routes/index.tsx b/packages/website/src/routes/index.tsx index 4abb20990..61f30953a 100644 --- a/packages/website/src/routes/index.tsx +++ b/packages/website/src/routes/index.tsx @@ -287,8 +287,8 @@ function FAQ() { Paseo is free and open source. It wraps CLI tools like Claude Code and Codex, which you'll need to have installed and configured with your - own credentials. Voice features currently require an OpenAI API key, - but local voice is coming soon. + own credentials. Voice is local-first by default and can optionally use + OpenAI speech providers if you configure them. Paseo itself doesn't send your code anywhere. Agents run locally and