From bc4e2daf6742cb82ea3b4a94998ed55f1234587d Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 6 Feb 2026 22:48:52 +0700 Subject: [PATCH] Refactor speech provider selection model and harden CI startup --- .../server/src/server/bootstrap.smoke.test.ts | 26 ++++-- packages/server/src/server/bootstrap.ts | 58 +++++++------ .../src/server/daemon-client.e2e.test.ts | 60 ++++++++------ .../daemon-e2e/connection-offer.e2e.test.ts | 1 + .../server/src/server/persisted-config.ts | 14 ++-- .../server/speech/providers/local/config.ts | 14 ++-- .../server/speech/providers/local/runtime.ts | 22 ++--- .../local/sherpa/speech-download.e2e.test.ts | 30 ++++--- .../speech/providers/openai/config.test.ts | 46 +++++++++++ .../server/speech/providers/openai/config.ts | 16 ++-- .../server/speech/providers/openai/runtime.ts | 22 ++--- .../speech/speech-config-resolver.test.ts | 49 +++++++---- .../server/speech/speech-config-resolver.ts | 81 ++++++++++++------- .../src/server/speech/speech-runtime.ts | 79 +++++++++++++----- .../server/src/server/speech/speech-types.ts | 12 ++- .../src/server/voice-local-agent.e2e.test.ts | 13 ++- 16 files changed, 368 insertions(+), 175 deletions(-) create mode 100644 packages/server/src/server/speech/providers/openai/config.test.ts diff --git a/packages/server/src/server/bootstrap.smoke.test.ts b/packages/server/src/server/bootstrap.smoke.test.ts index ac19a9d90..5f6d0c8a0 100644 --- a/packages/server/src/server/bootstrap.smoke.test.ts +++ b/packages/server/src/server/bootstrap.smoke.test.ts @@ -13,9 +13,16 @@ describe("paseo daemon bootstrap", () => { const daemonHandle = await createTestPaseoDaemon({ openai: { apiKey: "test-openai-api-key" }, speech: { - dictationSttProvider: "openai", - voiceSttProvider: "openai", - voiceTtsProvider: "openai", + providers: { + dictationStt: { provider: "openai", explicit: true }, + voiceStt: { provider: "openai", explicit: true }, + voiceTts: { provider: "openai", explicit: true }, + }, + localModels: { + dictationStt: "parakeet-tdt-0.6b-v3-int8", + voiceStt: "parakeet-tdt-0.6b-v3-int8", + voiceTts: "pocket-tts-onnx-int8", + }, }, }); try { @@ -56,9 +63,16 @@ describe("paseo daemon bootstrap", () => { appBaseUrl: "https://app.paseo.sh", openai: undefined, speech: { - dictationSttProvider: "openai", - voiceSttProvider: "openai", - voiceTtsProvider: "openai", + providers: { + dictationStt: { provider: "openai", explicit: true }, + voiceStt: { provider: "openai", explicit: true }, + voiceTts: { provider: "openai", explicit: true }, + }, + localModels: { + dictationStt: "parakeet-tdt-0.6b-v3-int8", + voiceStt: "parakeet-tdt-0.6b-v3-int8", + voiceTts: "pocket-tts-onnx-int8", + }, }, }; diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 19aece6c0..754916a82 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -43,6 +43,7 @@ import { VoiceAssistantWebSocketServer } from "./websocket-server.js"; import { DownloadTokenStore } from "./file-download/token-store.js"; import type { OpenAiSpeechProviderConfig } from "./speech/providers/openai/config.js"; import type { LocalSpeechProviderConfig } from "./speech/providers/local/config.js"; +import type { RequestedSpeechProviders } from "./speech/speech-types.js"; import { initializeSpeechRuntime } from "./speech/speech-runtime.js"; import { AgentManager } from "./agent/agent-manager.js"; import { AgentStorage } from "./agent/agent-storage.js"; @@ -105,15 +106,15 @@ export type PaseoOpenAIConfig = OpenAiSpeechProviderConfig; export type PaseoLocalSpeechConfig = LocalSpeechProviderConfig; export type PaseoSpeechConfig = { - dictationSttProvider?: "openai" | "local"; - voiceSttProvider?: "openai" | "local"; - voiceTtsProvider?: "openai" | "local"; + providers: RequestedSpeechProviders; local?: PaseoLocalSpeechConfig; - dictationLocalSttModel?: string; - voiceLocalSttModel?: string; - voiceLocalTtsModel?: string; - voiceLocalTtsSpeakerId?: number; - voiceLocalTtsSpeed?: number; + localModels: { + dictationStt: string; + voiceStt: string; + voiceTts: string; + voiceTtsSpeakerId?: number; + voiceTtsSpeed?: number; + }; }; export type PaseoDaemonConfig = { @@ -347,25 +348,34 @@ export async function createPaseoDaemon( voiceEnabledProviders.find((provider) => voiceLlmAvailability[provider]) ?? null; } - if (!resolvedVoiceLlmProvider) { - logger.error( - { requestedVoiceLlmProvider, voiceLlmAvailability }, - "No voice LLM provider available" - ); - throw new Error("No voice LLM provider available"); - } + let resolvedVoiceLlmModeId: string | null = null; + let resolvedVoiceLlmModel: string | null = null; - const resolvedVoiceProviderDefinition = AGENT_PROVIDER_DEFINITIONS.find( - (definition) => definition.id === resolvedVoiceLlmProvider - ); - if (!resolvedVoiceProviderDefinition?.voice?.enabled) { - throw new Error( - `Provider '${resolvedVoiceLlmProvider}' is missing voice metadata in agent registry` + if (!resolvedVoiceLlmProvider) { + if (voiceLlmProviderExplicit) { + logger.error( + { requestedVoiceLlmProvider, voiceLlmAvailability }, + "No voice LLM provider available" + ); + throw new Error("No voice LLM provider available"); + } + logger.warn( + { requestedVoiceLlmProvider, voiceLlmAvailability }, + "No default voice LLM provider available; voice mode will be disabled until a provider is configured" ); + } else { + const resolvedVoiceProviderDefinition = AGENT_PROVIDER_DEFINITIONS.find( + (definition) => definition.id === resolvedVoiceLlmProvider + ); + if (!resolvedVoiceProviderDefinition?.voice?.enabled) { + throw new Error( + `Provider '${resolvedVoiceLlmProvider}' is missing voice metadata in agent registry` + ); + } + resolvedVoiceLlmModeId = resolvedVoiceProviderDefinition.voice.defaultModeId; + resolvedVoiceLlmModel = + config.voiceLlmModel ?? resolvedVoiceProviderDefinition.voice.defaultModel ?? null; } - const resolvedVoiceLlmModeId = resolvedVoiceProviderDefinition.voice.defaultModeId; - const resolvedVoiceLlmModel = - config.voiceLlmModel ?? resolvedVoiceProviderDefinition.voice.defaultModel ?? null; logger.info( { diff --git a/packages/server/src/server/daemon-client.e2e.test.ts b/packages/server/src/server/daemon-client.e2e.test.ts index f6fe778ec..48119151a 100644 --- a/packages/server/src/server/daemon-client.e2e.test.ts +++ b/packages/server/src/server/daemon-client.e2e.test.ts @@ -93,33 +93,47 @@ describe("daemon client E2E", () => { let ctx: DaemonTestContext; beforeAll(async () => { - const speechConfig = openaiApiKey - ? { - dictationSttProvider: "openai" as const, - voiceSttProvider: "openai" as const, - voiceTtsProvider: "openai" as const, - } - : { - dictationSttProvider: "local" as const, - voiceSttProvider: "local" as const, - voiceTtsProvider: "local" as const, - local: { - modelsDir: localModelsDir, - }, - dictationLocalSttModel: - process.env.PASEO_DICTATION_LOCAL_STT_MODEL ?? - "zipformer-bilingual-zh-en-2023-02-20", - voiceLocalSttModel: - process.env.PASEO_VOICE_LOCAL_STT_MODEL ?? - "zipformer-bilingual-zh-en-2023-02-20", - voiceLocalTtsModel: - process.env.PASEO_VOICE_LOCAL_TTS_MODEL ?? "kitten-nano-en-v0_1-fp16", - }; + const speechConfig = + openaiApiKey + ? { + providers: { + dictationStt: { provider: "openai" as const, explicit: true }, + voiceStt: { provider: "openai" as const, explicit: true }, + voiceTts: { provider: "openai" as const, explicit: true }, + }, + localModels: { + dictationStt: "parakeet-tdt-0.6b-v3-int8", + voiceStt: "parakeet-tdt-0.6b-v3-int8", + voiceTts: "pocket-tts-onnx-int8", + }, + } + : hasLocalSpeech + ? { + providers: { + dictationStt: { provider: "local" as const, explicit: true }, + voiceStt: { provider: "local" as const, explicit: true }, + voiceTts: { provider: "local" as const, explicit: true }, + }, + local: { + modelsDir: localModelsDir, + }, + localModels: { + dictationStt: + process.env.PASEO_DICTATION_LOCAL_STT_MODEL ?? + "zipformer-bilingual-zh-en-2023-02-20", + voiceStt: + process.env.PASEO_VOICE_LOCAL_STT_MODEL ?? + "zipformer-bilingual-zh-en-2023-02-20", + voiceTts: + process.env.PASEO_VOICE_LOCAL_TTS_MODEL ?? "kitten-nano-en-v0_1-fp16", + }, + } + : undefined; ctx = await createDaemonTestContext({ dictationFinalTimeoutMs: 5000, ...(openaiApiKey ? { openai: { apiKey: openaiApiKey } } : {}), - speech: speechConfig, + ...(speechConfig ? { speech: speechConfig } : {}), }); }, 60000); diff --git a/packages/server/src/server/daemon-e2e/connection-offer.e2e.test.ts b/packages/server/src/server/daemon-e2e/connection-offer.e2e.test.ts index 3ba4cda14..6913d0c88 100644 --- a/packages/server/src/server/daemon-e2e/connection-offer.e2e.test.ts +++ b/packages/server/src/server/daemon-e2e/connection-offer.e2e.test.ts @@ -186,6 +186,7 @@ describe("ConnectionOfferV2 (daemon E2E)", () => { PASEO_HOME: tempHome, PASEO_LISTEN: `0.0.0.0:${port}`, OPENAI_API_KEY: "", + PASEO_LOCAL_AUTO_DOWNLOAD: "0", PASEO_LOG_FORMAT: "json", }; diff --git a/packages/server/src/server/persisted-config.ts b/packages/server/src/server/persisted-config.ts index 690e8bef3..a9fc694f6 100644 --- a/packages/server/src/server/persisted-config.ts +++ b/packages/server/src/server/persisted-config.ts @@ -32,15 +32,11 @@ const ProvidersSchema = z }) .strict(); -const SpeechProviderIdSchema = z.preprocess( - (value) => { - if (typeof value !== "string") { - return value; - } - return value.trim().toLowerCase(); - }, - z.enum(["openai", "local"]) -); +const SpeechProviderIdSchema = z + .string() + .trim() + .toLowerCase() + .pipe(z.enum(["openai", "local"])); const FeatureDictationSchema = z .object({ diff --git a/packages/server/src/server/speech/providers/local/config.ts b/packages/server/src/server/speech/providers/local/config.ts index dea7a1bd8..1919d961d 100644 --- a/packages/server/src/server/speech/providers/local/config.ts +++ b/packages/server/src/server/speech/providers/local/config.ts @@ -68,7 +68,7 @@ const LocalSpeechResolutionSchema = z.object({ }); function persistedLocalFeatureModel( - provider: RequestedSpeechProviders[keyof RequestedSpeechProviders], + provider: RequestedSpeechProviders[keyof RequestedSpeechProviders]["provider"], model: string | undefined ): string | undefined { if (provider !== "local") { @@ -83,9 +83,9 @@ function shouldIncludeLocalProviderConfig(params: { persisted: PersistedConfig; }): boolean { const localRequestedByFeature = - params.providers.dictationSttProvider === "local" || - params.providers.voiceSttProvider === "local" || - params.providers.voiceTtsProvider === "local"; + params.providers.dictationStt.provider === "local" || + params.providers.voiceStt.provider === "local" || + params.providers.voiceTts.provider === "local"; return ( localRequestedByFeature || @@ -114,21 +114,21 @@ export function resolveLocalSpeechConfig(params: { dictationLocalSttModel: params.env.PASEO_DICTATION_LOCAL_STT_MODEL ?? persistedLocalFeatureModel( - params.providers.dictationSttProvider, + params.providers.dictationStt.provider, params.persisted.features?.dictation?.stt?.model ) ?? DEFAULT_LOCAL_STT_MODEL, voiceLocalSttModel: params.env.PASEO_VOICE_LOCAL_STT_MODEL ?? persistedLocalFeatureModel( - params.providers.voiceSttProvider, + params.providers.voiceStt.provider, params.persisted.features?.voiceMode?.stt?.model ) ?? DEFAULT_LOCAL_STT_MODEL, voiceLocalTtsModel: params.env.PASEO_VOICE_LOCAL_TTS_MODEL ?? persistedLocalFeatureModel( - params.providers.voiceTtsProvider, + params.providers.voiceTts.provider, params.persisted.features?.voiceMode?.tts?.model ) ?? DEFAULT_LOCAL_TTS_MODEL, diff --git a/packages/server/src/server/speech/providers/local/runtime.ts b/packages/server/src/server/speech/providers/local/runtime.ts index a060ac587..f72c23b52 100644 --- a/packages/server/src/server/speech/providers/local/runtime.ts +++ b/packages/server/src/server/speech/providers/local/runtime.ts @@ -63,13 +63,13 @@ function resolveConfiguredLocalModels( ): ResolvedLocalModels { return { dictationLocalSttModel: LocalSttModelIdSchema.parse( - speechConfig?.dictationLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL + speechConfig?.localModels.dictationStt ?? DEFAULT_LOCAL_STT_MODEL ), voiceLocalSttModel: LocalSttModelIdSchema.parse( - speechConfig?.voiceLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL + speechConfig?.localModels.voiceStt ?? DEFAULT_LOCAL_STT_MODEL ), voiceLocalTtsModel: LocalTtsModelIdSchema.parse( - speechConfig?.voiceLocalTtsModel ?? DEFAULT_LOCAL_TTS_MODEL + speechConfig?.localModels.voiceTts ?? DEFAULT_LOCAL_TTS_MODEL ), }; } @@ -90,13 +90,13 @@ function computeRequiredLocalModelIds(params: { models: ResolvedLocalModels; }): LocalSpeechModelId[] { const ids = new Set(); - if (params.providers.dictationSttProvider === "local") { + if (params.providers.dictationStt.provider === "local") { ids.add(params.models.dictationLocalSttModel); } - if (params.providers.voiceSttProvider === "local") { + if (params.providers.voiceStt.provider === "local") { ids.add(params.models.voiceLocalSttModel); } - if (params.providers.voiceTtsProvider === "local") { + if (params.providers.voiceTts.provider === "local") { ids.add(params.models.voiceLocalTtsModel); } return Array.from(ids); @@ -260,7 +260,7 @@ export async function initializeLocalSpeechServices(params: { } }; - if (providers.voiceSttProvider === "local") { + if (providers.voiceStt.provider === "local") { if (!localConfig) { logger.warn( { configured: false }, @@ -276,7 +276,7 @@ export async function initializeLocalSpeechServices(params: { } } - if (providers.dictationSttProvider === "local") { + if (providers.dictationStt.provider === "local") { if (!localConfig) { logger.warn( { configured: false }, @@ -299,7 +299,7 @@ export async function initializeLocalSpeechServices(params: { } } - if (providers.voiceTtsProvider === "local") { + if (providers.voiceTts.provider === "local") { if (!localConfig) { logger.warn( { configured: false }, @@ -323,8 +323,8 @@ export async function initializeLocalSpeechServices(params: { { preset: localModels.voiceLocalTtsModel, modelDir, - speakerId: speechConfig?.voiceLocalTtsSpeakerId, - speed: speechConfig?.voiceLocalTtsSpeed, + speakerId: speechConfig?.localModels.voiceTtsSpeakerId, + speed: speechConfig?.localModels.voiceTtsSpeed, }, logger ); diff --git a/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts b/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts index 9cef0f608..e7ac5589a 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts @@ -115,23 +115,27 @@ describe("speech models (download E2E)", () => { paseoHomeRoot, dictationFinalTimeoutMs: 8000, speech: { - dictationSttProvider: "local", - voiceSttProvider: "local", - voiceTtsProvider: "local", + providers: { + dictationStt: { provider: "local", explicit: true }, + voiceStt: { provider: "local", explicit: true }, + voiceTts: { provider: "local", explicit: true }, + }, local: { modelsDir, autoDownload: false, }, - dictationLocalSttModel: - set === "parakeet-pocket" - ? "parakeet-tdt-0.6b-v3-int8" - : "zipformer-bilingual-zh-en-2023-02-20", - voiceLocalSttModel: - set === "parakeet-pocket" - ? "parakeet-tdt-0.6b-v3-int8" - : "zipformer-bilingual-zh-en-2023-02-20", - voiceLocalTtsModel: - set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16", + localModels: { + dictationStt: + set === "parakeet-pocket" + ? "parakeet-tdt-0.6b-v3-int8" + : "zipformer-bilingual-zh-en-2023-02-20", + voiceStt: + set === "parakeet-pocket" + ? "parakeet-tdt-0.6b-v3-int8" + : "zipformer-bilingual-zh-en-2023-02-20", + voiceTts: + set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16", + }, }, }); diff --git a/packages/server/src/server/speech/providers/openai/config.test.ts b/packages/server/src/server/speech/providers/openai/config.test.ts new file mode 100644 index 000000000..10b04364f --- /dev/null +++ b/packages/server/src/server/speech/providers/openai/config.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, test } from "vitest"; + +import { PersistedConfigSchema } from "../../../persisted-config.js"; +import { resolveOpenAiSpeechConfig } from "./config.js"; + +describe("resolveOpenAiSpeechConfig", () => { + test("treats empty OPENAI_API_KEY as unset", () => { + const persisted = PersistedConfigSchema.parse({}); + const env = { + OPENAI_API_KEY: "", + } as NodeJS.ProcessEnv; + + const resolved = resolveOpenAiSpeechConfig({ + env, + persisted, + providers: { + dictationStt: { provider: "local", explicit: false }, + voiceStt: { provider: "local", explicit: false }, + voiceTts: { provider: "local", explicit: false }, + }, + }); + + expect(resolved).toBeUndefined(); + }); + + test("uses trimmed OPENAI_API_KEY when configured", () => { + const persisted = PersistedConfigSchema.parse({}); + const env = { + OPENAI_API_KEY: " sk-test ", + } as NodeJS.ProcessEnv; + + const resolved = resolveOpenAiSpeechConfig({ + env, + persisted, + providers: { + dictationStt: { provider: "openai", explicit: true }, + voiceStt: { provider: "openai", explicit: true }, + voiceTts: { provider: "openai", explicit: true }, + }, + }); + + expect(resolved?.apiKey).toBe("sk-test"); + expect(resolved?.stt?.apiKey).toBe("sk-test"); + expect(resolved?.tts?.apiKey).toBe("sk-test"); + }); +}); diff --git a/packages/server/src/server/speech/providers/openai/config.ts b/packages/server/src/server/speech/providers/openai/config.ts index a849c2855..b71b0ad09 100644 --- a/packages/server/src/server/speech/providers/openai/config.ts +++ b/packages/server/src/server/speech/providers/openai/config.ts @@ -35,7 +35,11 @@ const OptionalFiniteNumberSchema = NumberLikeSchema .pipe(z.coerce.number().finite()) .optional(); -const OptionalTrimmedStringSchema = z.string().trim().min(1).optional(); +const OptionalTrimmedStringSchema = z + .string() + .trim() + .optional() + .transform((value) => (value && value.length > 0 ? value : undefined)); const OpenAiSpeechResolutionSchema = z.object({ apiKey: OptionalTrimmedStringSchema, @@ -70,27 +74,27 @@ export function resolveOpenAiSpeechConfig(params: { params.persisted.features?.dictation?.stt?.confidenceThreshold, sttModel: params.env.STT_MODEL ?? - (params.providers.voiceSttProvider === "openai" + (params.providers.voiceStt.provider === "openai" ? params.persisted.features?.voiceMode?.stt?.model : undefined) ?? - (params.providers.dictationSttProvider === "openai" + (params.providers.dictationStt.provider === "openai" ? params.persisted.features?.dictation?.stt?.model : undefined), ttsVoice: params.env.TTS_VOICE ?? - (params.providers.voiceTtsProvider === "openai" + (params.providers.voiceTts.provider === "openai" ? params.persisted.features?.voiceMode?.tts?.voice : undefined) ?? "alloy", ttsModel: params.env.TTS_MODEL ?? - (params.providers.voiceTtsProvider === "openai" + (params.providers.voiceTts.provider === "openai" ? params.persisted.features?.voiceMode?.tts?.model : undefined) ?? DEFAULT_OPENAI_TTS_MODEL, realtimeTranscriptionModel: params.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ?? - (params.providers.dictationSttProvider === "openai" + (params.providers.dictationStt.provider === "openai" ? params.persisted.features?.dictation?.stt?.model : undefined) ?? DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL, diff --git a/packages/server/src/server/speech/providers/openai/runtime.ts b/packages/server/src/server/speech/providers/openai/runtime.ts index 57394496a..64d29eaf6 100644 --- a/packages/server/src/server/speech/providers/openai/runtime.ts +++ b/packages/server/src/server/speech/providers/openai/runtime.ts @@ -60,13 +60,16 @@ export function validateOpenAiCredentialRequirements(params: { const openAiCredentials = resolveOpenAiCredentials(openaiConfig); const missingOpenAiCredentialsFor: string[] = []; - if (providers.voiceSttProvider === "openai" && !openAiCredentials.openaiSttApiKey) { + if (providers.voiceStt.provider === "openai" && !openAiCredentials.openaiSttApiKey) { missingOpenAiCredentialsFor.push("voice.stt"); } - if (providers.voiceTtsProvider === "openai" && !openAiCredentials.openaiTtsApiKey) { + if (providers.voiceTts.provider === "openai" && !openAiCredentials.openaiTtsApiKey) { missingOpenAiCredentialsFor.push("voice.tts"); } - if (providers.dictationSttProvider === "openai" && !openAiCredentials.openaiDictationApiKey) { + if ( + providers.dictationStt.provider === "openai" && + !openAiCredentials.openaiDictationApiKey + ) { missingOpenAiCredentialsFor.push("dictation.stt"); } @@ -74,9 +77,9 @@ export function validateOpenAiCredentialRequirements(params: { logger.error( { requestedProviders: { - dictationStt: providers.dictationSttProvider, - voiceStt: providers.voiceSttProvider, - voiceTts: providers.voiceTtsProvider, + dictationStt: providers.dictationStt.provider, + voiceStt: providers.voiceStt.provider, + voiceTts: providers.voiceTts.provider, }, missingOpenAiCredentialsFor, }, @@ -101,9 +104,10 @@ export function initializeOpenAiSpeechServices(params: { let ttsService = existing.ttsService; let dictationSttService = existing.dictationSttService; - const needsOpenAiStt = !sttService && providers.voiceSttProvider === "openai"; - const needsOpenAiTts = !ttsService && providers.voiceTtsProvider === "openai"; - const needsOpenAiDictation = !dictationSttService && providers.dictationSttProvider === "openai"; + const needsOpenAiStt = !sttService && providers.voiceStt.provider === "openai"; + const needsOpenAiTts = !ttsService && providers.voiceTts.provider === "openai"; + const needsOpenAiDictation = + !dictationSttService && providers.dictationStt.provider === "openai"; if ( (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) && diff --git a/packages/server/src/server/speech/speech-config-resolver.test.ts b/packages/server/src/server/speech/speech-config-resolver.test.ts index bb34fdb01..211b6265f 100644 --- a/packages/server/src/server/speech/speech-config-resolver.test.ts +++ b/packages/server/src/server/speech/speech-config-resolver.test.ts @@ -18,16 +18,25 @@ describe("resolveSpeechConfig", () => { }); expect(result.openai).toBeUndefined(); - expect(result.speech.dictationSttProvider).toBe("local"); - expect(result.speech.voiceSttProvider).toBe("local"); - expect(result.speech.voiceTtsProvider).toBe("local"); + expect(result.speech.providers.dictationStt).toEqual({ + provider: "local", + explicit: false, + }); + expect(result.speech.providers.voiceStt).toEqual({ + provider: "local", + explicit: false, + }); + expect(result.speech.providers.voiceTts).toEqual({ + provider: "local", + explicit: false, + }); expect(result.speech.local).toEqual({ modelsDir: path.join(paseoHome, "models", "local-speech"), autoDownload: true, }); - expect(result.speech.dictationLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8"); - expect(result.speech.voiceLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8"); - expect(result.speech.voiceLocalTtsModel).toBe("pocket-tts-onnx-int8"); + expect(result.speech.localModels.dictationStt).toBe("parakeet-tdt-0.6b-v3-int8"); + expect(result.speech.localModels.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8"); + expect(result.speech.localModels.voiceTts).toBe("pocket-tts-onnx-int8"); }); test("resolves feature-scoped local model env vars", () => { @@ -65,11 +74,23 @@ describe("resolveSpeechConfig", () => { modelsDir: "/tmp/models", autoDownload: false, }); - expect(result.speech.dictationLocalSttModel).toBe("zipformer-bilingual-zh-en-2023-02-20"); - expect(result.speech.voiceLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8"); - expect(result.speech.voiceLocalTtsModel).toBe("kitten-nano-en-v0_1-fp16"); - expect(result.speech.voiceLocalTtsSpeakerId).toBe(5); - expect(result.speech.voiceLocalTtsSpeed).toBe(1.35); + expect(result.speech.providers.dictationStt).toEqual({ + provider: "local", + explicit: true, + }); + expect(result.speech.providers.voiceStt).toEqual({ + provider: "openai", + explicit: true, + }); + expect(result.speech.providers.voiceTts).toEqual({ + provider: "local", + explicit: true, + }); + expect(result.speech.localModels.dictationStt).toBe("zipformer-bilingual-zh-en-2023-02-20"); + expect(result.speech.localModels.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8"); + expect(result.speech.localModels.voiceTts).toBe("kitten-nano-en-v0_1-fp16"); + expect(result.speech.localModels.voiceTtsSpeakerId).toBe(5); + expect(result.speech.localModels.voiceTtsSpeed).toBe(1.35); expect(result.openai?.apiKey).toBe("env-key"); expect(result.openai?.stt?.model).toBe("gpt-4o-transcribe"); }); @@ -87,8 +108,8 @@ describe("resolveSpeechConfig", () => { persisted, }); - expect(result.speech.dictationLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8"); - expect(result.speech.voiceLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8"); - expect(result.speech.voiceLocalTtsModel).toBe("pocket-tts-onnx-int8"); + expect(result.speech.localModels.dictationStt).toBe("parakeet-tdt-0.6b-v3-int8"); + expect(result.speech.localModels.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8"); + expect(result.speech.localModels.voiceTts).toBe("pocket-tts-onnx-int8"); }); }); diff --git a/packages/server/src/server/speech/speech-config-resolver.ts b/packages/server/src/server/speech/speech-config-resolver.ts index c2b53eb15..1389e8738 100644 --- a/packages/server/src/server/speech/speech-config-resolver.ts +++ b/packages/server/src/server/speech/speech-config-resolver.ts @@ -6,6 +6,7 @@ import { resolveLocalSpeechConfig } from "./providers/local/config.js"; import { resolveOpenAiSpeechConfig } from "./providers/openai/config.js"; import { SpeechProviderIdSchema, + type RequestedSpeechProvider, type RequestedSpeechProviders, } from "./speech-types.js"; @@ -17,29 +18,53 @@ const OptionalSpeechProviderSchema = z .optional(); const RequestedSpeechProvidersSchema = z.object({ - dictationSttProvider: OptionalSpeechProviderSchema.default("local"), - voiceSttProvider: OptionalSpeechProviderSchema.default("local"), - voiceTtsProvider: OptionalSpeechProviderSchema.default("local"), + dictationStt: OptionalSpeechProviderSchema.default("local"), + voiceStt: OptionalSpeechProviderSchema.default("local"), + voiceTts: OptionalSpeechProviderSchema.default("local"), }); function resolveRequestedSpeechProviders(params: { env: NodeJS.ProcessEnv; persisted: PersistedConfig; }): RequestedSpeechProviders { - return RequestedSpeechProvidersSchema.parse({ - dictationSttProvider: - params.env.PASEO_DICTATION_STT_PROVIDER ?? - params.persisted.features?.dictation?.stt?.provider ?? - "local", - voiceSttProvider: - params.env.PASEO_VOICE_STT_PROVIDER ?? - params.persisted.features?.voiceMode?.stt?.provider ?? - "local", - voiceTtsProvider: - params.env.PASEO_VOICE_TTS_PROVIDER ?? - params.persisted.features?.voiceMode?.tts?.provider ?? - "local", + const resolveFeatureProvider = ( + configuredValue: string | undefined, + parsedValue: z.infer + ): RequestedSpeechProvider => ({ + provider: parsedValue, + explicit: configuredValue !== undefined, }); + + const dictationSttProviderFromConfig = + params.env.PASEO_DICTATION_STT_PROVIDER ?? + params.persisted.features?.dictation?.stt?.provider; + const voiceSttProviderFromConfig = + params.env.PASEO_VOICE_STT_PROVIDER ?? + params.persisted.features?.voiceMode?.stt?.provider; + const voiceTtsProviderFromConfig = + params.env.PASEO_VOICE_TTS_PROVIDER ?? + params.persisted.features?.voiceMode?.tts?.provider; + + const parsed = RequestedSpeechProvidersSchema.parse({ + dictationStt: dictationSttProviderFromConfig ?? "local", + voiceStt: voiceSttProviderFromConfig ?? "local", + voiceTts: voiceTtsProviderFromConfig ?? "local", + }); + + return { + dictationStt: resolveFeatureProvider( + dictationSttProviderFromConfig, + parsed.dictationStt + ), + voiceStt: resolveFeatureProvider( + voiceSttProviderFromConfig, + parsed.voiceStt + ), + voiceTts: resolveFeatureProvider( + voiceTtsProviderFromConfig, + parsed.voiceTts + ), + }; } export function resolveSpeechConfig(params: { @@ -71,21 +96,21 @@ export function resolveSpeechConfig(params: { return { openai, speech: { - dictationSttProvider: providers.dictationSttProvider, - voiceSttProvider: providers.voiceSttProvider, - voiceTtsProvider: providers.voiceTtsProvider, + providers, ...(local.local ? { local: local.local } : {}), - dictationLocalSttModel: local.dictationLocalSttModel, - voiceLocalSttModel: local.voiceLocalSttModel, - voiceLocalTtsModel: local.voiceLocalTtsModel, - ...(local.voiceLocalTtsSpeakerId !== undefined - ? { voiceLocalTtsSpeakerId: local.voiceLocalTtsSpeakerId } - : {}), - ...(local.voiceLocalTtsSpeed !== undefined - ? { voiceLocalTtsSpeed: local.voiceLocalTtsSpeed } - : {}), + localModels: { + dictationStt: local.dictationLocalSttModel, + voiceStt: local.voiceLocalSttModel, + voiceTts: local.voiceLocalTtsModel, + ...(local.voiceLocalTtsSpeakerId !== undefined + ? { voiceTtsSpeakerId: local.voiceLocalTtsSpeakerId } + : {}), + ...(local.voiceLocalTtsSpeed !== undefined + ? { voiceTtsSpeed: local.voiceLocalTtsSpeed } + : {}), + }, }, }; } diff --git a/packages/server/src/server/speech/speech-runtime.ts b/packages/server/src/server/speech/speech-runtime.ts index 363eb0ebb..9b1280cbd 100644 --- a/packages/server/src/server/speech/speech-runtime.ts +++ b/packages/server/src/server/speech/speech-runtime.ts @@ -17,10 +17,15 @@ import type { RequestedSpeechProviders } from "./speech-types.js"; function resolveRequestedSpeechProviders( speechConfig: PaseoSpeechConfig | null ): RequestedSpeechProviders { + const fromConfig = speechConfig?.providers; + if (fromConfig) { + return fromConfig; + } + return { - dictationSttProvider: speechConfig?.dictationSttProvider ?? "local", - voiceSttProvider: speechConfig?.voiceSttProvider ?? "local", - voiceTtsProvider: speechConfig?.voiceTtsProvider ?? "local", + dictationStt: { provider: "local", explicit: false }, + voiceStt: { provider: "local", explicit: false }, + voiceTts: { provider: "local", explicit: false }, }; } @@ -54,9 +59,9 @@ export async function initializeSpeechRuntime(params: { logger.info( { requestedProviders: { - dictationStt: providers.dictationSttProvider, - voiceStt: providers.voiceSttProvider, - voiceTts: providers.voiceTtsProvider, + dictationStt: providers.dictationStt.provider, + voiceStt: providers.voiceStt.provider, + voiceTts: providers.voiceTts.provider, }, availability: { openai: getOpenAiSpeechAvailability(openaiConfig), @@ -98,30 +103,66 @@ export async function initializeSpeechRuntime(params: { !openAiSpeech.sttService ? "voice.stt" : null, !openAiSpeech.ttsService ? "voice.tts" : null, ].filter((feature): feature is string => feature !== null); + const explicitlyConfiguredUnavailableFeatures = unavailableFeatures.filter((feature) => { + if (feature === "dictation.stt") { + return providers.dictationStt.explicit; + } + if (feature === "voice.stt") { + return providers.voiceStt.explicit; + } + return providers.voiceTts.explicit; + }); - if (unavailableFeatures.length > 0) { + if (explicitlyConfiguredUnavailableFeatures.length > 0) { logger.error( { requestedProviders: { - dictationStt: providers.dictationSttProvider, - voiceStt: providers.voiceSttProvider, - voiceTts: providers.voiceTtsProvider, + dictationStt: providers.dictationStt.provider, + voiceStt: providers.voiceStt.provider, + voiceTts: providers.voiceTts.provider, + }, + explicitProviders: { + dictationStt: providers.dictationStt.explicit, + voiceStt: providers.voiceStt.explicit, + voiceTts: providers.voiceTts.explicit, + }, + effectiveProviders, + unavailableFeatures: explicitlyConfiguredUnavailableFeatures, + }, + "Speech provider reconciliation failed: configured features are unavailable" + ); + throw new Error( + `Configured speech features unavailable: ${explicitlyConfiguredUnavailableFeatures.join(", ")}` + ); + } + + if (unavailableFeatures.length > 0) { + logger.warn( + { + requestedProviders: { + dictationStt: providers.dictationStt.provider, + voiceStt: providers.voiceStt.provider, + voiceTts: providers.voiceTts.provider, + }, + explicitProviders: { + dictationStt: providers.dictationStt.explicit, + voiceStt: providers.voiceStt.explicit, + voiceTts: providers.voiceTts.explicit, }, effectiveProviders, unavailableFeatures, }, - "Speech provider reconciliation failed: configured features are unavailable" + "Speech provider reconciliation completed with unavailable default features" + ); + } else { + logger.info( + { + effectiveProviders, + }, + "Speech provider reconciliation completed" ); - throw new Error(`Configured speech features unavailable: ${unavailableFeatures.join(", ")}`); } - logger.info( - { - effectiveProviders, - }, - "Speech provider reconciliation completed" - ); - return { sttService: openAiSpeech.sttService, ttsService: openAiSpeech.ttsService, diff --git a/packages/server/src/server/speech/speech-types.ts b/packages/server/src/server/speech/speech-types.ts index d7ffd32d7..d9606addc 100644 --- a/packages/server/src/server/speech/speech-types.ts +++ b/packages/server/src/server/speech/speech-types.ts @@ -3,8 +3,14 @@ import { z } from "zod"; export const SpeechProviderIdSchema = z.enum(["openai", "local"]); export type SpeechProviderId = z.infer; +export const RequestedSpeechProviderSchema = z.object({ + provider: SpeechProviderIdSchema, + explicit: z.boolean(), +}); +export type RequestedSpeechProvider = z.infer; + export type RequestedSpeechProviders = { - dictationSttProvider: SpeechProviderId; - voiceSttProvider: SpeechProviderId; - voiceTtsProvider: SpeechProviderId; + dictationStt: RequestedSpeechProvider; + voiceStt: RequestedSpeechProvider; + voiceTts: RequestedSpeechProvider; }; diff --git a/packages/server/src/server/voice-local-agent.e2e.test.ts b/packages/server/src/server/voice-local-agent.e2e.test.ts index e61ab19e4..8854d7090 100644 --- a/packages/server/src/server/voice-local-agent.e2e.test.ts +++ b/packages/server/src/server/voice-local-agent.e2e.test.ts @@ -48,9 +48,16 @@ function waitForSignal( agentClients: {}, openai: { apiKey: openaiApiKey! }, speech: { - dictationSttProvider: "openai", - voiceSttProvider: "openai", - voiceTtsProvider: "openai", + providers: { + dictationStt: { provider: "openai", explicit: true }, + voiceStt: { provider: "openai", explicit: true }, + voiceTts: { provider: "openai", explicit: true }, + }, + localModels: { + dictationStt: "parakeet-tdt-0.6b-v3-int8", + voiceStt: "parakeet-tdt-0.6b-v3-int8", + voiceTts: "pocket-tts-onnx-int8", + }, }, voiceLlmProvider: "codex", voiceLlmProviderExplicit: true,