From afb1f1580d13134ce53a49e6995382da3224a652 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 6 Feb 2026 22:19:12 +0700 Subject: [PATCH] speech: move defaults and runtime into provider modules --- packages/server/src/server/bootstrap.ts | 17 +- .../server/speech/providers/local/config.ts | 176 ++++++ .../server/speech/providers/local/runtime.ts | 372 ++++++++++++ .../providers/local/sherpa/model-catalog.ts | 1 + .../server/speech/providers/openai/config.ts | 132 +++++ .../server/speech/providers/openai/runtime.ts | 167 ++++++ .../server/speech/speech-config-resolver.ts | 302 +++------- .../src/server/speech/speech-defaults.ts | 9 - .../src/server/speech/speech-runtime.ts | 534 +----------------- .../server/src/server/speech/speech-types.ts | 6 + 10 files changed, 938 insertions(+), 778 deletions(-) create mode 100644 packages/server/src/server/speech/providers/local/config.ts create mode 100644 packages/server/src/server/speech/providers/local/runtime.ts create mode 100644 packages/server/src/server/speech/providers/openai/config.ts create mode 100644 packages/server/src/server/speech/providers/openai/runtime.ts delete mode 100644 packages/server/src/server/speech/speech-defaults.ts diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 12d6d61ac..19aece6c0 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -41,8 +41,8 @@ function parseListenString(listen: string): ListenTarget { import { VoiceAssistantWebSocketServer } from "./websocket-server.js"; import { DownloadTokenStore } from "./file-download/token-store.js"; -import type { STTConfig } from "./speech/providers/openai/stt.js"; -import type { TTSConfig } from "./speech/providers/openai/tts.js"; +import type { OpenAiSpeechProviderConfig } from "./speech/providers/openai/config.js"; +import type { LocalSpeechProviderConfig } from "./speech/providers/local/config.js"; import { initializeSpeechRuntime } from "./speech/speech-runtime.js"; import { AgentManager } from "./agent/agent-manager.js"; import { AgentStorage } from "./agent/agent-storage.js"; @@ -101,17 +101,8 @@ function resolveVoiceMcpBridgeCommand(logger: Logger): { command: string; baseAr return resolved; } -export type PaseoOpenAIConfig = { - apiKey?: string; - stt?: Partial & { apiKey?: string }; - tts?: Partial & { apiKey?: string }; - realtimeTranscriptionModel?: string; -}; - -export type PaseoLocalSpeechConfig = { - modelsDir: string; - autoDownload?: boolean; -}; +export type PaseoOpenAIConfig = OpenAiSpeechProviderConfig; +export type PaseoLocalSpeechConfig = LocalSpeechProviderConfig; export type PaseoSpeechConfig = { dictationSttProvider?: "openai" | "local"; diff --git a/packages/server/src/server/speech/providers/local/config.ts b/packages/server/src/server/speech/providers/local/config.ts new file mode 100644 index 000000000..e1905bb57 --- /dev/null +++ b/packages/server/src/server/speech/providers/local/config.ts @@ -0,0 +1,176 @@ +import path from "node:path"; + +import { z } from "zod"; + +import type { PersistedConfig } from "../../../persisted-config.js"; +import type { RequestedSpeechProviders } from "../../speech-types.js"; +import { + DEFAULT_LOCAL_STT_MODEL, + DEFAULT_LOCAL_TTS_MODEL, + LocalSttModelIdSchema, + LocalTtsModelIdSchema, + type LocalSpeechModelId, + type LocalSttModelId, + type LocalTtsModelId, +} from "./sherpa/model-catalog.js"; + +export type LocalSpeechProviderConfig = { + modelsDir: string; + autoDownload?: boolean; +}; + +export type ResolvedLocalSpeechConfig = { + local: LocalSpeechProviderConfig | undefined; + dictationLocalSttModel: LocalSttModelId; + voiceLocalSttModel: LocalSttModelId; + voiceLocalTtsModel: LocalTtsModelId; + voiceLocalTtsSpeakerId?: number; + voiceLocalTtsSpeed?: number; +}; + +export type { LocalSpeechModelId, LocalSttModelId, LocalTtsModelId }; + +const DEFAULT_LOCAL_MODELS_SUBDIR = path.join("models", "local-speech"); + +const OptionalBooleanFlagSchema = z.preprocess((value) => { + if (typeof value === "boolean") { + return value; + } + if (typeof value !== "string") { + return value; + } + const normalized = value.trim().toLowerCase(); + if (normalized === "1" || normalized === "true" || normalized === "yes") { + return true; + } + if (normalized === "0" || normalized === "false" || normalized === "no") { + return false; + } + return undefined; +}, z.boolean().optional()); + +const OptionalFiniteNumberSchema = z.preprocess((value) => { + if (typeof value === "number") { + return Number.isFinite(value) ? value : undefined; + } + if (typeof value !== "string") { + return value; + } + const parsed = Number.parseFloat(value); + return Number.isFinite(parsed) ? parsed : undefined; +}, z.number().optional()); + +const OptionalIntegerSchema = z.preprocess((value) => { + if (typeof value === "number") { + return Number.isInteger(value) ? value : undefined; + } + if (typeof value !== "string") { + return value; + } + const parsed = Number.parseInt(value, 10); + return Number.isFinite(parsed) ? parsed : undefined; +}, z.number().int().optional()); + +const LocalSpeechResolutionSchema = z.object({ + includeProviderConfig: z.boolean(), + modelsDir: z.string().trim().min(1), + autoDownload: OptionalBooleanFlagSchema.default(true), + dictationLocalSttModel: LocalSttModelIdSchema.default(DEFAULT_LOCAL_STT_MODEL), + voiceLocalSttModel: LocalSttModelIdSchema.default(DEFAULT_LOCAL_STT_MODEL), + voiceLocalTtsModel: LocalTtsModelIdSchema.default(DEFAULT_LOCAL_TTS_MODEL), + voiceLocalTtsSpeakerId: OptionalIntegerSchema, + voiceLocalTtsSpeed: OptionalFiniteNumberSchema, +}); + +function persistedLocalFeatureModel( + provider: RequestedSpeechProviders[keyof RequestedSpeechProviders], + model: string | undefined +): string | undefined { + if (provider !== "local") { + return undefined; + } + return model; +} + +function shouldIncludeLocalProviderConfig(params: { + providers: RequestedSpeechProviders; + env: NodeJS.ProcessEnv; + persisted: PersistedConfig; +}): boolean { + const localRequestedByFeature = + params.providers.dictationSttProvider === "local" || + params.providers.voiceSttProvider === "local" || + params.providers.voiceTtsProvider === "local"; + + return ( + localRequestedByFeature || + params.env.PASEO_LOCAL_MODELS_DIR !== undefined || + params.persisted.providers?.local !== undefined + ); +} + +export function resolveLocalSpeechConfig(params: { + paseoHome: string; + env: NodeJS.ProcessEnv; + persisted: PersistedConfig; + providers: RequestedSpeechProviders; +}): ResolvedLocalSpeechConfig { + const includeProviderConfig = shouldIncludeLocalProviderConfig(params); + + const parsed = LocalSpeechResolutionSchema.parse({ + includeProviderConfig, + modelsDir: + params.env.PASEO_LOCAL_MODELS_DIR ?? + params.persisted.providers?.local?.modelsDir ?? + path.join(params.paseoHome, DEFAULT_LOCAL_MODELS_SUBDIR), + autoDownload: + params.env.PASEO_LOCAL_AUTO_DOWNLOAD ?? + params.persisted.providers?.local?.autoDownload, + dictationLocalSttModel: + params.env.PASEO_DICTATION_LOCAL_STT_MODEL ?? + persistedLocalFeatureModel( + params.providers.dictationSttProvider, + params.persisted.features?.dictation?.stt?.model + ) ?? + DEFAULT_LOCAL_STT_MODEL, + voiceLocalSttModel: + params.env.PASEO_VOICE_LOCAL_STT_MODEL ?? + persistedLocalFeatureModel( + params.providers.voiceSttProvider, + params.persisted.features?.voiceMode?.stt?.model + ) ?? + DEFAULT_LOCAL_STT_MODEL, + voiceLocalTtsModel: + params.env.PASEO_VOICE_LOCAL_TTS_MODEL ?? + persistedLocalFeatureModel( + params.providers.voiceTtsProvider, + params.persisted.features?.voiceMode?.tts?.model + ) ?? + DEFAULT_LOCAL_TTS_MODEL, + voiceLocalTtsSpeakerId: + params.env.PASEO_VOICE_LOCAL_TTS_SPEAKER_ID ?? + params.persisted.features?.voiceMode?.tts?.speakerId, + voiceLocalTtsSpeed: + params.env.PASEO_VOICE_LOCAL_TTS_SPEED ?? + params.persisted.features?.voiceMode?.tts?.speed, + }); + + return { + local: + parsed.includeProviderConfig + ? { + modelsDir: parsed.modelsDir, + autoDownload: parsed.autoDownload, + } + : undefined, + dictationLocalSttModel: parsed.dictationLocalSttModel, + voiceLocalSttModel: parsed.voiceLocalSttModel, + voiceLocalTtsModel: parsed.voiceLocalTtsModel, + ...(parsed.voiceLocalTtsSpeakerId !== undefined + ? { voiceLocalTtsSpeakerId: parsed.voiceLocalTtsSpeakerId } + : {}), + ...(parsed.voiceLocalTtsSpeed !== undefined + ? { voiceLocalTtsSpeed: parsed.voiceLocalTtsSpeed } + : {}), + }; +} diff --git a/packages/server/src/server/speech/providers/local/runtime.ts b/packages/server/src/server/speech/providers/local/runtime.ts new file mode 100644 index 000000000..a060ac587 --- /dev/null +++ b/packages/server/src/server/speech/providers/local/runtime.ts @@ -0,0 +1,372 @@ +import type { Logger } from "pino"; + +import type { PaseoSpeechConfig } from "../../../bootstrap.js"; +import type { SpeechToTextProvider, TextToSpeechProvider } from "../../speech-provider.js"; +import type { RequestedSpeechProviders } from "../../speech-types.js"; +import { PocketTtsOnnxTTS } from "./pocket/pocket-tts-onnx.js"; +import { + ensureSherpaOnnxModels, + getSherpaOnnxModelDir, +} from "./sherpa/model-downloader.js"; +import { + DEFAULT_LOCAL_STT_MODEL, + DEFAULT_LOCAL_TTS_MODEL, + LocalSttModelIdSchema, + LocalTtsModelIdSchema, + type LocalSpeechModelId, + type LocalSttModelId, + type LocalTtsModelId, +} from "./sherpa/model-catalog.js"; +import { SherpaOfflineRecognizerEngine } from "./sherpa/sherpa-offline-recognizer.js"; +import { SherpaOnlineRecognizerEngine } from "./sherpa/sherpa-online-recognizer.js"; +import { SherpaOnnxParakeetSTT } from "./sherpa/sherpa-parakeet-stt.js"; +import { SherpaParakeetRealtimeTranscriptionSession } from "./sherpa/sherpa-parakeet-realtime-session.js"; +import { SherpaRealtimeTranscriptionSession } from "./sherpa/sherpa-realtime-session.js"; +import { SherpaOnnxSTT } from "./sherpa/sherpa-stt.js"; +import { SherpaOnnxTTS } from "./sherpa/sherpa-tts.js"; + +type LocalSttEngine = + | { kind: "offline"; engine: SherpaOfflineRecognizerEngine } + | { kind: "online"; engine: SherpaOnlineRecognizerEngine }; + +type ResolvedLocalModels = { + dictationLocalSttModel: LocalSttModelId; + voiceLocalSttModel: LocalSttModelId; + voiceLocalTtsModel: LocalTtsModelId; +}; + +type LocalSpeechAvailability = { + configured: boolean; + modelsDir: string | null; + autoDownload: boolean | null; +}; + +export type InitializedLocalSpeech = { + sttService: SpeechToTextProvider | null; + ttsService: TextToSpeechProvider | null; + dictationSttService: SpeechToTextProvider | null; + localVoiceTtsProvider: TextToSpeechProvider | null; + localModelConfig: { + modelsDir: string; + defaultModelIds: LocalSpeechModelId[]; + } | null; + availability: LocalSpeechAvailability; + cleanup: () => void; +}; + +function buildModelDownloadHint(modelId: LocalSpeechModelId): string { + return `Use 'paseo speech download --model ${modelId}' to download this model.`; +} + +function resolveConfiguredLocalModels( + speechConfig: PaseoSpeechConfig | null +): ResolvedLocalModels { + return { + dictationLocalSttModel: LocalSttModelIdSchema.parse( + speechConfig?.dictationLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL + ), + voiceLocalSttModel: LocalSttModelIdSchema.parse( + speechConfig?.voiceLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL + ), + voiceLocalTtsModel: LocalTtsModelIdSchema.parse( + speechConfig?.voiceLocalTtsModel ?? DEFAULT_LOCAL_TTS_MODEL + ), + }; +} + +export function getLocalSpeechAvailability( + speechConfig: PaseoSpeechConfig | null +): LocalSpeechAvailability { + const localConfig = speechConfig?.local ?? null; + return { + configured: Boolean(localConfig), + modelsDir: localConfig?.modelsDir ?? null, + autoDownload: localConfig?.autoDownload ?? null, + }; +} + +function computeRequiredLocalModelIds(params: { + providers: RequestedSpeechProviders; + models: ResolvedLocalModels; +}): LocalSpeechModelId[] { + const ids = new Set(); + if (params.providers.dictationSttProvider === "local") { + ids.add(params.models.dictationLocalSttModel); + } + if (params.providers.voiceSttProvider === "local") { + ids.add(params.models.voiceLocalSttModel); + } + if (params.providers.voiceTtsProvider === "local") { + ids.add(params.models.voiceLocalTtsModel); + } + return Array.from(ids); +} + +async function createLocalSttEngine(params: { + modelId: LocalSttModelId; + modelsDir: string; + logger: Logger; +}): Promise { + const { modelId, modelsDir, logger } = params; + + if (modelId === "parakeet-tdt-0.6b-v3-int8") { + const modelDir = getSherpaOnnxModelDir(modelsDir, modelId); + return { + kind: "offline", + engine: new SherpaOfflineRecognizerEngine( + { + model: { + kind: "nemo_transducer", + encoder: `${modelDir}/encoder.int8.onnx`, + decoder: `${modelDir}/decoder.int8.onnx`, + joiner: `${modelDir}/joiner.int8.onnx`, + tokens: `${modelDir}/tokens.txt`, + }, + numThreads: 2, + debug: 0, + }, + logger + ), + }; + } + + if (modelId === "paraformer-bilingual-zh-en") { + const modelDir = getSherpaOnnxModelDir(modelsDir, modelId); + return { + kind: "online", + engine: new SherpaOnlineRecognizerEngine( + { + model: { + kind: "paraformer", + encoder: `${modelDir}/encoder.int8.onnx`, + decoder: `${modelDir}/decoder.int8.onnx`, + tokens: `${modelDir}/tokens.txt`, + }, + numThreads: 1, + debug: 0, + }, + logger + ), + }; + } + + if (modelId === "zipformer-bilingual-zh-en-2023-02-20") { + const modelDir = getSherpaOnnxModelDir(modelsDir, modelId); + return { + kind: "online", + engine: new SherpaOnlineRecognizerEngine( + { + model: { + kind: "transducer", + encoder: `${modelDir}/encoder-epoch-99-avg-1.onnx`, + decoder: `${modelDir}/decoder-epoch-99-avg-1.onnx`, + joiner: `${modelDir}/joiner-epoch-99-avg-1.onnx`, + tokens: `${modelDir}/tokens.txt`, + modelType: "zipformer", + }, + numThreads: 1, + debug: 0, + }, + logger + ), + }; + } + + throw new Error(`Unsupported local STT model '${modelId}'`); +} + +export async function initializeLocalSpeechServices(params: { + providers: RequestedSpeechProviders; + speechConfig: PaseoSpeechConfig | null; + logger: Logger; +}): Promise { + const { providers, logger, speechConfig } = params; + const localConfig = speechConfig?.local ?? null; + const localModels = resolveConfiguredLocalModels(speechConfig); + + let sttService: SpeechToTextProvider | null = null; + let ttsService: TextToSpeechProvider | null = null; + let dictationSttService: SpeechToTextProvider | null = null; + let localVoiceTtsProvider: TextToSpeechProvider | null = null; + + const requiredLocalModelIds = computeRequiredLocalModelIds({ + providers, + models: localModels, + }); + + if (requiredLocalModelIds.length > 0 && localConfig) { + try { + logger.info( + { + modelsDir: localConfig.modelsDir, + modelIds: requiredLocalModelIds, + autoDownload: localConfig.autoDownload ?? true, + }, + "Ensuring local speech models" + ); + await ensureSherpaOnnxModels({ + modelsDir: localConfig.modelsDir, + modelIds: requiredLocalModelIds, + autoDownload: localConfig.autoDownload ?? true, + logger, + }); + } catch (err) { + logger.error( + { + err, + modelsDir: localConfig.modelsDir, + modelIds: requiredLocalModelIds, + autoDownload: localConfig.autoDownload ?? true, + hint: + "Use `paseo speech models` to inspect status and " + + "`paseo speech download --model ` to fetch missing models.", + }, + "Failed to ensure local speech models" + ); + } + } + + const localSttEngines = new Map(); + + const getLocalSttEngine = async ( + modelId: LocalSttModelId + ): Promise => { + const existing = localSttEngines.get(modelId); + if (existing) { + return existing; + } + if (!localConfig) { + return null; + } + try { + const created = await createLocalSttEngine({ + modelId, + modelsDir: localConfig.modelsDir, + logger, + }); + localSttEngines.set(modelId, created); + return created; + } catch (err) { + logger.error( + { + err, + modelsDir: localConfig.modelsDir, + modelId, + hint: buildModelDownloadHint(modelId), + }, + "Failed to initialize local STT engine (models missing or invalid)" + ); + return null; + } + }; + + if (providers.voiceSttProvider === "local") { + if (!localConfig) { + logger.warn( + { configured: false }, + "Local STT selected for voice but local provider config is missing; STT will be unavailable" + ); + } else { + const voiceEngine = await getLocalSttEngine(localModels.voiceLocalSttModel); + if (voiceEngine?.kind === "offline") { + sttService = new SherpaOnnxParakeetSTT({ engine: voiceEngine.engine }, logger); + } else if (voiceEngine?.kind === "online") { + sttService = new SherpaOnnxSTT({ engine: voiceEngine.engine }, logger); + } + } + } + + if (providers.dictationSttProvider === "local") { + if (!localConfig) { + logger.warn( + { configured: false }, + "Local STT selected for dictation but local provider config is missing; dictation STT will be unavailable" + ); + } else { + const dictationEngine = await getLocalSttEngine(localModels.dictationLocalSttModel); + if (dictationEngine?.kind === "offline") { + dictationSttService = { + id: "local", + createSession: () => + new SherpaParakeetRealtimeTranscriptionSession({ engine: dictationEngine.engine }), + }; + } else if (dictationEngine?.kind === "online") { + dictationSttService = { + id: "local", + createSession: () => new SherpaRealtimeTranscriptionSession({ engine: dictationEngine.engine }), + }; + } + } + } + + if (providers.voiceTtsProvider === "local") { + if (!localConfig) { + logger.warn( + { configured: false }, + "Local TTS selected for voice but local provider config is missing; TTS will be unavailable" + ); + } else { + try { + if (localModels.voiceLocalTtsModel === "pocket-tts-onnx-int8") { + const modelDir = getSherpaOnnxModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel); + localVoiceTtsProvider = await PocketTtsOnnxTTS.create( + { + modelDir, + precision: "int8", + targetChunkMs: 50, + }, + logger + ); + } else { + const modelDir = getSherpaOnnxModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel); + localVoiceTtsProvider = new SherpaOnnxTTS( + { + preset: localModels.voiceLocalTtsModel, + modelDir, + speakerId: speechConfig?.voiceLocalTtsSpeakerId, + speed: speechConfig?.voiceLocalTtsSpeed, + }, + logger + ); + } + ttsService = localVoiceTtsProvider; + } catch (err) { + logger.error( + { + err, + modelsDir: localConfig.modelsDir, + modelId: localModels.voiceLocalTtsModel, + hint: buildModelDownloadHint(localModels.voiceLocalTtsModel), + }, + "Failed to initialize local TTS engine (models missing or invalid)" + ); + } + } + } + + const cleanup = () => { + const maybeFreeable = localVoiceTtsProvider as unknown as { free?: () => void } | null; + if (typeof maybeFreeable?.free === "function") { + maybeFreeable.free(); + } + for (const engine of localSttEngines.values()) { + engine.engine.free(); + } + }; + + return { + sttService, + ttsService, + dictationSttService, + localVoiceTtsProvider, + localModelConfig: + localConfig + ? { + modelsDir: localConfig.modelsDir, + defaultModelIds: requiredLocalModelIds, + } + : null, + availability: getLocalSpeechAvailability(speechConfig), + cleanup, + }; +} diff --git a/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts b/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts index 10d85c2e2..62c8bc16f 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts @@ -115,6 +115,7 @@ export const SHERPA_ONNX_MODEL_CATALOG = { } as const satisfies Record; export type SherpaOnnxModelId = keyof typeof SHERPA_ONNX_MODEL_CATALOG; +export type LocalSpeechModelId = SherpaOnnxModelId; type ModelIdByKind = { [Id in SherpaOnnxModelId]: (typeof SHERPA_ONNX_MODEL_CATALOG)[Id]["kind"] extends K diff --git a/packages/server/src/server/speech/providers/openai/config.ts b/packages/server/src/server/speech/providers/openai/config.ts new file mode 100644 index 000000000..06a88ee27 --- /dev/null +++ b/packages/server/src/server/speech/providers/openai/config.ts @@ -0,0 +1,132 @@ +import { z } from "zod"; + +import type { PersistedConfig } from "../../../persisted-config.js"; +import type { RequestedSpeechProviders } from "../../speech-types.js"; +import type { STTConfig } from "./stt.js"; +import type { TTSConfig } from "./tts.js"; + +export const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe"; +export const DEFAULT_OPENAI_TTS_MODEL = "tts-1"; + +export type OpenAiSpeechProviderConfig = { + apiKey?: string; + stt?: Partial & { apiKey?: string }; + tts?: Partial & { apiKey?: string }; + realtimeTranscriptionModel?: string; +}; + +const OpenAiTtsVoiceSchema = z.enum([ + "alloy", + "echo", + "fable", + "onyx", + "nova", + "shimmer", +]); + +const OpenAiTtsModelSchema = z.enum(["tts-1", "tts-1-hd"]); + +const OptionalFiniteNumberSchema = z.preprocess((value) => { + if (typeof value === "number") { + return Number.isFinite(value) ? value : undefined; + } + if (typeof value !== "string") { + return value; + } + const parsed = Number.parseFloat(value); + return Number.isFinite(parsed) ? parsed : undefined; +}, z.number().optional()); + +const OptionalTrimmedStringSchema = z.preprocess((value) => { + if (typeof value !== "string") { + return value; + } + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : undefined; +}, z.string().optional()); + +const OpenAiSpeechResolutionSchema = z.object({ + apiKey: OptionalTrimmedStringSchema, + sttConfidenceThreshold: OptionalFiniteNumberSchema, + sttModel: OptionalTrimmedStringSchema, + ttsVoice: z.preprocess((value) => { + if (typeof value !== "string") { + return value; + } + const normalized = value.trim().toLowerCase(); + return normalized.length > 0 ? normalized : undefined; + }, OpenAiTtsVoiceSchema.default("alloy")), + ttsModel: z.preprocess((value) => { + if (typeof value !== "string") { + return value; + } + const normalized = value.trim().toLowerCase(); + return normalized.length > 0 ? normalized : undefined; + }, OpenAiTtsModelSchema.default(DEFAULT_OPENAI_TTS_MODEL)), + realtimeTranscriptionModel: OptionalTrimmedStringSchema.default( + DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL + ), +}); + +export function resolveOpenAiSpeechConfig(params: { + env: NodeJS.ProcessEnv; + persisted: PersistedConfig; + providers: RequestedSpeechProviders; +}): OpenAiSpeechProviderConfig | undefined { + const parsed = OpenAiSpeechResolutionSchema.parse({ + apiKey: params.env.OPENAI_API_KEY ?? params.persisted.providers?.openai?.apiKey, + sttConfidenceThreshold: + params.env.STT_CONFIDENCE_THRESHOLD ?? + params.persisted.features?.dictation?.stt?.confidenceThreshold, + sttModel: + params.env.STT_MODEL ?? + (params.providers.voiceSttProvider === "openai" + ? params.persisted.features?.voiceMode?.stt?.model + : undefined) ?? + (params.providers.dictationSttProvider === "openai" + ? params.persisted.features?.dictation?.stt?.model + : undefined), + ttsVoice: + params.env.TTS_VOICE ?? + (params.providers.voiceTtsProvider === "openai" + ? params.persisted.features?.voiceMode?.tts?.voice + : undefined) ?? + "alloy", + ttsModel: + params.env.TTS_MODEL ?? + (params.providers.voiceTtsProvider === "openai" + ? params.persisted.features?.voiceMode?.tts?.model + : undefined) ?? + DEFAULT_OPENAI_TTS_MODEL, + realtimeTranscriptionModel: + params.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ?? + (params.providers.dictationSttProvider === "openai" + ? params.persisted.features?.dictation?.stt?.model + : undefined) ?? + DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL, + }); + + if (!parsed.apiKey) { + return undefined; + } + + return { + apiKey: parsed.apiKey, + stt: { + apiKey: parsed.apiKey, + ...(parsed.sttConfidenceThreshold !== undefined + ? { confidenceThreshold: parsed.sttConfidenceThreshold } + : {}), + ...(parsed.sttModel + ? { model: parsed.sttModel } + : {}), + }, + tts: { + apiKey: parsed.apiKey, + voice: parsed.ttsVoice, + model: parsed.ttsModel, + responseFormat: "pcm", + }, + realtimeTranscriptionModel: parsed.realtimeTranscriptionModel, + }; +} diff --git a/packages/server/src/server/speech/providers/openai/runtime.ts b/packages/server/src/server/speech/providers/openai/runtime.ts new file mode 100644 index 000000000..57394496a --- /dev/null +++ b/packages/server/src/server/speech/providers/openai/runtime.ts @@ -0,0 +1,167 @@ +import type { Logger } from "pino"; + +import type { SpeechToTextProvider, TextToSpeechProvider } from "../../speech-provider.js"; +import type { RequestedSpeechProviders } from "../../speech-types.js"; +import { + DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL, + DEFAULT_OPENAI_TTS_MODEL, + type OpenAiSpeechProviderConfig, +} from "./config.js"; +import { OpenAIRealtimeTranscriptionSession } from "./realtime-transcription-session.js"; +import { OpenAISTT } from "./stt.js"; +import { OpenAITTS } from "./tts.js"; + +type OpenAiCredentialState = { + openaiSttApiKey: string | undefined; + openaiTtsApiKey: string | undefined; + openaiDictationApiKey: string | undefined; +}; + +export type OpenAiSpeechAvailability = { + stt: boolean; + tts: boolean; + dictationStt: boolean; +}; + +export type SpeechServices = { + sttService: SpeechToTextProvider | null; + ttsService: TextToSpeechProvider | null; + dictationSttService: SpeechToTextProvider | null; +}; + +function resolveOpenAiCredentials( + openaiConfig: OpenAiSpeechProviderConfig | undefined +): OpenAiCredentialState { + const openaiApiKey = openaiConfig?.apiKey; + return { + openaiSttApiKey: openaiConfig?.stt?.apiKey ?? openaiApiKey, + openaiTtsApiKey: openaiConfig?.tts?.apiKey ?? openaiApiKey, + openaiDictationApiKey: openaiApiKey, + }; +} + +export function getOpenAiSpeechAvailability( + openaiConfig: OpenAiSpeechProviderConfig | undefined +): OpenAiSpeechAvailability { + const credentials = resolveOpenAiCredentials(openaiConfig); + return { + stt: Boolean(credentials.openaiSttApiKey), + tts: Boolean(credentials.openaiTtsApiKey), + dictationStt: Boolean(credentials.openaiDictationApiKey), + }; +} + +export function validateOpenAiCredentialRequirements(params: { + providers: RequestedSpeechProviders; + openaiConfig: OpenAiSpeechProviderConfig | undefined; + logger: Logger; +}): void { + const { providers, logger, openaiConfig } = params; + const openAiCredentials = resolveOpenAiCredentials(openaiConfig); + + const missingOpenAiCredentialsFor: string[] = []; + if (providers.voiceSttProvider === "openai" && !openAiCredentials.openaiSttApiKey) { + missingOpenAiCredentialsFor.push("voice.stt"); + } + if (providers.voiceTtsProvider === "openai" && !openAiCredentials.openaiTtsApiKey) { + missingOpenAiCredentialsFor.push("voice.tts"); + } + if (providers.dictationSttProvider === "openai" && !openAiCredentials.openaiDictationApiKey) { + missingOpenAiCredentialsFor.push("dictation.stt"); + } + + if (missingOpenAiCredentialsFor.length > 0) { + logger.error( + { + requestedProviders: { + dictationStt: providers.dictationSttProvider, + voiceStt: providers.voiceSttProvider, + voiceTts: providers.voiceTtsProvider, + }, + missingOpenAiCredentialsFor, + }, + "Invalid speech configuration: OpenAI provider selected but credentials are missing" + ); + throw new Error( + `Missing OpenAI credentials for configured speech features: ${missingOpenAiCredentialsFor.join(", ")}` + ); + } +} + +export function initializeOpenAiSpeechServices(params: { + providers: RequestedSpeechProviders; + openaiConfig: OpenAiSpeechProviderConfig | undefined; + existing: SpeechServices; + logger: Logger; +}): SpeechServices { + const { providers, openaiConfig, existing, logger } = params; + const openAiCredentials = resolveOpenAiCredentials(openaiConfig); + + let sttService = existing.sttService; + let ttsService = existing.ttsService; + let dictationSttService = existing.dictationSttService; + + const needsOpenAiStt = !sttService && providers.voiceSttProvider === "openai"; + const needsOpenAiTts = !ttsService && providers.voiceTtsProvider === "openai"; + const needsOpenAiDictation = !dictationSttService && providers.dictationSttProvider === "openai"; + + if ( + (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) && + (openAiCredentials.openaiSttApiKey || + openAiCredentials.openaiTtsApiKey || + openAiCredentials.openaiDictationApiKey) + ) { + logger.info("OpenAI speech provider initialized"); + + if (needsOpenAiStt && openAiCredentials.openaiSttApiKey) { + const { apiKey: _sttApiKey, ...sttConfig } = openaiConfig?.stt ?? {}; + sttService = new OpenAISTT( + { + apiKey: openAiCredentials.openaiSttApiKey, + ...sttConfig, + }, + logger + ); + } + + if (needsOpenAiTts && openAiCredentials.openaiTtsApiKey) { + const { apiKey: _ttsApiKey, ...ttsConfig } = openaiConfig?.tts ?? {}; + ttsService = new OpenAITTS( + { + apiKey: openAiCredentials.openaiTtsApiKey, + voice: "alloy", + model: DEFAULT_OPENAI_TTS_MODEL, + responseFormat: "pcm", + ...ttsConfig, + }, + logger + ); + } + + const dictationApiKey = openAiCredentials.openaiDictationApiKey; + if (needsOpenAiDictation && dictationApiKey) { + dictationSttService = { + id: "openai", + createSession: ({ logger: sessionLogger, language, prompt }) => + new OpenAIRealtimeTranscriptionSession({ + apiKey: dictationApiKey, + logger: sessionLogger, + transcriptionModel: + openaiConfig?.realtimeTranscriptionModel ?? + DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL, + ...(language ? { language } : {}), + ...(prompt ? { prompt } : {}), + turnDetection: null, + }), + }; + } + } else if (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) { + logger.warn("OpenAI speech providers are configured but credentials are missing"); + } + + return { + sttService, + ttsService, + dictationSttService, + }; +} diff --git a/packages/server/src/server/speech/speech-config-resolver.ts b/packages/server/src/server/speech/speech-config-resolver.ts index e4a54f9e3..de9071740 100644 --- a/packages/server/src/server/speech/speech-config-resolver.ts +++ b/packages/server/src/server/speech/speech-config-resolver.ts @@ -1,33 +1,13 @@ -import path from "node:path"; - import { z } from "zod"; import type { PersistedConfig } from "../persisted-config.js"; import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js"; +import { resolveLocalSpeechConfig } from "./providers/local/config.js"; +import { resolveOpenAiSpeechConfig } from "./providers/openai/config.js"; import { - LocalSttModelIdSchema, - LocalTtsModelIdSchema, -} from "./providers/local/sherpa/model-catalog.js"; -import { - DEFAULT_LOCAL_STT_MODEL, - DEFAULT_LOCAL_TTS_MODEL, - DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL, - DEFAULT_OPENAI_TTS_MODEL, -} from "./speech-defaults.js"; -import { SpeechProviderIdSchema } from "./speech-types.js"; - -const DEFAULT_LOCAL_MODELS_SUBDIR = path.join("models", "local-speech"); - -const OpenAiTtsVoiceSchema = z.enum([ - "alloy", - "echo", - "fable", - "onyx", - "nova", - "shimmer", -]); - -const OpenAiTtsModelSchema = z.enum(["tts-1", "tts-1-hd"]); + SpeechProviderIdSchema, + type RequestedSpeechProviders, +} from "./speech-types.js"; const OptionalSpeechProviderSchema = z.preprocess((value) => { if (typeof value !== "string") { @@ -37,136 +17,31 @@ const OptionalSpeechProviderSchema = z.preprocess((value) => { return normalized.length > 0 ? normalized : undefined; }, SpeechProviderIdSchema.optional()); -const OptionalBooleanFlagSchema = z.preprocess((value) => { - if (typeof value === "boolean") { - return value; - } - if (typeof value !== "string") { - return value; - } - const normalized = value.trim().toLowerCase(); - if (normalized === "1" || normalized === "true" || normalized === "yes") { - return true; - } - if (normalized === "0" || normalized === "false" || normalized === "no") { - return false; - } - return undefined; -}, z.boolean().optional()); +const RequestedSpeechProvidersSchema = z.object({ + dictationSttProvider: OptionalSpeechProviderSchema.default("local"), + voiceSttProvider: OptionalSpeechProviderSchema.default("local"), + voiceTtsProvider: OptionalSpeechProviderSchema.default("local"), +}); -const OptionalFiniteNumberSchema = z.preprocess((value) => { - if (typeof value === "number") { - return Number.isFinite(value) ? value : undefined; - } - if (typeof value !== "string") { - return value; - } - const parsed = Number.parseFloat(value); - return Number.isFinite(parsed) ? parsed : undefined; -}, z.number().optional()); - -const OptionalIntegerSchema = z.preprocess((value) => { - if (typeof value === "number") { - return Number.isInteger(value) ? value : undefined; - } - if (typeof value !== "string") { - return value; - } - const parsed = Number.parseInt(value, 10); - return Number.isFinite(parsed) ? parsed : undefined; -}, z.number().int().optional()); - -const OptionalTrimmedStringSchema = z.preprocess((value) => { - if (typeof value !== "string") { - return value; - } - const trimmed = value.trim(); - return trimmed.length > 0 ? trimmed : undefined; -}, z.string().optional()); - -const ResolvedSpeechConfigSchema = z - .object({ - dictationSttProvider: OptionalSpeechProviderSchema.default("local"), - voiceSttProvider: OptionalSpeechProviderSchema.default("local"), - voiceTtsProvider: OptionalSpeechProviderSchema.default("local"), - localModelsDir: z.string().trim().min(1), - localAutoDownload: OptionalBooleanFlagSchema.default(true), - dictationLocalSttModel: LocalSttModelIdSchema.default(DEFAULT_LOCAL_STT_MODEL), - voiceLocalSttModel: LocalSttModelIdSchema.default(DEFAULT_LOCAL_STT_MODEL), - voiceLocalTtsModel: LocalTtsModelIdSchema.default(DEFAULT_LOCAL_TTS_MODEL), - voiceLocalTtsSpeakerId: OptionalIntegerSchema, - voiceLocalTtsSpeed: OptionalFiniteNumberSchema, - anyLocalRequested: z.boolean(), - openaiApiKey: OptionalTrimmedStringSchema, - openaiSttConfidenceThreshold: OptionalFiniteNumberSchema, - openaiSttModel: OptionalTrimmedStringSchema, - openaiTtsVoice: z.preprocess((value) => { - if (typeof value !== "string") { - return value; - } - const normalized = value.trim().toLowerCase(); - return normalized.length > 0 ? normalized : undefined; - }, OpenAiTtsVoiceSchema.default("alloy")), - openaiTtsModel: z.preprocess((value) => { - if (typeof value !== "string") { - return value; - } - const normalized = value.trim().toLowerCase(); - return normalized.length > 0 ? normalized : undefined; - }, OpenAiTtsModelSchema.default(DEFAULT_OPENAI_TTS_MODEL)), - openaiRealtimeTranscriptionModel: OptionalTrimmedStringSchema.default( - DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL - ), - }) - .transform((input): { openai: PaseoOpenAIConfig | undefined; speech: PaseoSpeechConfig } => { - const openai = input.openaiApiKey - ? { - apiKey: input.openaiApiKey, - stt: { - apiKey: input.openaiApiKey, - ...(input.openaiSttConfidenceThreshold !== undefined - ? { confidenceThreshold: input.openaiSttConfidenceThreshold } - : {}), - ...(input.openaiSttModel - ? { model: input.openaiSttModel } - : {}), - }, - tts: { - apiKey: input.openaiApiKey, - voice: input.openaiTtsVoice, - model: input.openaiTtsModel, - responseFormat: "pcm" as const, - }, - realtimeTranscriptionModel: input.openaiRealtimeTranscriptionModel, - } - : undefined; - - return { - openai, - speech: { - dictationSttProvider: input.dictationSttProvider, - voiceSttProvider: input.voiceSttProvider, - voiceTtsProvider: input.voiceTtsProvider, - ...(input.anyLocalRequested - ? { - local: { - modelsDir: input.localModelsDir, - autoDownload: input.localAutoDownload, - }, - } - : {}), - dictationLocalSttModel: input.dictationLocalSttModel, - voiceLocalSttModel: input.voiceLocalSttModel, - voiceLocalTtsModel: input.voiceLocalTtsModel, - ...(input.voiceLocalTtsSpeakerId !== undefined - ? { voiceLocalTtsSpeakerId: input.voiceLocalTtsSpeakerId } - : {}), - ...(input.voiceLocalTtsSpeed !== undefined - ? { voiceLocalTtsSpeed: input.voiceLocalTtsSpeed } - : {}), - }, - }; +function resolveRequestedSpeechProviders(params: { + env: NodeJS.ProcessEnv; + persisted: PersistedConfig; +}): RequestedSpeechProviders { + return RequestedSpeechProvidersSchema.parse({ + dictationSttProvider: + params.env.PASEO_DICTATION_STT_PROVIDER ?? + params.persisted.features?.dictation?.stt?.provider ?? + "local", + voiceSttProvider: + params.env.PASEO_VOICE_STT_PROVIDER ?? + params.persisted.features?.voiceMode?.stt?.provider ?? + "local", + voiceTtsProvider: + params.env.PASEO_VOICE_TTS_PROVIDER ?? + params.persisted.features?.voiceMode?.tts?.provider ?? + "local", }); +} export function resolveSpeechConfig(params: { paseoHome: string; @@ -176,89 +51,42 @@ export function resolveSpeechConfig(params: { openai: PaseoOpenAIConfig | undefined; speech: PaseoSpeechConfig; } { - const { paseoHome, env, persisted } = params; - - const dictationSttProvider = - env.PASEO_DICTATION_STT_PROVIDER - ?? persisted.features?.dictation?.stt?.provider - ?? "local"; - - const voiceSttProvider = - env.PASEO_VOICE_STT_PROVIDER - ?? persisted.features?.voiceMode?.stt?.provider - ?? "local"; - - const voiceTtsProvider = - env.PASEO_VOICE_TTS_PROVIDER - ?? persisted.features?.voiceMode?.tts?.provider - ?? "local"; - - const anyLocalRequested = - dictationSttProvider === "local" || - voiceSttProvider === "local" || - voiceTtsProvider === "local" || - env.PASEO_LOCAL_MODELS_DIR !== undefined || - persisted.providers?.local !== undefined; - - return ResolvedSpeechConfigSchema.parse({ - dictationSttProvider, - voiceSttProvider, - voiceTtsProvider, - localModelsDir: - env.PASEO_LOCAL_MODELS_DIR - ?? persisted.providers?.local?.modelsDir - ?? path.join(paseoHome, DEFAULT_LOCAL_MODELS_SUBDIR), - localAutoDownload: - env.PASEO_LOCAL_AUTO_DOWNLOAD - ?? persisted.providers?.local?.autoDownload, - dictationLocalSttModel: - env.PASEO_DICTATION_LOCAL_STT_MODEL - ?? persisted.features?.dictation?.stt?.model - ?? DEFAULT_LOCAL_STT_MODEL, - voiceLocalSttModel: - env.PASEO_VOICE_LOCAL_STT_MODEL - ?? persisted.features?.voiceMode?.stt?.model - ?? DEFAULT_LOCAL_STT_MODEL, - voiceLocalTtsModel: - env.PASEO_VOICE_LOCAL_TTS_MODEL - ?? persisted.features?.voiceMode?.tts?.model - ?? DEFAULT_LOCAL_TTS_MODEL, - voiceLocalTtsSpeakerId: - env.PASEO_VOICE_LOCAL_TTS_SPEAKER_ID - ?? persisted.features?.voiceMode?.tts?.speakerId, - voiceLocalTtsSpeed: - env.PASEO_VOICE_LOCAL_TTS_SPEED - ?? persisted.features?.voiceMode?.tts?.speed, - anyLocalRequested, - openaiApiKey: env.OPENAI_API_KEY ?? persisted.providers?.openai?.apiKey, - openaiSttConfidenceThreshold: - env.STT_CONFIDENCE_THRESHOLD - ?? persisted.features?.dictation?.stt?.confidenceThreshold, - openaiSttModel: - env.STT_MODEL - ?? (voiceSttProvider === "openai" - ? persisted.features?.voiceMode?.stt?.model - : undefined) - ?? (dictationSttProvider === "openai" - ? persisted.features?.dictation?.stt?.model - : undefined), - openaiTtsVoice: - env.TTS_VOICE - ?? (voiceTtsProvider === "openai" - ? persisted.features?.voiceMode?.tts?.voice - : undefined) - ?? "alloy", - openaiTtsModel: - env.TTS_MODEL - ?? (voiceTtsProvider === "openai" - ? persisted.features?.voiceMode?.tts?.model - : undefined) - ?? DEFAULT_OPENAI_TTS_MODEL, - openaiRealtimeTranscriptionModel: - env.OPENAI_REALTIME_TRANSCRIPTION_MODEL - ?? (dictationSttProvider === "openai" - ? persisted.features?.dictation?.stt?.model - : undefined) - ?? DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL, + const providers = resolveRequestedSpeechProviders({ + env: params.env, + persisted: params.persisted, }); + + const local = resolveLocalSpeechConfig({ + paseoHome: params.paseoHome, + env: params.env, + persisted: params.persisted, + providers, + }); + + const openai = resolveOpenAiSpeechConfig({ + env: params.env, + persisted: params.persisted, + providers, + }); + + return { + openai, + speech: { + dictationSttProvider: providers.dictationSttProvider, + voiceSttProvider: providers.voiceSttProvider, + voiceTtsProvider: providers.voiceTtsProvider, + ...(local.local + ? { local: local.local } + : {}), + dictationLocalSttModel: local.dictationLocalSttModel, + voiceLocalSttModel: local.voiceLocalSttModel, + voiceLocalTtsModel: local.voiceLocalTtsModel, + ...(local.voiceLocalTtsSpeakerId !== undefined + ? { voiceLocalTtsSpeakerId: local.voiceLocalTtsSpeakerId } + : {}), + ...(local.voiceLocalTtsSpeed !== undefined + ? { voiceLocalTtsSpeed: local.voiceLocalTtsSpeed } + : {}), + }, + }; } diff --git a/packages/server/src/server/speech/speech-defaults.ts b/packages/server/src/server/speech/speech-defaults.ts deleted file mode 100644 index c249734b9..000000000 --- a/packages/server/src/server/speech/speech-defaults.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { - DEFAULT_LOCAL_STT_MODEL, - DEFAULT_LOCAL_TTS_MODEL, -} from "./providers/local/sherpa/model-catalog.js"; - -export { DEFAULT_LOCAL_STT_MODEL, DEFAULT_LOCAL_TTS_MODEL }; - -export const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe"; -export const DEFAULT_OPENAI_TTS_MODEL = "tts-1"; diff --git a/packages/server/src/server/speech/speech-runtime.ts b/packages/server/src/server/speech/speech-runtime.ts index 938f81f59..363eb0ebb 100644 --- a/packages/server/src/server/speech/speech-runtime.ts +++ b/packages/server/src/server/speech/speech-runtime.ts @@ -2,71 +2,17 @@ import type { Logger } from "pino"; import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js"; import { - OpenAISTT, -} from "./providers/openai/stt.js"; + getLocalSpeechAvailability, + initializeLocalSpeechServices, +} from "./providers/local/runtime.js"; +import type { LocalSpeechModelId } from "./providers/local/config.js"; import { - OpenAITTS, -} from "./providers/openai/tts.js"; -import { OpenAIRealtimeTranscriptionSession } from "./providers/openai/realtime-transcription-session.js"; + getOpenAiSpeechAvailability, + initializeOpenAiSpeechServices, + validateOpenAiCredentialRequirements, +} from "./providers/openai/runtime.js"; import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech-provider.js"; -import { SherpaOnlineRecognizerEngine } from "./providers/local/sherpa/sherpa-online-recognizer.js"; -import { SherpaOfflineRecognizerEngine } from "./providers/local/sherpa/sherpa-offline-recognizer.js"; -import { SherpaOnnxSTT } from "./providers/local/sherpa/sherpa-stt.js"; -import { SherpaOnnxParakeetSTT } from "./providers/local/sherpa/sherpa-parakeet-stt.js"; -import { SherpaOnnxTTS } from "./providers/local/sherpa/sherpa-tts.js"; -import { SherpaRealtimeTranscriptionSession } from "./providers/local/sherpa/sherpa-realtime-session.js"; -import { SherpaParakeetRealtimeTranscriptionSession } from "./providers/local/sherpa/sherpa-parakeet-realtime-session.js"; -import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./providers/local/sherpa/model-downloader.js"; -import { - LocalSttModelIdSchema, - LocalTtsModelIdSchema, - type LocalSttModelId, - type LocalTtsModelId, - type SherpaOnnxModelId, -} from "./providers/local/sherpa/model-catalog.js"; -import { PocketTtsOnnxTTS } from "./providers/local/pocket/pocket-tts-onnx.js"; -import { - DEFAULT_LOCAL_STT_MODEL, - DEFAULT_LOCAL_TTS_MODEL, - DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL, - DEFAULT_OPENAI_TTS_MODEL, -} from "./speech-defaults.js"; -import type { SpeechProviderId } from "./speech-types.js"; - -type LocalSttEngine = - | { kind: "offline"; engine: SherpaOfflineRecognizerEngine } - | { kind: "online"; engine: SherpaOnlineRecognizerEngine }; - -type RequestedSpeechProviders = { - dictationSttProvider: SpeechProviderId; - voiceSttProvider: SpeechProviderId; - voiceTtsProvider: SpeechProviderId; -}; - -type ResolvedLocalModels = { - dictationLocalSttModel: LocalSttModelId; - voiceLocalSttModel: LocalSttModelId; - voiceLocalTtsModel: LocalTtsModelId; -}; - -type OpenAiCredentialState = { - openaiSttApiKey: string | undefined; - openaiTtsApiKey: string | undefined; - openaiDictationApiKey: string | undefined; -}; - -type InitializedLocalSpeech = { - sttService: SpeechToTextProvider | null; - ttsService: TextToSpeechProvider | null; - dictationSttService: SpeechToTextProvider | null; - localVoiceTtsProvider: TextToSpeechProvider | null; - localSttEngines: Map; - requiredLocalModelIds: SherpaOnnxModelId[]; -}; - -function buildModelDownloadHint(modelId: SherpaOnnxModelId): string { - return `Use 'paseo speech download --model ${modelId}' to download this model.`; -} +import type { RequestedSpeechProviders } from "./speech-types.js"; function resolveRequestedSpeechProviders( speechConfig: PaseoSpeechConfig | null @@ -78,425 +24,6 @@ function resolveRequestedSpeechProviders( }; } -function resolveConfiguredLocalModels( - speechConfig: PaseoSpeechConfig | null -): ResolvedLocalModels { - return { - dictationLocalSttModel: LocalSttModelIdSchema.parse( - speechConfig?.dictationLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL - ), - voiceLocalSttModel: LocalSttModelIdSchema.parse( - speechConfig?.voiceLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL - ), - voiceLocalTtsModel: LocalTtsModelIdSchema.parse( - speechConfig?.voiceLocalTtsModel ?? DEFAULT_LOCAL_TTS_MODEL - ), - }; -} - -function computeRequiredLocalModelIds(params: { - providers: RequestedSpeechProviders; - models: { - dictationLocalSttModel: SherpaOnnxModelId; - voiceLocalSttModel: SherpaOnnxModelId; - voiceLocalTtsModel: SherpaOnnxModelId; - }; -}): SherpaOnnxModelId[] { - const ids = new Set(); - if (params.providers.dictationSttProvider === "local") { - ids.add(params.models.dictationLocalSttModel); - } - if (params.providers.voiceSttProvider === "local") { - ids.add(params.models.voiceLocalSttModel); - } - if (params.providers.voiceTtsProvider === "local") { - ids.add(params.models.voiceLocalTtsModel); - } - return Array.from(ids); -} - -function resolveOpenAiCredentials(openaiConfig?: PaseoOpenAIConfig): OpenAiCredentialState { - const openaiApiKey = openaiConfig?.apiKey; - return { - openaiSttApiKey: openaiConfig?.stt?.apiKey ?? openaiApiKey, - openaiTtsApiKey: openaiConfig?.tts?.apiKey ?? openaiApiKey, - openaiDictationApiKey: openaiApiKey, - }; -} - -function validateOpenAiCredentialRequirements(params: { - providers: RequestedSpeechProviders; - openAiCredentials: OpenAiCredentialState; - logger: Logger; -}): void { - const { providers, openAiCredentials, logger } = params; - const missingOpenAiCredentialsFor: string[] = []; - if (providers.voiceSttProvider === "openai" && !openAiCredentials.openaiSttApiKey) { - missingOpenAiCredentialsFor.push("voice.stt"); - } - if (providers.voiceTtsProvider === "openai" && !openAiCredentials.openaiTtsApiKey) { - missingOpenAiCredentialsFor.push("voice.tts"); - } - if (providers.dictationSttProvider === "openai" && !openAiCredentials.openaiDictationApiKey) { - missingOpenAiCredentialsFor.push("dictation.stt"); - } - - if (missingOpenAiCredentialsFor.length > 0) { - logger.error( - { - requestedProviders: { - dictationStt: providers.dictationSttProvider, - voiceStt: providers.voiceSttProvider, - voiceTts: providers.voiceTtsProvider, - }, - missingOpenAiCredentialsFor, - }, - "Invalid speech configuration: OpenAI provider selected but credentials are missing" - ); - throw new Error( - `Missing OpenAI credentials for configured speech features: ${missingOpenAiCredentialsFor.join(", ")}` - ); - } -} - -async function createLocalSttEngine(params: { - modelId: LocalSttModelId; - modelsDir: string; - logger: Logger; -}): Promise { - const { modelId, modelsDir, logger } = params; - - if (modelId === "parakeet-tdt-0.6b-v3-int8") { - const modelDir = getSherpaOnnxModelDir(modelsDir, modelId); - return { - kind: "offline", - engine: new SherpaOfflineRecognizerEngine( - { - model: { - kind: "nemo_transducer", - encoder: `${modelDir}/encoder.int8.onnx`, - decoder: `${modelDir}/decoder.int8.onnx`, - joiner: `${modelDir}/joiner.int8.onnx`, - tokens: `${modelDir}/tokens.txt`, - }, - numThreads: 2, - debug: 0, - }, - logger - ), - }; - } - - if (modelId === "paraformer-bilingual-zh-en") { - const modelDir = getSherpaOnnxModelDir(modelsDir, modelId); - return { - kind: "online", - engine: new SherpaOnlineRecognizerEngine( - { - model: { - kind: "paraformer", - encoder: `${modelDir}/encoder.int8.onnx`, - decoder: `${modelDir}/decoder.int8.onnx`, - tokens: `${modelDir}/tokens.txt`, - }, - numThreads: 1, - debug: 0, - }, - logger - ), - }; - } - - if (modelId === "zipformer-bilingual-zh-en-2023-02-20") { - const modelDir = getSherpaOnnxModelDir(modelsDir, modelId); - return { - kind: "online", - engine: new SherpaOnlineRecognizerEngine( - { - model: { - kind: "transducer", - encoder: `${modelDir}/encoder-epoch-99-avg-1.onnx`, - decoder: `${modelDir}/decoder-epoch-99-avg-1.onnx`, - joiner: `${modelDir}/joiner-epoch-99-avg-1.onnx`, - tokens: `${modelDir}/tokens.txt`, - modelType: "zipformer", - }, - numThreads: 1, - debug: 0, - }, - logger - ), - }; - } - - throw new Error(`Unsupported local STT model '${modelId}'`); -} - -async function initializeLocalSpeechServices(params: { - providers: RequestedSpeechProviders; - localConfig: NonNullable | null; - localModels: ResolvedLocalModels; - speechConfig: PaseoSpeechConfig | null; - logger: Logger; -}): Promise { - const { providers, localConfig, localModels, speechConfig, logger } = params; - - const sttServices = { - sttService: null as SpeechToTextProvider | null, - ttsService: null as TextToSpeechProvider | null, - dictationSttService: null as SpeechToTextProvider | null, - }; - - const localSttEngines = new Map(); - let localVoiceTtsProvider: TextToSpeechProvider | null = null; - - const requiredLocalModelIds = computeRequiredLocalModelIds({ - providers, - models: localModels, - }); - - if (requiredLocalModelIds.length > 0 && localConfig) { - try { - logger.info( - { - modelsDir: localConfig.modelsDir, - modelIds: requiredLocalModelIds, - autoDownload: localConfig.autoDownload ?? true, - }, - "Ensuring local speech models" - ); - await ensureSherpaOnnxModels({ - modelsDir: localConfig.modelsDir, - modelIds: requiredLocalModelIds, - autoDownload: localConfig.autoDownload ?? true, - logger, - }); - } catch (err) { - logger.error( - { - err, - modelsDir: localConfig.modelsDir, - modelIds: requiredLocalModelIds, - autoDownload: localConfig.autoDownload ?? true, - hint: - "Use `paseo speech models` to inspect status and " + - "`paseo speech download --model ` to fetch missing models.", - }, - "Failed to ensure local speech models" - ); - } - } - - const getLocalSttEngine = async ( - modelId: LocalSttModelId - ): Promise => { - const existing = localSttEngines.get(modelId); - if (existing) { - return existing; - } - if (!localConfig) { - return null; - } - try { - const created = await createLocalSttEngine({ - modelId, - modelsDir: localConfig.modelsDir, - logger, - }); - localSttEngines.set(modelId, created); - return created; - } catch (err) { - logger.error( - { - err, - modelsDir: localConfig.modelsDir, - modelId, - hint: buildModelDownloadHint(modelId), - }, - "Failed to initialize local STT engine (models missing or invalid)" - ); - return null; - } - }; - - if (providers.voiceSttProvider === "local") { - if (!localConfig) { - logger.warn( - { configured: false }, - "Local STT selected for voice but local provider config is missing; STT will be unavailable" - ); - } else { - const voiceEngine = await getLocalSttEngine(localModels.voiceLocalSttModel); - if (voiceEngine?.kind === "offline") { - sttServices.sttService = new SherpaOnnxParakeetSTT({ engine: voiceEngine.engine }, logger); - } else if (voiceEngine?.kind === "online") { - sttServices.sttService = new SherpaOnnxSTT({ engine: voiceEngine.engine }, logger); - } - } - } - - if (providers.dictationSttProvider === "local") { - if (!localConfig) { - logger.warn( - { configured: false }, - "Local STT selected for dictation but local provider config is missing; dictation STT will be unavailable" - ); - } else { - const dictationEngine = await getLocalSttEngine(localModels.dictationLocalSttModel); - if (dictationEngine?.kind === "offline") { - sttServices.dictationSttService = { - id: "local", - createSession: () => - new SherpaParakeetRealtimeTranscriptionSession({ engine: dictationEngine.engine }), - }; - } else if (dictationEngine?.kind === "online") { - sttServices.dictationSttService = { - id: "local", - createSession: () => new SherpaRealtimeTranscriptionSession({ engine: dictationEngine.engine }), - }; - } - } - } - - if (providers.voiceTtsProvider === "local") { - if (!localConfig) { - logger.warn( - { configured: false }, - "Local TTS selected for voice but local provider config is missing; TTS will be unavailable" - ); - } else { - try { - if (localModels.voiceLocalTtsModel === "pocket-tts-onnx-int8") { - const modelDir = getSherpaOnnxModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel); - localVoiceTtsProvider = await PocketTtsOnnxTTS.create( - { - modelDir, - precision: "int8", - targetChunkMs: 50, - }, - logger - ); - } else { - const modelDir = getSherpaOnnxModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel); - localVoiceTtsProvider = new SherpaOnnxTTS( - { - preset: localModels.voiceLocalTtsModel, - modelDir, - speakerId: speechConfig?.voiceLocalTtsSpeakerId, - speed: speechConfig?.voiceLocalTtsSpeed, - }, - logger - ); - } - sttServices.ttsService = localVoiceTtsProvider; - } catch (err) { - logger.error( - { - err, - modelsDir: localConfig.modelsDir, - modelId: localModels.voiceLocalTtsModel, - hint: buildModelDownloadHint(localModels.voiceLocalTtsModel), - }, - "Failed to initialize local TTS engine (models missing or invalid)" - ); - } - } - } - - return { - ...sttServices, - localVoiceTtsProvider, - localSttEngines, - requiredLocalModelIds, - }; -} - -function initializeOpenAiSpeechServices(params: { - providers: RequestedSpeechProviders; - openaiConfig?: PaseoOpenAIConfig; - openAiCredentials: OpenAiCredentialState; - existing: { - sttService: SpeechToTextProvider | null; - ttsService: TextToSpeechProvider | null; - dictationSttService: SpeechToTextProvider | null; - }; - logger: Logger; -}): { - sttService: SpeechToTextProvider | null; - ttsService: TextToSpeechProvider | null; - dictationSttService: SpeechToTextProvider | null; -} { - const { providers, openaiConfig, openAiCredentials, existing, logger } = params; - - let sttService = existing.sttService; - let ttsService = existing.ttsService; - let dictationSttService = existing.dictationSttService; - - const needsOpenAiStt = !sttService && providers.voiceSttProvider === "openai"; - const needsOpenAiTts = !ttsService && providers.voiceTtsProvider === "openai"; - const needsOpenAiDictation = !dictationSttService && providers.dictationSttProvider === "openai"; - - if ( - (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) && - (openAiCredentials.openaiSttApiKey || - openAiCredentials.openaiTtsApiKey || - openAiCredentials.openaiDictationApiKey) - ) { - logger.info("OpenAI speech provider initialized"); - - if (needsOpenAiStt && openAiCredentials.openaiSttApiKey) { - const { apiKey: _sttApiKey, ...sttConfig } = openaiConfig?.stt ?? {}; - sttService = new OpenAISTT( - { - apiKey: openAiCredentials.openaiSttApiKey, - ...sttConfig, - }, - logger - ); - } - - if (needsOpenAiTts && openAiCredentials.openaiTtsApiKey) { - const { apiKey: _ttsApiKey, ...ttsConfig } = openaiConfig?.tts ?? {}; - ttsService = new OpenAITTS( - { - apiKey: openAiCredentials.openaiTtsApiKey, - voice: "alloy", - model: DEFAULT_OPENAI_TTS_MODEL, - responseFormat: "pcm", - ...ttsConfig, - }, - logger - ); - } - - const dictationApiKey = openAiCredentials.openaiDictationApiKey; - if (needsOpenAiDictation && dictationApiKey) { - dictationSttService = { - id: "openai", - createSession: ({ logger: sessionLogger, language, prompt }) => - new OpenAIRealtimeTranscriptionSession({ - apiKey: dictationApiKey, - logger: sessionLogger, - transcriptionModel: - openaiConfig?.realtimeTranscriptionModel - ?? DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL, - ...(language ? { language } : {}), - ...(prompt ? { prompt } : {}), - turnDetection: null, - }), - }; - } - } else if (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) { - logger.warn("OpenAI speech providers are configured but credentials are missing"); - } - - return { - sttService, - ttsService, - dictationSttService, - }; -} - export type InitializedSpeechRuntime = { sttService: SpeechToTextProvider | null; ttsService: TextToSpeechProvider | null; @@ -504,7 +31,7 @@ export type InitializedSpeechRuntime = { cleanup: () => void; localModelConfig: { modelsDir: string; - defaultModelIds: SherpaOnnxModelId[]; + defaultModelIds: LocalSpeechModelId[]; } | null; }; @@ -515,16 +42,12 @@ export async function initializeSpeechRuntime(params: { }): Promise { const logger = params.logger; const speechConfig = params.speechConfig ?? null; - const localConfig = speechConfig?.local ?? null; const openaiConfig = params.openaiConfig; - const providers = resolveRequestedSpeechProviders(speechConfig); - const localModels = resolveConfiguredLocalModels(speechConfig); - const openAiCredentials = resolveOpenAiCredentials(openaiConfig); validateOpenAiCredentialRequirements({ providers, - openAiCredentials, + openaiConfig, logger, }); @@ -536,16 +59,8 @@ export async function initializeSpeechRuntime(params: { voiceTts: providers.voiceTtsProvider, }, availability: { - openai: { - stt: Boolean(openAiCredentials.openaiSttApiKey), - tts: Boolean(openAiCredentials.openaiTtsApiKey), - dictationStt: Boolean(openAiCredentials.openaiDictationApiKey), - }, - local: { - configured: Boolean(localConfig), - modelsDir: localConfig?.modelsDir ?? null, - autoDownload: localConfig?.autoDownload ?? null, - }, + openai: getOpenAiSpeechAvailability(openaiConfig), + local: getLocalSpeechAvailability(speechConfig), }, }, "Speech provider reconciliation started" @@ -553,8 +68,6 @@ export async function initializeSpeechRuntime(params: { const localSpeech = await initializeLocalSpeechServices({ providers, - localConfig, - localModels, speechConfig, logger, }); @@ -562,7 +75,6 @@ export async function initializeSpeechRuntime(params: { const openAiSpeech = initializeOpenAiSpeechServices({ providers, openaiConfig, - openAiCredentials, existing: { sttService: localSpeech.sttService, ttsService: localSpeech.ttsService, @@ -610,27 +122,11 @@ export async function initializeSpeechRuntime(params: { "Speech provider reconciliation completed" ); - const cleanup = () => { - const maybeFreeable = localSpeech.localVoiceTtsProvider as unknown as { free?: () => void } | null; - if (typeof maybeFreeable?.free === "function") { - maybeFreeable.free(); - } - for (const engine of localSpeech.localSttEngines.values()) { - engine.engine.free(); - } - }; - return { sttService: openAiSpeech.sttService, ttsService: openAiSpeech.ttsService, dictationSttService: openAiSpeech.dictationSttService, - cleanup, - localModelConfig: - localConfig - ? { - modelsDir: localConfig.modelsDir, - defaultModelIds: localSpeech.requiredLocalModelIds, - } - : null, + cleanup: localSpeech.cleanup, + localModelConfig: localSpeech.localModelConfig, }; } diff --git a/packages/server/src/server/speech/speech-types.ts b/packages/server/src/server/speech/speech-types.ts index ff44f9126..d7ffd32d7 100644 --- a/packages/server/src/server/speech/speech-types.ts +++ b/packages/server/src/server/speech/speech-types.ts @@ -2,3 +2,9 @@ import { z } from "zod"; export const SpeechProviderIdSchema = z.enum(["openai", "local"]); export type SpeechProviderId = z.infer; + +export type RequestedSpeechProviders = { + dictationSttProvider: SpeechProviderId; + voiceSttProvider: SpeechProviderId; + voiceTtsProvider: SpeechProviderId; +};