diff --git a/packages/app/src/components/agent-input-area.tsx b/packages/app/src/components/agent-input-area.tsx index eddc9b801..d62f53583 100644 --- a/packages/app/src/components/agent-input-area.tsx +++ b/packages/app/src/components/agent-input-area.tsx @@ -4,6 +4,7 @@ import { Text, ActivityIndicator, Platform, + Alert, } from "react-native"; import { useState, useEffect, useRef, useCallback, useMemo } from "react"; import { StyleSheet, useUnistyles } from "react-native-unistyles"; @@ -469,6 +470,8 @@ export function AgentInputArea({ } void voice.startVoice(serverId, agentId).catch((error) => { console.error("[AgentInputArea] Failed to start voice mode", error); + const message = error instanceof Error ? error.message : "Voice features are not available right now."; + Alert.alert("Voice unavailable", message); }); }, [agentId, isConnected, serverId, voice]); diff --git a/packages/app/src/components/message-input.tsx b/packages/app/src/components/message-input.tsx index 98a77ddd9..8bf1445cd 100644 --- a/packages/app/src/components/message-input.tsx +++ b/packages/app/src/components/message-input.tsx @@ -10,6 +10,7 @@ import { Image, Platform, BackHandler, + Alert, } from "react-native"; import { useState, @@ -297,6 +298,8 @@ export const MessageInput = forwardRef( } void voice.startVoice(voiceServerId, voiceAgentId).catch((error) => { console.error("[MessageInput] Failed to start realtime voice", error); + const message = error instanceof Error ? error.message : "Voice features are not available right now."; + Alert.alert("Voice unavailable", message); }); }; @@ -461,6 +464,8 @@ export const MessageInput = forwardRef( } void voice.startVoice(voiceServerId, voiceAgentId).catch((error) => { console.error("[MessageInput] Failed to start realtime voice", error); + const message = error instanceof Error ? error.message : "Voice features are not available right now."; + Alert.alert("Voice unavailable", message); }); }, [ disabled, diff --git a/packages/cli/src/commands/onboard.ts b/packages/cli/src/commands/onboard.ts index 2b5100c69..7f589780d 100644 --- a/packages/cli/src/commands/onboard.ts +++ b/packages/cli/src/commands/onboard.ts @@ -3,11 +3,9 @@ import { Command } from 'commander' import { writeFileSync } from 'node:fs' import path from 'node:path' import { - ensureLocalSpeechModels, generateLocalPairingOffer, loadConfig, loadPersistedConfig, - type LocalSpeechModelId, type CliConfigOverrides, type PersistedConfig, } from '@getpaseo/server' @@ -154,7 +152,7 @@ async function resolveVoiceSelection(mode: OnboardOptions['voice']): Promise) => LocalSpeechDownloadLogger - info: (obj?: unknown, msg?: string) => void - error: (_obj?: unknown, _msg?: string) => void -} - -type LocalSpeechDownloadEvent = - | { - type: 'progress' - progress: LocalModelDownloadProgress - } - | { - type: 'phase' - phase: 'extracting' | 'verifying' | 'finalizing' | 'completed' - } - -function resolveRequiredLocalModelIds(config: ReturnType): LocalSpeechModelId[] { - const providers = config.speech?.providers - const local = config.speech?.local - - if (!providers || !local) { - return [] - } - - const ids = new Set() - - if (providers.dictationStt.enabled !== false && providers.dictationStt.provider === 'local') { - ids.add(local.models.dictationStt) - } - if (providers.voiceStt.enabled !== false && providers.voiceStt.provider === 'local') { - ids.add(local.models.voiceStt) - } - if (providers.voiceTts.enabled !== false && providers.voiceTts.provider === 'local') { - ids.add(local.models.voiceTts) - } - - return Array.from(ids) -} - -function parseLocalModelDownloadProgress(payload: unknown): LocalModelDownloadProgress | null { - if (!payload || typeof payload !== 'object') { - return null - } - - const value = payload as Record - const modelId = typeof value.modelId === 'string' ? value.modelId : null - const pctRaw = value.pct - const pct = typeof pctRaw === 'number' && Number.isFinite(pctRaw) ? Math.max(0, Math.min(100, Math.floor(pctRaw))) : null - - return { - modelId, - pct, - } -} - -function renderLocalModelProgress(params: { - modelId: LocalSpeechModelId - modelIndex: number - modelCount: number - pct: number | null -}): string { - const prefix = `Downloading speech model ${params.modelIndex}/${params.modelCount}: ${params.modelId}` - if (params.pct === null) { - return `${prefix}...` - } - return `${prefix} (${params.pct}%)` -} - -function createLocalSpeechDownloadLogger( - onEvent: (event: LocalSpeechDownloadEvent) => void -): LocalSpeechDownloadLogger { - const logger: LocalSpeechDownloadLogger = { - child: () => logger, - info: (obj?: unknown, msg?: string) => { - if (msg === 'Downloading model artifact') { - const progress = parseLocalModelDownloadProgress(obj) - if (!progress) { - return - } - onEvent({ - type: 'progress', - progress, - }) - return - } - if (msg === 'Extracting model archive') { - onEvent({ type: 'phase', phase: 'extracting' }) - return - } - if (msg === 'Verifying downloaded model files') { - onEvent({ type: 'phase', phase: 'verifying' }) - return - } - if (msg === 'Finalizing model artifacts') { - onEvent({ type: 'phase', phase: 'finalizing' }) - return - } - if (msg === 'Model download completed') { - onEvent({ type: 'phase', phase: 'completed' }) - return - } - }, - error: () => { - // no-op: onboarding handles surfaced errors from ensureLocalSpeechModels. - }, - } - return logger -} - -async function prepareLocalSpeechModelsBeforeStart(args: { - config: ReturnType - richUi: boolean -}): Promise { - const local = args.config.speech?.local - const modelIds = resolveRequiredLocalModelIds(args.config) - - if (!local || modelIds.length === 0) { - return - } - - if (local.autoDownload === false) { - log.warn('Local speech model auto-download is disabled. Voice may be unavailable until models are installed.') - return - } - - const modelList = modelIds.join(', ') - const modelCount = modelIds.length - const downloadSpinner = args.richUi ? spinner() : null - let lastPlainStatus = '' - - const emitStatus = (status: string): void => { - if (downloadSpinner) { - downloadSpinner.message(status) - return - } - if (status === lastPlainStatus) { - return - } - console.log(status) - lastPlainStatus = status - } - - if (downloadSpinner) { - downloadSpinner.start(`Preparing local speech models (${modelCount})...`) - } else { - log.message(`Preparing local speech models (${modelCount}): ${modelList}`) - } - - try { - for (const [index, modelId] of modelIds.entries()) { - const modelIndex = index + 1 - emitStatus(`Checking speech model ${modelIndex}/${modelCount}: ${modelId}`) - - const perModelLogger = createLocalSpeechDownloadLogger((event) => { - if (event.type === 'progress') { - const progress = event.progress - if (progress.modelId && progress.modelId !== modelId) { - return - } - emitStatus( - renderLocalModelProgress({ - modelId, - modelIndex, - modelCount, - pct: progress.pct, - }) - ) - return - } - - if (event.phase === 'extracting') { - emitStatus(`Extracting speech model ${modelIndex}/${modelCount}: ${modelId}`) - return - } - if (event.phase === 'verifying') { - emitStatus(`Verifying speech model ${modelIndex}/${modelCount}: ${modelId}`) - return - } - if (event.phase === 'finalizing') { - emitStatus(`Finalizing speech model ${modelIndex}/${modelCount}: ${modelId}`) - return - } - }) - - await ensureLocalSpeechModels({ - modelsDir: local.modelsDir, - modelIds: [modelId], - autoDownload: true, - logger: perModelLogger as any, - }) - - emitStatus(`Speech model ready ${modelIndex}/${modelCount}: ${modelId}`) - } - - if (downloadSpinner) { - downloadSpinner.stop(`Local speech models ready (${modelCount})`) - } else { - log.message(`Local speech models ready (${modelCount}): ${modelList}`) - } - } catch (error) { - const message = error instanceof Error ? error.message : String(error) - if (downloadSpinner) { - downloadSpinner.error(`Failed to prepare local speech models: ${message}`) - } else { - log.error(`Failed to prepare local speech models: ${message}`) - } - throw error - } -} - function parseDownloadProgress(logTail: string): DownloadProgress | null { const lines = logTail.split('\n').filter(Boolean) @@ -583,19 +366,10 @@ export async function runOnboard(options: OnboardOptions): Promise { const config = loadConfig(paseoHome, { cli: toCliOverrides(options) }) const voiceStatus = voiceEnabled - ? 'Voice features enabled. Local speech models will be downloaded if missing.' - : 'Voice features disabled. Local speech models will not be downloaded now.' + ? 'Voice features enabled. Local speech models will download in the background if missing.' + : 'Voice features disabled. Local speech models will not be downloaded.' log.message(voiceStatus) - try { - await prepareLocalSpeechModelsBeforeStart({ - config, - richUi, - }) - } catch { - process.exit(1) - } - const stateBeforeStart = resolveLocalDaemonState({ home: options.home }) const startSpinner = richUi ? spinner() : null diff --git a/packages/server/src/client/daemon-client.ts b/packages/server/src/client/daemon-client.ts index 638131cda..45ecbf177 100644 --- a/packages/server/src/client/daemon-client.ts +++ b/packages/server/src/client/daemon-client.ts @@ -1465,7 +1465,7 @@ export class DaemonClient { ...(agentId ? { agentId } : {}), requestId, }); - return this.sendRequest({ + const response = await this.sendRequest({ requestId, message, timeout: 10000, @@ -1479,6 +1479,14 @@ export class DaemonClient { return msg.payload; }, }); + if (!response.accepted) { + const codeSuffix = + typeof response.reasonCode === "string" && response.reasonCode.trim().length > 0 + ? ` (${response.reasonCode})` + : ""; + throw new Error((response.error ?? "Failed to set voice mode") + codeSuffix); + } + return response; } async sendVoiceAudioChunk( diff --git a/packages/server/src/server/agent/stt-manager.ts b/packages/server/src/server/agent/stt-manager.ts index 878e05abc..6fa69e715 100644 --- a/packages/server/src/server/agent/stt-manager.ts +++ b/packages/server/src/server/agent/stt-manager.ts @@ -39,12 +39,16 @@ export interface SessionTranscriptionResult extends TranscriptionResult { export class STTManager { private readonly sessionId: string; private readonly logger: pino.Logger; - private readonly stt: SpeechToTextProvider | null; + private readonly resolveStt: () => SpeechToTextProvider | null; - constructor(sessionId: string, logger: pino.Logger, stt: SpeechToTextProvider | null) { + constructor( + sessionId: string, + logger: pino.Logger, + stt: SpeechToTextProvider | null | (() => SpeechToTextProvider | null) + ) { this.sessionId = sessionId; this.logger = logger.child({ module: "agent", component: "stt-manager", sessionId }); - this.stt = stt; + this.resolveStt = typeof stt === "function" ? stt : () => stt; } /** @@ -55,7 +59,8 @@ export class STTManager { format: string, metadata?: TranscriptionMetadata ): Promise { - if (!this.stt) { + const stt = this.resolveStt(); + if (!stt) { throw new Error("STT not configured"); } @@ -81,7 +86,7 @@ export class STTManager { this.logger.warn({ err: error }, "Failed to persist debug audio"); } - const session = this.stt.createSession({ + const session = stt.createSession({ logger: this.logger.child({ component: "stt-session" }), language: "en", }); diff --git a/packages/server/src/server/agent/tts-manager.ts b/packages/server/src/server/agent/tts-manager.ts index 223b03707..a03a6fd87 100644 --- a/packages/server/src/server/agent/tts-manager.ts +++ b/packages/server/src/server/agent/tts-manager.ts @@ -95,11 +95,15 @@ function splitTextForTts(text: string, maxChars: number): string[] { export class TTSManager { private pendingPlaybacks: Map = new Map(); private readonly logger: pino.Logger; - private readonly tts: TextToSpeechProvider | null; + private readonly resolveTts: () => TextToSpeechProvider | null; - constructor(sessionId: string, logger: pino.Logger, tts: TextToSpeechProvider | null) { + constructor( + sessionId: string, + logger: pino.Logger, + tts: TextToSpeechProvider | null | (() => TextToSpeechProvider | null) + ) { this.logger = logger.child({ module: "agent", component: "tts-manager", sessionId }); - this.tts = tts; + this.resolveTts = typeof tts === "function" ? tts : () => tts; } /** @@ -143,7 +147,8 @@ export class TTSManager { abortSignal: AbortSignal, isVoiceMode: boolean ): Promise { - if (!this.tts) { + const tts = this.resolveTts(); + if (!tts) { throw new Error("TTS not configured"); } @@ -153,7 +158,7 @@ export class TTSManager { } // Generate TTS audio stream - const { stream, format } = await this.tts.synthesizeSpeech(text); + const { stream, format } = await tts.synthesizeSpeech(text); if (abortSignal.aborted) { this.logger.debug("Aborted after generating audio"); diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index b907f79a8..efe0858e7 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -457,9 +457,10 @@ export async function createPaseoDaemon( }, }); const { - sttService, - ttsService, - dictationSttService, + resolveVoiceStt, + resolveVoiceTts, + resolveDictationStt, + getSpeechReadiness, cleanup: cleanupSpeechRuntime, localModelConfig, } = await initializeSpeechRuntime({ @@ -478,7 +479,7 @@ export async function createPaseoDaemon( config.paseoHome, createInMemoryAgentMcpTransport, { allowedOrigins, allowedHosts: config.allowedHosts }, - { stt: sttService, tts: ttsService }, + { stt: resolveVoiceStt, tts: resolveVoiceTts }, terminalManager, { voiceAgentMcpStdio: { @@ -496,8 +497,9 @@ export async function createPaseoDaemon( }, { finalTimeoutMs: config.dictationFinalTimeoutMs, - stt: dictationSttService, + stt: resolveDictationStt, localModels: localModelConfig ?? undefined, + getSpeechReadiness, }, config.agentProviderSettings ); diff --git a/packages/server/src/server/dictation/dictation-stream-manager.ts b/packages/server/src/server/dictation/dictation-stream-manager.ts index 40667903a..32732c34d 100644 --- a/packages/server/src/server/dictation/dictation-stream-manager.ts +++ b/packages/server/src/server/dictation/dictation-stream-manager.ts @@ -117,7 +117,7 @@ export class DictationStreamManager { private readonly logger: pino.Logger; private readonly emit: (msg: DictationStreamOutboundMessage) => void; private readonly sessionId: string; - private readonly stt: SpeechToTextProvider | null; + private readonly resolveStt: () => SpeechToTextProvider | null; private readonly finalTimeoutMs: number; private readonly autoCommitSeconds: number; private readonly streams = new Map(); @@ -126,14 +126,19 @@ export class DictationStreamManager { logger: pino.Logger; emit: (msg: DictationStreamOutboundMessage) => void; sessionId: string; - stt: SpeechToTextProvider | null; + stt: SpeechToTextProvider | null | (() => SpeechToTextProvider | null); finalTimeoutMs?: number; autoCommitSeconds?: number; }) { this.logger = params.logger.child({ component: "dictation-stream-manager" }); this.emit = params.emit; this.sessionId = params.sessionId; - this.stt = params.stt; + if (typeof params.stt === "function") { + this.resolveStt = params.stt; + } else { + const sttProvider = params.stt; + this.resolveStt = () => sttProvider; + } this.finalTimeoutMs = params.finalTimeoutMs ?? DEFAULT_DICTATION_FINAL_TIMEOUT_MS; this.autoCommitSeconds = params.autoCommitSeconds ?? @@ -150,7 +155,8 @@ export class DictationStreamManager { public async handleStart(dictationId: string, format: string): Promise { this.cleanupDictationStream(dictationId); - if (!this.stt) { + const sttProvider = this.resolveStt(); + if (!sttProvider) { this.failDictationStream(dictationId, "Dictation STT not configured", false); return; } @@ -159,11 +165,18 @@ export class DictationStreamManager { process.env.PASEO_DICTATION_TRANSCRIPTION_PROMPT ?? "Transcribe only what the speaker says. Do not add words. Preserve punctuation and casing. If the audio is silence or non-speech noise, return an empty transcript."; - const stt = this.stt.createSession({ - logger: this.logger.child({ dictationId }), - language: "en", - prompt: transcriptionPrompt, - }); + let stt: ReturnType; + try { + stt = sttProvider.createSession({ + logger: this.logger.child({ dictationId }), + language: "en", + prompt: transcriptionPrompt, + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + this.failDictationStream(dictationId, message, false); + return; + } stt.on("committed", ({ segmentId }) => { const state = this.streams.get(dictationId); @@ -221,7 +234,18 @@ export class DictationStreamManager { void this.failAndCleanupDictationStream(dictationId, message, true); }); - await stt.connect(); + try { + await stt.connect(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + this.failDictationStream(dictationId, message, true); + try { + stt.close(); + } catch { + // no-op + } + return; + } const inputRate = parsePcmRateFromFormat(format, 16000) ?? 16000; if (!Number.isFinite(inputRate) || inputRate <= 0) { diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 539708c6c..db4d16a7b 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -130,6 +130,7 @@ import { listLocalSpeechModels, type LocalSpeechModelId, } from "./speech/providers/local/models.js"; +import type { SpeechReadinessSnapshot } from "./speech/speech-runtime.js"; import type pino from "pino"; const execAsync = promisify(exec); @@ -323,8 +324,8 @@ export type SessionOptions = { agentManager: AgentManager; agentStorage: AgentStorage; createAgentMcpTransport: AgentMcpTransportFactory; - stt: SpeechToTextProvider | null; - tts: TextToSpeechProvider | null; + stt: SpeechToTextProvider | null | (() => SpeechToTextProvider | null); + tts: TextToSpeechProvider | null | (() => TextToSpeechProvider | null); terminalManager: TerminalManager | null; voice?: { voiceAgentMcpStdio?: VoiceMcpStdioConfig | null; @@ -339,15 +340,37 @@ export type SessionOptions = { }; dictation?: { finalTimeoutMs?: number; - stt?: SpeechToTextProvider | null; + stt?: SpeechToTextProvider | null | (() => SpeechToTextProvider | null); localModels?: { modelsDir: string; defaultModelIds: LocalSpeechModelId[]; }; + getSpeechReadiness?: () => SpeechReadinessSnapshot; }; agentProviderRuntimeSettings?: AgentProviderRuntimeSettingsMap; }; +type VoiceFeatureUnavailableContext = { + reasonCode: SpeechReadinessSnapshot["voiceFeature"]["reasonCode"]; + message: string; + retryable: boolean; + missingModelIds: LocalSpeechModelId[]; +}; + +class VoiceFeatureUnavailableError extends Error { + readonly reasonCode: SpeechReadinessSnapshot["voiceFeature"]["reasonCode"]; + readonly retryable: boolean; + readonly missingModelIds: LocalSpeechModelId[]; + + constructor(context: VoiceFeatureUnavailableContext) { + super(context.message); + this.name = "VoiceFeatureUnavailableError"; + this.reasonCode = context.reasonCode; + this.retryable = context.retryable; + this.missingModelIds = [...context.missingModelIds]; + } +} + function convertPCMToWavBuffer( pcmBuffer: Buffer, sampleRate: number, @@ -533,6 +556,7 @@ export class Session { private readonly unregisterVoiceCallerContext?: (agentId: string) => void; private readonly ensureVoiceMcpSocketForAgent?: (agentId: string) => Promise; private readonly removeVoiceMcpSocketForAgent?: (agentId: string) => Promise; + private readonly getSpeechReadiness?: () => SpeechReadinessSnapshot; private readonly agentProviderRuntimeSettings: AgentProviderRuntimeSettingsMap | undefined; private voiceModeAgentId: string | null = null; private voiceModeBaseConfig: VoiceModeBaseConfig | null = null; @@ -584,6 +608,7 @@ export class Session { this.unregisterVoiceCallerContext = voiceBridge?.unregisterVoiceCallerContext; this.ensureVoiceMcpSocketForAgent = voiceBridge?.ensureVoiceMcpSocketForAgent; this.removeVoiceMcpSocketForAgent = voiceBridge?.removeVoiceMcpSocketForAgent; + this.getSpeechReadiness = dictation?.getSpeechReadiness; this.agentProviderRuntimeSettings = agentProviderRuntimeSettings; this.abortController = new AbortController(); this.sessionLogger = logger.child({ @@ -1192,6 +1217,22 @@ export class Session { break; case "dictation_stream_start": + { + const unavailable = this.resolveVoiceFeatureUnavailableContext("dictation"); + if (unavailable) { + this.emit({ + type: "dictation_stream_error", + payload: { + dictationId: msg.dictationId, + error: unavailable.message, + retryable: unavailable.retryable, + reasonCode: unavailable.reasonCode, + missingModelIds: unavailable.missingModelIds, + }, + }); + break; + } + } await this.dictationStreamManager.handleStart(msg.dictationId, msg.format); break; @@ -1716,6 +1757,50 @@ export class Session { } } + private toVoiceFeatureUnavailableContext( + state: SpeechReadinessSnapshot["voiceFeature"] + ): VoiceFeatureUnavailableContext { + return { + reasonCode: state.reasonCode, + message: state.message, + retryable: state.retryable, + missingModelIds: [...state.missingModelIds], + }; + } + + private resolveVoiceFeatureUnavailableContext( + mode: "voice_mode" | "dictation" + ): VoiceFeatureUnavailableContext | null { + const readiness = this.getSpeechReadiness?.(); + if (!readiness) { + return null; + } + + if (mode === "voice_mode") { + if (!readiness.realtimeVoice.enabled) { + return this.toVoiceFeatureUnavailableContext(readiness.realtimeVoice); + } + if (!readiness.voiceFeature.available) { + return this.toVoiceFeatureUnavailableContext(readiness.voiceFeature); + } + if (!readiness.realtimeVoice.available) { + return this.toVoiceFeatureUnavailableContext(readiness.realtimeVoice); + } + return null; + } + + if (!readiness.dictation.enabled) { + return this.toVoiceFeatureUnavailableContext(readiness.dictation); + } + if (!readiness.voiceFeature.available) { + return this.toVoiceFeatureUnavailableContext(readiness.voiceFeature); + } + if (!readiness.dictation.available) { + return this.toVoiceFeatureUnavailableContext(readiness.dictation); + } + return null; + } + /** * Handle voice mode toggle */ @@ -1726,6 +1811,11 @@ export class Session { ): Promise { try { if (enabled) { + const unavailable = this.resolveVoiceFeatureUnavailableContext("voice_mode"); + if (unavailable) { + throw new VoiceFeatureUnavailableError(unavailable); + } + const normalizedAgentId = this.parseVoiceTargetAgentId( agentId ?? "", "set_voice_mode" @@ -1784,6 +1874,14 @@ export class Session { } catch (error) { const errorMessage = error instanceof Error ? error.message : "Failed to set voice mode"; + const unavailable = + error instanceof VoiceFeatureUnavailableError + ? { + reasonCode: error.reasonCode, + retryable: error.retryable, + missingModelIds: error.missingModelIds, + } + : null; this.sessionLogger.error( { err: error, @@ -1801,6 +1899,9 @@ export class Session { agentId: this.voiceModeAgentId, accepted: false, error: errorMessage, + ...(unavailable ? { reasonCode: unavailable.reasonCode } : {}), + ...(unavailable ? { retryable: unavailable.retryable } : {}), + ...(unavailable ? { missingModelIds: unavailable.missingModelIds } : {}), }, }); return; diff --git a/packages/server/src/server/speech/providers/local/runtime.ts b/packages/server/src/server/speech/providers/local/runtime.ts index 41c559626..44303f7a1 100644 --- a/packages/server/src/server/speech/providers/local/runtime.ts +++ b/packages/server/src/server/speech/providers/local/runtime.ts @@ -186,10 +186,12 @@ export async function initializeLocalSpeechServices(params: { providers: RequestedSpeechProviders; speechConfig: PaseoSpeechConfig | null; logger: Logger; + modelEnsureAutoDownload?: boolean; }): Promise { const { providers, logger, speechConfig } = params; const localConfig = speechConfig?.local ?? null; const localModels = resolveConfiguredLocalModels(speechConfig); + const modelEnsureAutoDownload = params.modelEnsureAutoDownload ?? (localConfig?.autoDownload ?? true); let sttService: SpeechToTextProvider | null = null; let ttsService: TextToSpeechProvider | null = null; @@ -207,14 +209,14 @@ export async function initializeLocalSpeechServices(params: { { modelsDir: localConfig.modelsDir, modelIds: requiredLocalModelIds, - autoDownload: localConfig.autoDownload ?? true, + autoDownload: modelEnsureAutoDownload, }, "Ensuring local speech models" ); await ensureLocalSpeechModels({ modelsDir: localConfig.modelsDir, modelIds: requiredLocalModelIds, - autoDownload: localConfig.autoDownload ?? true, + autoDownload: modelEnsureAutoDownload, logger, }); } catch (err) { @@ -223,7 +225,7 @@ export async function initializeLocalSpeechServices(params: { err, modelsDir: localConfig.modelsDir, modelIds: requiredLocalModelIds, - autoDownload: localConfig.autoDownload ?? true, + autoDownload: modelEnsureAutoDownload, hint: "Use `paseo speech models` to inspect status and " + "`paseo speech download --model ` to fetch missing models.", diff --git a/packages/server/src/server/speech/speech-runtime.test.ts b/packages/server/src/server/speech/speech-runtime.test.ts new file mode 100644 index 000000000..fc2fe097c --- /dev/null +++ b/packages/server/src/server/speech/speech-runtime.test.ts @@ -0,0 +1,140 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import pino from "pino"; + +import type { PaseoSpeechConfig } from "../bootstrap.js"; +import type { InitializedLocalSpeech } from "./providers/local/runtime.js"; +import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech-provider.js"; +import { initializeSpeechRuntime } from "./speech-runtime.js"; + +const { initializeLocalSpeechServicesMock } = vi.hoisted(() => ({ + initializeLocalSpeechServicesMock: vi.fn< + (args: unknown) => Promise + >(), +})); + +vi.mock("./providers/local/runtime.js", () => ({ + initializeLocalSpeechServices: initializeLocalSpeechServicesMock, +})); + +vi.mock("./providers/openai/runtime.js", () => ({ + getOpenAiSpeechAvailability: () => ({ configured: false }), + initializeOpenAiSpeechServices: (args: { + existing: { + sttService: SpeechToTextProvider | null; + ttsService: TextToSpeechProvider | null; + dictationSttService: SpeechToTextProvider | null; + }; + }) => ({ + sttService: args.existing.sttService, + ttsService: args.existing.ttsService, + dictationSttService: args.existing.dictationSttService, + }), + validateOpenAiCredentialRequirements: () => {}, +})); + +vi.mock("./providers/local/models.js", () => ({ + ensureLocalSpeechModels: vi.fn(async () => {}), + getLocalSpeechModelDir: vi.fn(() => ""), + listLocalSpeechModels: vi.fn(() => []), +})); + +function createStubStt(id: string): SpeechToTextProvider { + return { + id, + createSession: vi.fn(() => { + throw new Error("not used in this test"); + }), + }; +} + +function createStubTts(id: string): TextToSpeechProvider { + return { + id, + synthesizeSpeech: vi.fn(async () => { + throw new Error("not used in this test"); + }), + }; +} + +function createSpeechConfig( + providers: PaseoSpeechConfig["providers"] +): PaseoSpeechConfig { + return { providers }; +} + +describe("initializeSpeechRuntime readiness", () => { + beforeEach(() => { + initializeLocalSpeechServicesMock.mockReset(); + }); + + it("keeps voice feature available when only dictation is enabled and ready", async () => { + const dictationStt = createStubStt("dictation-local"); + + initializeLocalSpeechServicesMock.mockResolvedValue({ + sttService: null, + ttsService: null, + dictationSttService: dictationStt, + localVoiceTtsProvider: null, + localModelConfig: null, + availability: { + configured: false, + modelsDir: null, + autoDownload: null, + }, + cleanup: () => {}, + }); + + const runtime = await initializeSpeechRuntime({ + logger: pino({ level: "silent" }), + speechConfig: createSpeechConfig({ + dictationStt: { provider: "local", enabled: true, explicit: true }, + voiceStt: { provider: "local", enabled: false, explicit: true }, + voiceTts: { provider: "local", enabled: false, explicit: true }, + }), + }); + + const readiness = runtime.getSpeechReadiness(); + expect(readiness.dictation.available).toBe(true); + expect(readiness.realtimeVoice.reasonCode).toBe("disabled"); + expect(readiness.voiceFeature.available).toBe(true); + expect(readiness.voiceFeature.reasonCode).toBe("ready"); + + runtime.cleanup(); + }); + + it("keeps voice feature available when only realtime voice is enabled and ready", async () => { + const voiceStt = createStubStt("voice-local"); + const voiceTts = createStubTts("tts-local"); + + initializeLocalSpeechServicesMock.mockResolvedValue({ + sttService: voiceStt, + ttsService: voiceTts, + dictationSttService: null, + localVoiceTtsProvider: voiceTts, + localModelConfig: null, + availability: { + configured: false, + modelsDir: null, + autoDownload: null, + }, + cleanup: () => {}, + }); + + const runtime = await initializeSpeechRuntime({ + logger: pino({ level: "silent" }), + speechConfig: createSpeechConfig({ + dictationStt: { provider: "local", enabled: false, explicit: true }, + voiceStt: { provider: "local", enabled: true, explicit: true }, + voiceTts: { provider: "local", enabled: true, explicit: true }, + }), + }); + + const readiness = runtime.getSpeechReadiness(); + expect(readiness.realtimeVoice.available).toBe(true); + expect(readiness.dictation.reasonCode).toBe("disabled"); + expect(readiness.voiceFeature.available).toBe(true); + expect(readiness.voiceFeature.reasonCode).toBe("ready"); + + runtime.cleanup(); + }); +}); diff --git a/packages/server/src/server/speech/speech-runtime.ts b/packages/server/src/server/speech/speech-runtime.ts index dbd35e3a8..6b88be981 100644 --- a/packages/server/src/server/speech/speech-runtime.ts +++ b/packages/server/src/server/speech/speech-runtime.ts @@ -1,11 +1,15 @@ +import { stat } from "node:fs/promises"; +import { join } from "node:path"; import type { Logger } from "pino"; import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js"; -import { - getLocalSpeechAvailability, - initializeLocalSpeechServices, -} from "./providers/local/runtime.js"; import type { LocalSpeechModelId } from "./providers/local/config.js"; +import { + ensureLocalSpeechModels, + getLocalSpeechModelDir, + listLocalSpeechModels, +} from "./providers/local/models.js"; +import { initializeLocalSpeechServices } from "./providers/local/runtime.js"; import { getOpenAiSpeechAvailability, initializeOpenAiSpeechServices, @@ -14,6 +18,39 @@ import { import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech-provider.js"; import type { RequestedSpeechProviders } from "./speech-types.js"; +const SPEECH_RUNTIME_MONITOR_INTERVAL_MS = 3000; + +export type SpeechReadinessReasonCode = + | "ready" + | "disabled" + | "model_download_in_progress" + | "models_missing" + | "model_download_failed" + | "stt_unavailable" + | "tts_unavailable"; + +export type SpeechReadinessState = { + enabled: boolean; + available: boolean; + reasonCode: SpeechReadinessReasonCode; + message: string; + retryable: boolean; + missingModelIds: LocalSpeechModelId[]; +}; + +export type SpeechReadinessSnapshot = { + generatedAt: string; + requiredLocalModelIds: LocalSpeechModelId[]; + missingLocalModelIds: LocalSpeechModelId[]; + download: { + inProgress: boolean; + error: string | null; + }; + realtimeVoice: SpeechReadinessState; + dictation: SpeechReadinessState; + voiceFeature: SpeechReadinessState; +}; + function resolveRequestedSpeechProviders( speechConfig: PaseoSpeechConfig | null ): RequestedSpeechProviders { @@ -29,10 +66,248 @@ function resolveRequestedSpeechProviders( }; } -export type InitializedSpeechRuntime = { +async function hasRequiredLocalModelFile(filePath: string): Promise { + try { + const fileStat = await stat(filePath); + if (fileStat.isDirectory()) { + return true; + } + return fileStat.isFile() && fileStat.size > 0; + } catch { + return false; + } +} + +async function findMissingRequiredLocalModels(params: { + modelsDir: string | null; + requiredModelIds: LocalSpeechModelId[]; +}): Promise { + const { modelsDir, requiredModelIds } = params; + if (!modelsDir || requiredModelIds.length === 0) { + return []; + } + + const specsById = new Map(listLocalSpeechModels().map((model) => [model.id, model])); + const missing = new Set(); + + for (const modelId of requiredModelIds) { + const spec = specsById.get(modelId); + if (!spec) { + missing.add(modelId); + continue; + } + const modelDir = getLocalSpeechModelDir(modelsDir, modelId); + for (const relPath of spec.requiredFiles) { + const filePath = join(modelDir, relPath); + if (!(await hasRequiredLocalModelFile(filePath))) { + missing.add(modelId); + break; + } + } + } + + return Array.from(missing); +} + +function joinModelIds(modelIds: LocalSpeechModelId[]): string { + if (modelIds.length === 0) { + return "none"; + } + return modelIds.join(", "); +} + +function buildRealtimeVoiceReadiness(params: { + providers: RequestedSpeechProviders; + sttService: SpeechToTextProvider | null; + ttsService: TextToSpeechProvider | null; +}): SpeechReadinessState { + const voiceSttEnabled = params.providers.voiceStt.enabled !== false; + const voiceTtsEnabled = params.providers.voiceTts.enabled !== false; + const enabled = voiceSttEnabled || voiceTtsEnabled; + if (!enabled) { + return { + enabled: false, + available: false, + reasonCode: "disabled", + message: "Realtime voice is disabled in daemon config.", + retryable: false, + missingModelIds: [], + }; + } + if (voiceSttEnabled && !params.sttService) { + return { + enabled: true, + available: false, + reasonCode: "stt_unavailable", + message: "Realtime voice is unavailable: speech-to-text service is not ready.", + retryable: false, + missingModelIds: [], + }; + } + if (voiceTtsEnabled && !params.ttsService) { + return { + enabled: true, + available: false, + reasonCode: "tts_unavailable", + message: "Realtime voice is unavailable: text-to-speech service is not ready.", + retryable: false, + missingModelIds: [], + }; + } + return { + enabled: true, + available: true, + reasonCode: "ready", + message: "Realtime voice is ready.", + retryable: false, + missingModelIds: [], + }; +} + +function buildDictationReadiness(params: { + providers: RequestedSpeechProviders; + dictationSttService: SpeechToTextProvider | null; +}): SpeechReadinessState { + const enabled = params.providers.dictationStt.enabled !== false; + if (!enabled) { + return { + enabled: false, + available: false, + reasonCode: "disabled", + message: "Dictation is disabled in daemon config.", + retryable: false, + missingModelIds: [], + }; + } + if (!params.dictationSttService) { + return { + enabled: true, + available: false, + reasonCode: "stt_unavailable", + message: "Dictation is unavailable: speech-to-text service is not ready.", + retryable: false, + missingModelIds: [], + }; + } + return { + enabled: true, + available: true, + reasonCode: "ready", + message: "Dictation is ready.", + retryable: false, + missingModelIds: [], + }; +} + +function buildVoiceFeatureReadiness(params: { + realtimeVoice: SpeechReadinessState; + dictation: SpeechReadinessState; + missingLocalModelIds: LocalSpeechModelId[]; + backgroundDownloadInProgress: boolean; + backgroundDownloadError: string | null; + localAutoDownloadEnabled: boolean; +}): SpeechReadinessState { + const enabled = params.realtimeVoice.enabled || params.dictation.enabled; + if (!enabled) { + return { + enabled: false, + available: false, + reasonCode: "disabled", + message: "Voice features are disabled in daemon config.", + retryable: false, + missingModelIds: [], + }; + } + + if (params.missingLocalModelIds.length > 0) { + const missingModelIds = [...params.missingLocalModelIds]; + if (params.backgroundDownloadInProgress) { + return { + enabled: true, + available: false, + reasonCode: "model_download_in_progress", + message: `Voice features are unavailable while models download in the background (${joinModelIds(missingModelIds)}).`, + retryable: true, + missingModelIds, + }; + } + if (params.backgroundDownloadError) { + return { + enabled: true, + available: false, + reasonCode: "model_download_failed", + message: `Voice features are unavailable: model download failed (${params.backgroundDownloadError}).`, + retryable: false, + missingModelIds, + }; + } + return { + enabled: true, + available: false, + reasonCode: "models_missing", + message: `Voice features are unavailable: missing local models (${joinModelIds(missingModelIds)}).`, + retryable: params.localAutoDownloadEnabled, + missingModelIds, + }; + } + + return { + enabled: true, + available: true, + reasonCode: "ready", + message: "Voice features are ready.", + retryable: false, + missingModelIds: [], + }; +} + +function describeRequestedProviders(providers: RequestedSpeechProviders): { + dictationStt: { provider: string; enabled: boolean; explicit: boolean }; + voiceStt: { provider: string; enabled: boolean; explicit: boolean }; + voiceTts: { provider: string; enabled: boolean; explicit: boolean }; +} { + return { + dictationStt: { + provider: providers.dictationStt.provider, + enabled: providers.dictationStt.enabled !== false, + explicit: providers.dictationStt.explicit, + }, + voiceStt: { + provider: providers.voiceStt.provider, + enabled: providers.voiceStt.enabled !== false, + explicit: providers.voiceStt.explicit, + }, + voiceTts: { + provider: providers.voiceTts.provider, + enabled: providers.voiceTts.enabled !== false, + explicit: providers.voiceTts.explicit, + }, + }; +} + +function resolveEffectiveProviderIds(params: { sttService: SpeechToTextProvider | null; ttsService: TextToSpeechProvider | null; dictationSttService: SpeechToTextProvider | null; + localVoiceTtsProvider: TextToSpeechProvider | null; +}): { dictationStt: string; voiceStt: string; voiceTts: string } { + return { + dictationStt: params.dictationSttService?.id ?? "unavailable", + voiceStt: params.sttService?.id ?? "unavailable", + voiceTts: + !params.ttsService + ? "unavailable" + : params.ttsService === params.localVoiceTtsProvider + ? "local" + : "openai", + }; +} + +export type InitializedSpeechRuntime = { + resolveVoiceStt: () => SpeechToTextProvider | null; + resolveVoiceTts: () => TextToSpeechProvider | null; + resolveDictationStt: () => SpeechToTextProvider | null; + getSpeechReadiness: () => SpeechReadinessSnapshot; cleanup: () => void; localModelConfig: { modelsDir: string; @@ -45,10 +320,12 @@ export async function initializeSpeechRuntime(params: { openaiConfig?: PaseoOpenAIConfig; speechConfig?: PaseoSpeechConfig; }): Promise { - const logger = params.logger; + const logger = params.logger.child({ module: "speech-runtime" }); const speechConfig = params.speechConfig ?? null; const openaiConfig = params.openaiConfig; const providers = resolveRequestedSpeechProviders(speechConfig); + const requestedProviders = describeRequestedProviders(providers); + const localAutoDownloadEnabled = speechConfig?.local?.autoDownload ?? true; validateOpenAiCredentialRequirements({ providers, @@ -58,145 +335,264 @@ export async function initializeSpeechRuntime(params: { logger.info( { - requestedProviders: { - dictationStt: { - provider: providers.dictationStt.provider, - enabled: providers.dictationStt.enabled !== false, - }, - voiceStt: { - provider: providers.voiceStt.provider, - enabled: providers.voiceStt.enabled !== false, - }, - voiceTts: { - provider: providers.voiceTts.provider, - enabled: providers.voiceTts.enabled !== false, - }, - }, + requestedProviders, availability: { openai: getOpenAiSpeechAvailability(openaiConfig), - local: getLocalSpeechAvailability(speechConfig), }, }, "Speech provider reconciliation started" ); - const localSpeech = await initializeLocalSpeechServices({ - providers, - speechConfig, - logger, - }); + let sttService: SpeechToTextProvider | null = null; + let ttsService: TextToSpeechProvider | null = null; + let dictationSttService: SpeechToTextProvider | null = null; + let localModelConfig: { + modelsDir: string; + defaultModelIds: LocalSpeechModelId[]; + } | null = null; + let localCleanup = () => {}; + let localVoiceTtsProvider: TextToSpeechProvider | null = null; - const openAiSpeech = initializeOpenAiSpeechServices({ - providers, - openaiConfig, - existing: { - sttService: localSpeech.sttService, - ttsService: localSpeech.ttsService, - dictationSttService: localSpeech.dictationSttService, - }, - logger, - }); + let missingLocalModelIds: LocalSpeechModelId[] = []; + let backgroundDownloadInProgress = false; + let backgroundDownloadError: string | null = null; + let stopped = false; + let monitorTimeout: ReturnType | null = null; + let reconcileInFlight: Promise | null = null; - const effectiveProviders = { - dictationStt: openAiSpeech.dictationSttService?.id ?? "unavailable", - voiceStt: openAiSpeech.sttService?.id ?? "unavailable", - voiceTts: - !openAiSpeech.ttsService - ? "unavailable" - : openAiSpeech.ttsService === localSpeech.localVoiceTtsProvider - ? "local" - : "openai", + const computeReadinessSnapshot = (): SpeechReadinessSnapshot => { + const realtimeVoice = buildRealtimeVoiceReadiness({ + providers, + sttService, + ttsService, + }); + const dictation = buildDictationReadiness({ + providers, + dictationSttService, + }); + const voiceFeature = buildVoiceFeatureReadiness({ + realtimeVoice, + dictation, + missingLocalModelIds, + backgroundDownloadInProgress, + backgroundDownloadError, + localAutoDownloadEnabled, + }); + return { + generatedAt: new Date().toISOString(), + requiredLocalModelIds: localModelConfig?.defaultModelIds ?? [], + missingLocalModelIds: [...missingLocalModelIds], + download: { + inProgress: backgroundDownloadInProgress, + error: backgroundDownloadError, + }, + realtimeVoice: { + ...realtimeVoice, + }, + dictation: { + ...dictation, + }, + voiceFeature: { + ...voiceFeature, + }, + }; }; - const unavailableFeatures = [ - providers.dictationStt.enabled !== false && !openAiSpeech.dictationSttService - ? "dictation.stt" - : null, - providers.voiceStt.enabled !== false && !openAiSpeech.sttService ? "voice.stt" : null, - providers.voiceTts.enabled !== false && !openAiSpeech.ttsService ? "voice.tts" : null, - ].filter((feature): feature is string => feature !== null); - const explicitlyConfiguredUnavailableFeatures = unavailableFeatures.filter((feature) => { - if (feature === "dictation.stt") { - return providers.dictationStt.explicit; - } - if (feature === "voice.stt") { - return providers.voiceStt.explicit; - } - return providers.voiceTts.explicit; - }); - if (explicitlyConfiguredUnavailableFeatures.length > 0) { - logger.error( - { - requestedProviders: { - dictationStt: { - provider: providers.dictationStt.provider, - enabled: providers.dictationStt.enabled !== false, - }, - voiceStt: { - provider: providers.voiceStt.provider, - enabled: providers.voiceStt.enabled !== false, - }, - voiceTts: { - provider: providers.voiceTts.provider, - enabled: providers.voiceTts.enabled !== false, - }, - }, - explicitProviders: { - dictationStt: providers.dictationStt.explicit, - voiceStt: providers.voiceStt.explicit, - voiceTts: providers.voiceTts.explicit, - }, - effectiveProviders, - unavailableFeatures: explicitlyConfiguredUnavailableFeatures, - }, - "Speech provider reconciliation failed: configured features are unavailable" - ); - throw new Error( - `Configured speech features unavailable: ${explicitlyConfiguredUnavailableFeatures.join(", ")}` - ); - } + const refreshMissingLocalModels = async (): Promise => { + missingLocalModelIds = await findMissingRequiredLocalModels({ + modelsDir: localModelConfig?.modelsDir ?? null, + requiredModelIds: localModelConfig?.defaultModelIds ?? [], + }); + }; - if (unavailableFeatures.length > 0) { - logger.warn( - { - requestedProviders: { - dictationStt: { - provider: providers.dictationStt.provider, - enabled: providers.dictationStt.enabled !== false, - }, - voiceStt: { - provider: providers.voiceStt.provider, - enabled: providers.voiceStt.enabled !== false, - }, - voiceTts: { - provider: providers.voiceTts.provider, - enabled: providers.voiceTts.enabled !== false, - }, - }, - explicitProviders: { - dictationStt: providers.dictationStt.explicit, - voiceStt: providers.voiceStt.explicit, - voiceTts: providers.voiceTts.explicit, - }, - effectiveProviders, - unavailableFeatures, + const reconcileServices = async (modelEnsureAutoDownload: boolean): Promise => { + const nextLocalSpeech = await initializeLocalSpeechServices({ + providers, + speechConfig, + logger, + modelEnsureAutoDownload, + }); + const nextOpenAiSpeech = initializeOpenAiSpeechServices({ + providers, + openaiConfig, + existing: { + sttService: nextLocalSpeech.sttService, + ttsService: nextLocalSpeech.ttsService, + dictationSttService: nextLocalSpeech.dictationSttService, }, - "Speech provider reconciliation completed with unavailable default features" - ); - } else { + logger, + }); + + const previousLocalCleanup = localCleanup; + sttService = nextOpenAiSpeech.sttService; + ttsService = nextOpenAiSpeech.ttsService; + dictationSttService = nextOpenAiSpeech.dictationSttService; + localModelConfig = nextLocalSpeech.localModelConfig; + localVoiceTtsProvider = nextLocalSpeech.localVoiceTtsProvider; + localCleanup = nextLocalSpeech.cleanup; + previousLocalCleanup(); + + await refreshMissingLocalModels(); + + const effectiveProviders = resolveEffectiveProviderIds({ + sttService, + ttsService, + dictationSttService, + localVoiceTtsProvider, + }); + const unavailableFeatures = [ + providers.dictationStt.enabled !== false && !dictationSttService ? "dictation.stt" : null, + providers.voiceStt.enabled !== false && !sttService ? "voice.stt" : null, + providers.voiceTts.enabled !== false && !ttsService ? "voice.tts" : null, + ].filter((feature): feature is string => feature !== null); + + if (unavailableFeatures.length > 0) { + logger.warn( + { + requestedProviders, + effectiveProviders, + unavailableFeatures, + missingLocalModelIds, + }, + "Speech provider reconciliation completed with unavailable features" + ); + } else { + logger.info( + { + requestedProviders, + effectiveProviders, + }, + "Speech provider reconciliation completed" + ); + } + }; + + const runReconcile = async (params: { modelEnsureAutoDownload: boolean }): Promise => { + if (reconcileInFlight) { + await reconcileInFlight; + return; + } + reconcileInFlight = reconcileServices(params.modelEnsureAutoDownload).finally(() => { + reconcileInFlight = null; + }); + await reconcileInFlight; + }; + + const scheduleMonitor = (): void => { + if (stopped || monitorTimeout) { + return; + } + monitorTimeout = setTimeout(() => { + monitorTimeout = null; + void runMonitorTick(); + }, SPEECH_RUNTIME_MONITOR_INTERVAL_MS); + }; + + const startBackgroundDownload = (): void => { + if (stopped || backgroundDownloadInProgress) { + return; + } + const modelsDir = localModelConfig?.modelsDir ?? null; + const modelIds = [...missingLocalModelIds]; + if (!modelsDir || modelIds.length === 0 || !localAutoDownloadEnabled) { + return; + } + + backgroundDownloadInProgress = true; + backgroundDownloadError = null; + logger.info( { - effectiveProviders, + modelsDir, + modelIds, }, - "Speech provider reconciliation completed" + "Starting background download for missing local speech models" ); + + void (async () => { + try { + await ensureLocalSpeechModels({ + modelsDir, + modelIds, + autoDownload: true, + logger, + }); + await runReconcile({ modelEnsureAutoDownload: false }); + backgroundDownloadError = null; + } catch (error) { + backgroundDownloadError = error instanceof Error ? error.message : String(error); + logger.error( + { + err: error, + modelIds, + }, + "Background local speech model download failed" + ); + } finally { + backgroundDownloadInProgress = false; + await refreshMissingLocalModels().catch((error) => { + logger.warn({ err: error }, "Failed to refresh local speech model status after download"); + }); + scheduleMonitor(); + } + })(); + }; + + const runMonitorTick = async (): Promise => { + if (stopped) { + return; + } + try { + await refreshMissingLocalModels(); + const snapshot = computeReadinessSnapshot(); + if ( + snapshot.voiceFeature.enabled && + !snapshot.voiceFeature.available && + missingLocalModelIds.length === 0 && + !backgroundDownloadInProgress + ) { + await runReconcile({ modelEnsureAutoDownload: false }); + } + + if ( + missingLocalModelIds.length > 0 && + localAutoDownloadEnabled && + !backgroundDownloadInProgress && + !backgroundDownloadError + ) { + startBackgroundDownload(); + } + } catch (error) { + logger.warn({ err: error }, "Speech runtime monitor tick failed"); + } finally { + scheduleMonitor(); + } + }; + + await runReconcile({ modelEnsureAutoDownload: false }); + const snapshot = computeReadinessSnapshot(); + if (snapshot.voiceFeature.enabled && !snapshot.voiceFeature.available) { + if (missingLocalModelIds.length > 0 && localAutoDownloadEnabled) { + startBackgroundDownload(); + } + scheduleMonitor(); } + const cleanup = (): void => { + stopped = true; + if (monitorTimeout) { + clearTimeout(monitorTimeout); + monitorTimeout = null; + } + localCleanup(); + }; + return { - sttService: openAiSpeech.sttService, - ttsService: openAiSpeech.ttsService, - dictationSttService: openAiSpeech.dictationSttService, - cleanup: localSpeech.cleanup, - localModelConfig: localSpeech.localModelConfig, + resolveVoiceStt: () => sttService, + resolveVoiceTts: () => ttsService, + resolveDictationStt: () => dictationSttService, + getSpeechReadiness: () => computeReadinessSnapshot(), + cleanup, + localModelConfig, }; } diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index fe934b0c5..7bd527b86 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -26,6 +26,7 @@ import type { AgentProviderRuntimeSettingsMap } from "./agent/provider-launch-co import { PushTokenStore } from "./push/token-store.js"; import { PushService } from "./push/push-service.js"; import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js"; +import type { SpeechReadinessSnapshot } from "./speech/speech-runtime.js"; import type { LocalSpeechModelId } from "./speech/providers/local/models.js"; import type { VoiceCallerContext, @@ -93,16 +94,23 @@ export class VoiceAssistantWebSocketServer { private readonly pushTokenStore: PushTokenStore; private readonly pushService: PushService; private readonly createAgentMcpTransport: AgentMcpTransportFactory; - private readonly stt: SpeechToTextProvider | null; - private readonly tts: TextToSpeechProvider | null; + private readonly stt: + | SpeechToTextProvider + | null + | (() => SpeechToTextProvider | null); + private readonly tts: + | TextToSpeechProvider + | null + | (() => TextToSpeechProvider | null); private readonly terminalManager: TerminalManager | null; private readonly dictation: { finalTimeoutMs?: number; - stt?: SpeechToTextProvider | null; + stt?: SpeechToTextProvider | null | (() => SpeechToTextProvider | null); localModels?: { modelsDir: string; defaultModelIds: LocalSpeechModelId[]; }; + getSpeechReadiness?: () => SpeechReadinessSnapshot; } | null; private readonly voice: { voiceAgentMcpStdio?: VoiceMcpStdioConfig | null; @@ -126,7 +134,10 @@ export class VoiceAssistantWebSocketServer { paseoHome: string, createAgentMcpTransport: AgentMcpTransportFactory, wsConfig: WebSocketServerConfig, - speech?: { stt: SpeechToTextProvider | null; tts: TextToSpeechProvider | null }, + speech?: { + stt: SpeechToTextProvider | null | (() => SpeechToTextProvider | null); + tts: TextToSpeechProvider | null | (() => TextToSpeechProvider | null); + }, terminalManager?: TerminalManager | null, voice?: { voiceAgentMcpStdio?: VoiceMcpStdioConfig | null; @@ -135,11 +146,12 @@ export class VoiceAssistantWebSocketServer { }, dictation?: { finalTimeoutMs?: number; - stt?: SpeechToTextProvider | null; + stt?: SpeechToTextProvider | null | (() => SpeechToTextProvider | null); localModels?: { modelsDir: string; defaultModelIds: LocalSpeechModelId[]; }; + getSpeechReadiness?: () => SpeechReadinessSnapshot; }, agentProviderRuntimeSettings?: AgentProviderRuntimeSettingsMap ) { diff --git a/packages/server/src/shared/messages.ts b/packages/server/src/shared/messages.ts index cc4ed0d52..e3dacaecc 100644 --- a/packages/server/src/shared/messages.ts +++ b/packages/server/src/shared/messages.ts @@ -692,6 +692,9 @@ export const SetVoiceModeResponseMessageSchema = z.object({ agentId: z.string().nullable(), accepted: z.boolean(), error: z.string().nullable(), + reasonCode: z.string().optional(), + retryable: z.boolean().optional(), + missingModelIds: z.array(z.string()).optional(), }), }); @@ -1160,6 +1163,8 @@ export const DictationStreamErrorMessageSchema = z.object({ dictationId: z.string(), error: z.string(), retryable: z.boolean(), + reasonCode: z.string().optional(), + missingModelIds: z.array(z.string()).optional(), debugRecordingPath: z.string().optional(), }), });