feat(voice): background local model downloads with runtime gating

2026-07-29 12:01:31 +00:00 · 2026-02-13 23:23:28 +07:00
parent c41815dcb7
commit cfd061a86d
14 changed files with 874 additions and 392 deletions
--- a/packages/app/src/components/agent-input-area.tsx
+++ b/packages/app/src/components/agent-input-area.tsx
@@ -4,6 +4,7 @@ import {
  Text,
  ActivityIndicator,
  Platform,
+  Alert,
 } from "react-native";
 import { useState, useEffect, useRef, useCallback, useMemo } from "react";
 import { StyleSheet, useUnistyles } from "react-native-unistyles";
@@ -469,6 +470,8 @@ export function AgentInputArea({
    }
    void voice.startVoice(serverId, agentId).catch((error) => {
      console.error("[AgentInputArea] Failed to start voice mode", error);
+      const message = error instanceof Error ? error.message : "Voice features are not available right now.";
+      Alert.alert("Voice unavailable", message);
    });
  }, [agentId, isConnected, serverId, voice]);

--- a/packages/app/src/components/message-input.tsx
+++ b/packages/app/src/components/message-input.tsx
@@ -10,6 +10,7 @@ import {
  Image,
  Platform,
  BackHandler,
+  Alert,
 } from "react-native";
 import {
  useState,
@@ -297,6 +298,8 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
      }
      void voice.startVoice(voiceServerId, voiceAgentId).catch((error) => {
        console.error("[MessageInput] Failed to start realtime voice", error);
+        const message = error instanceof Error ? error.message : "Voice features are not available right now.";
+        Alert.alert("Voice unavailable", message);
      });
    };

@@ -461,6 +464,8 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
    }
    void voice.startVoice(voiceServerId, voiceAgentId).catch((error) => {
      console.error("[MessageInput] Failed to start realtime voice", error);
+      const message = error instanceof Error ? error.message : "Voice features are not available right now.";
+      Alert.alert("Voice unavailable", message);
    });
  }, [
    disabled,
--- a/packages/cli/src/commands/onboard.ts
+++ b/packages/cli/src/commands/onboard.ts
@@ -3,11 +3,9 @@ import { Command } from 'commander'
 import { writeFileSync } from 'node:fs'
 import path from 'node:path'
 import {
-  ensureLocalSpeechModels,
  generateLocalPairingOffer,
  loadConfig,
  loadPersistedConfig,
-  type LocalSpeechModelId,
  type CliConfigOverrides,
  type PersistedConfig,
 } from '@getpaseo/server'
@@ -154,7 +152,7 @@ async function resolveVoiceSelection(mode: OnboardOptions['voice']): Promise<boo
  }

  const answer = await confirm({
-    message: 'Enable voice features? (downloads local STT/TTS models now)',
+    message: 'Enable voice features? (downloads local STT/TTS models in background)',
    active: 'Yes',
    inactive: 'No',
    initialValue: false,
@@ -172,221 +170,6 @@ type DownloadProgress = {
  pct: number | null
 }

-type LocalModelDownloadProgress = {
-  modelId: string | null
-  pct: number | null
-}
-
-type LocalSpeechDownloadLogger = {
-  child: (_bindings: Record<string, unknown>) => LocalSpeechDownloadLogger
-  info: (obj?: unknown, msg?: string) => void
-  error: (_obj?: unknown, _msg?: string) => void
-}
-
-type LocalSpeechDownloadEvent =
-  | {
-      type: 'progress'
-      progress: LocalModelDownloadProgress
-    }
-  | {
-      type: 'phase'
-      phase: 'extracting' | 'verifying' | 'finalizing' | 'completed'
-    }
-
-function resolveRequiredLocalModelIds(config: ReturnType<typeof loadConfig>): LocalSpeechModelId[] {
-  const providers = config.speech?.providers
-  const local = config.speech?.local
-
-  if (!providers || !local) {
-    return []
-  }
-
-  const ids = new Set<LocalSpeechModelId>()
-
-  if (providers.dictationStt.enabled !== false && providers.dictationStt.provider === 'local') {
-    ids.add(local.models.dictationStt)
-  }
-  if (providers.voiceStt.enabled !== false && providers.voiceStt.provider === 'local') {
-    ids.add(local.models.voiceStt)
-  }
-  if (providers.voiceTts.enabled !== false && providers.voiceTts.provider === 'local') {
-    ids.add(local.models.voiceTts)
-  }
-
-  return Array.from(ids)
-}
-
-function parseLocalModelDownloadProgress(payload: unknown): LocalModelDownloadProgress | null {
-  if (!payload || typeof payload !== 'object') {
-    return null
-  }
-
-  const value = payload as Record<string, unknown>
-  const modelId = typeof value.modelId === 'string' ? value.modelId : null
-  const pctRaw = value.pct
-  const pct = typeof pctRaw === 'number' && Number.isFinite(pctRaw) ? Math.max(0, Math.min(100, Math.floor(pctRaw))) : null
-
-  return {
-    modelId,
-    pct,
-  }
-}
-
-function renderLocalModelProgress(params: {
-  modelId: LocalSpeechModelId
-  modelIndex: number
-  modelCount: number
-  pct: number | null
-}): string {
-  const prefix = `Downloading speech model ${params.modelIndex}/${params.modelCount}: ${params.modelId}`
-  if (params.pct === null) {
-    return `${prefix}...`
-  }
-  return `${prefix} (${params.pct}%)`
-}
-
-function createLocalSpeechDownloadLogger(
-  onEvent: (event: LocalSpeechDownloadEvent) => void
-): LocalSpeechDownloadLogger {
-  const logger: LocalSpeechDownloadLogger = {
-    child: () => logger,
-    info: (obj?: unknown, msg?: string) => {
-      if (msg === 'Downloading model artifact') {
-        const progress = parseLocalModelDownloadProgress(obj)
-        if (!progress) {
-          return
-        }
-        onEvent({
-          type: 'progress',
-          progress,
-        })
-        return
-      }
-      if (msg === 'Extracting model archive') {
-        onEvent({ type: 'phase', phase: 'extracting' })
-        return
-      }
-      if (msg === 'Verifying downloaded model files') {
-        onEvent({ type: 'phase', phase: 'verifying' })
-        return
-      }
-      if (msg === 'Finalizing model artifacts') {
-        onEvent({ type: 'phase', phase: 'finalizing' })
-        return
-      }
-      if (msg === 'Model download completed') {
-        onEvent({ type: 'phase', phase: 'completed' })
-        return
-      }
-    },
-    error: () => {
-      // no-op: onboarding handles surfaced errors from ensureLocalSpeechModels.
-    },
-  }
-  return logger
-}
-
-async function prepareLocalSpeechModelsBeforeStart(args: {
-  config: ReturnType<typeof loadConfig>
-  richUi: boolean
-}): Promise<void> {
-  const local = args.config.speech?.local
-  const modelIds = resolveRequiredLocalModelIds(args.config)
-
-  if (!local || modelIds.length === 0) {
-    return
-  }
-
-  if (local.autoDownload === false) {
-    log.warn('Local speech model auto-download is disabled. Voice may be unavailable until models are installed.')
-    return
-  }
-
-  const modelList = modelIds.join(', ')
-  const modelCount = modelIds.length
-  const downloadSpinner = args.richUi ? spinner() : null
-  let lastPlainStatus = ''
-
-  const emitStatus = (status: string): void => {
-    if (downloadSpinner) {
-      downloadSpinner.message(status)
-      return
-    }
-    if (status === lastPlainStatus) {
-      return
-    }
-    console.log(status)
-    lastPlainStatus = status
-  }
-
-  if (downloadSpinner) {
-    downloadSpinner.start(`Preparing local speech models (${modelCount})...`)
-  } else {
-    log.message(`Preparing local speech models (${modelCount}): ${modelList}`)
-  }
-
-  try {
-    for (const [index, modelId] of modelIds.entries()) {
-      const modelIndex = index + 1
-      emitStatus(`Checking speech model ${modelIndex}/${modelCount}: ${modelId}`)
-
-      const perModelLogger = createLocalSpeechDownloadLogger((event) => {
-        if (event.type === 'progress') {
-          const progress = event.progress
-          if (progress.modelId && progress.modelId !== modelId) {
-            return
-          }
-          emitStatus(
-            renderLocalModelProgress({
-              modelId,
-              modelIndex,
-              modelCount,
-              pct: progress.pct,
-            })
-          )
-          return
-        }
-
-        if (event.phase === 'extracting') {
-          emitStatus(`Extracting speech model ${modelIndex}/${modelCount}: ${modelId}`)
-          return
-        }
-        if (event.phase === 'verifying') {
-          emitStatus(`Verifying speech model ${modelIndex}/${modelCount}: ${modelId}`)
-          return
-        }
-        if (event.phase === 'finalizing') {
-          emitStatus(`Finalizing speech model ${modelIndex}/${modelCount}: ${modelId}`)
-          return
-        }
-      })
-
-      await ensureLocalSpeechModels({
-        modelsDir: local.modelsDir,
-        modelIds: [modelId],
-        autoDownload: true,
-        logger: perModelLogger as any,
-      })
-
-      emitStatus(`Speech model ready ${modelIndex}/${modelCount}: ${modelId}`)
-    }
-
-    if (downloadSpinner) {
-      downloadSpinner.stop(`Local speech models ready (${modelCount})`)
-    } else {
-      log.message(`Local speech models ready (${modelCount}): ${modelList}`)
-    }
-  } catch (error) {
-    const message = error instanceof Error ? error.message : String(error)
-    if (downloadSpinner) {
-      downloadSpinner.error(`Failed to prepare local speech models: ${message}`)
-    } else {
-      log.error(`Failed to prepare local speech models: ${message}`)
-    }
-    throw error
-  }
-}
-
 function parseDownloadProgress(logTail: string): DownloadProgress | null {
  const lines = logTail.split('\n').filter(Boolean)

@@ -583,19 +366,10 @@ export async function runOnboard(options: OnboardOptions): Promise<void> {
  const config = loadConfig(paseoHome, { cli: toCliOverrides(options) })

  const voiceStatus = voiceEnabled
-    ? 'Voice features enabled. Local speech models will be downloaded if missing.'
-    : 'Voice features disabled. Local speech models will not be downloaded now.'
+    ? 'Voice features enabled. Local speech models will download in the background if missing.'
+    : 'Voice features disabled. Local speech models will not be downloaded.'
  log.message(voiceStatus)

-  try {
-    await prepareLocalSpeechModelsBeforeStart({
-      config,
-      richUi,
-    })
-  } catch {
-    process.exit(1)
-  }
-
  const stateBeforeStart = resolveLocalDaemonState({ home: options.home })
  const startSpinner = richUi ? spinner() : null

--- a/packages/server/src/client/daemon-client.ts
+++ b/packages/server/src/client/daemon-client.ts
@@ -1465,7 +1465,7 @@ export class DaemonClient {
      ...(agentId ? { agentId } : {}),
      requestId,
    });
-    return this.sendRequest({
+    const response = await this.sendRequest({
      requestId,
      message,
      timeout: 10000,
@@ -1479,6 +1479,14 @@ export class DaemonClient {
        return msg.payload;
      },
    });
+    if (!response.accepted) {
+      const codeSuffix =
+        typeof response.reasonCode === "string" && response.reasonCode.trim().length > 0
+          ? ` (${response.reasonCode})`
+          : "";
+      throw new Error((response.error ?? "Failed to set voice mode") + codeSuffix);
+    }
+    return response;
  }

  async sendVoiceAudioChunk(
--- a/packages/server/src/server/agent/stt-manager.ts
+++ b/packages/server/src/server/agent/stt-manager.ts
@@ -39,12 +39,16 @@ export interface SessionTranscriptionResult extends TranscriptionResult {
 export class STTManager {
  private readonly sessionId: string;
  private readonly logger: pino.Logger;
-  private readonly stt: SpeechToTextProvider | null;
+  private readonly resolveStt: () => SpeechToTextProvider | null;

-  constructor(sessionId: string, logger: pino.Logger, stt: SpeechToTextProvider | null) {
+  constructor(
+    sessionId: string,
+    logger: pino.Logger,
+    stt: SpeechToTextProvider | null | (() => SpeechToTextProvider | null)
+  ) {
    this.sessionId = sessionId;
    this.logger = logger.child({ module: "agent", component: "stt-manager", sessionId });
-    this.stt = stt;
+    this.resolveStt = typeof stt === "function" ? stt : () => stt;
  }

  /**
@@ -55,7 +59,8 @@ export class STTManager {
    format: string,
    metadata?: TranscriptionMetadata
  ): Promise<SessionTranscriptionResult> {
-    if (!this.stt) {
+    const stt = this.resolveStt();
+    if (!stt) {
      throw new Error("STT not configured");
    }

@@ -81,7 +86,7 @@ export class STTManager {
      this.logger.warn({ err: error }, "Failed to persist debug audio");
    }

-    const session = this.stt.createSession({
+    const session = stt.createSession({
      logger: this.logger.child({ component: "stt-session" }),
      language: "en",
    });
--- a/packages/server/src/server/agent/tts-manager.ts
+++ b/packages/server/src/server/agent/tts-manager.ts
@@ -95,11 +95,15 @@ function splitTextForTts(text: string, maxChars: number): string[] {
 export class TTSManager {
  private pendingPlaybacks: Map<string, PendingPlayback> = new Map();
  private readonly logger: pino.Logger;
-  private readonly tts: TextToSpeechProvider | null;
+  private readonly resolveTts: () => TextToSpeechProvider | null;

-  constructor(sessionId: string, logger: pino.Logger, tts: TextToSpeechProvider | null) {
+  constructor(
+    sessionId: string,
+    logger: pino.Logger,
+    tts: TextToSpeechProvider | null | (() => TextToSpeechProvider | null)
+  ) {
    this.logger = logger.child({ module: "agent", component: "tts-manager", sessionId });
-    this.tts = tts;
+    this.resolveTts = typeof tts === "function" ? tts : () => tts;
  }

  /**
@@ -143,7 +147,8 @@ export class TTSManager {
    abortSignal: AbortSignal,
    isVoiceMode: boolean
  ): Promise<void> {
-    if (!this.tts) {
+    const tts = this.resolveTts();
+    if (!tts) {
      throw new Error("TTS not configured");
    }

@@ -153,7 +158,7 @@ export class TTSManager {
    }

    // Generate TTS audio stream
-    const { stream, format } = await this.tts.synthesizeSpeech(text);
+    const { stream, format } = await tts.synthesizeSpeech(text);

    if (abortSignal.aborted) {
      this.logger.debug("Aborted after generating audio");
--- a/packages/server/src/server/bootstrap.ts
+++ b/packages/server/src/server/bootstrap.ts
@@ -457,9 +457,10 @@ export async function createPaseoDaemon(
    },
  });
  const {
-    sttService,
-    ttsService,
-    dictationSttService,
+    resolveVoiceStt,
+    resolveVoiceTts,
+    resolveDictationStt,
+    getSpeechReadiness,
    cleanup: cleanupSpeechRuntime,
    localModelConfig,
  } = await initializeSpeechRuntime({
@@ -478,7 +479,7 @@ export async function createPaseoDaemon(
    config.paseoHome,
    createInMemoryAgentMcpTransport,
    { allowedOrigins, allowedHosts: config.allowedHosts },
-    { stt: sttService, tts: ttsService },
+    { stt: resolveVoiceStt, tts: resolveVoiceTts },
    terminalManager,
    {
      voiceAgentMcpStdio: {
@@ -496,8 +497,9 @@ export async function createPaseoDaemon(
    },
    {
      finalTimeoutMs: config.dictationFinalTimeoutMs,
-      stt: dictationSttService,
+      stt: resolveDictationStt,
      localModels: localModelConfig ?? undefined,
+      getSpeechReadiness,
    },
    config.agentProviderSettings
  );
--- a/packages/server/src/server/dictation/dictation-stream-manager.ts
+++ b/packages/server/src/server/dictation/dictation-stream-manager.ts
@@ -117,7 +117,7 @@ export class DictationStreamManager {
  private readonly logger: pino.Logger;
  private readonly emit: (msg: DictationStreamOutboundMessage) => void;
  private readonly sessionId: string;
-  private readonly stt: SpeechToTextProvider | null;
+  private readonly resolveStt: () => SpeechToTextProvider | null;
  private readonly finalTimeoutMs: number;
  private readonly autoCommitSeconds: number;
  private readonly streams = new Map<string, DictationStreamState>();
@@ -126,14 +126,19 @@ export class DictationStreamManager {
    logger: pino.Logger;
    emit: (msg: DictationStreamOutboundMessage) => void;
    sessionId: string;
-    stt: SpeechToTextProvider | null;
+    stt: SpeechToTextProvider | null | (() => SpeechToTextProvider | null);
    finalTimeoutMs?: number;
    autoCommitSeconds?: number;
  }) {
    this.logger = params.logger.child({ component: "dictation-stream-manager" });
    this.emit = params.emit;
    this.sessionId = params.sessionId;
-    this.stt = params.stt;
+    if (typeof params.stt === "function") {
+      this.resolveStt = params.stt;
+    } else {
+      const sttProvider = params.stt;
+      this.resolveStt = () => sttProvider;
+    }
    this.finalTimeoutMs = params.finalTimeoutMs ?? DEFAULT_DICTATION_FINAL_TIMEOUT_MS;
    this.autoCommitSeconds =
      params.autoCommitSeconds ??
@@ -150,7 +155,8 @@ export class DictationStreamManager {
  public async handleStart(dictationId: string, format: string): Promise<void> {
    this.cleanupDictationStream(dictationId);

-    if (!this.stt) {
+    const sttProvider = this.resolveStt();
+    if (!sttProvider) {
      this.failDictationStream(dictationId, "Dictation STT not configured", false);
      return;
    }
@@ -159,11 +165,18 @@ export class DictationStreamManager {
      process.env.PASEO_DICTATION_TRANSCRIPTION_PROMPT ??
      "Transcribe only what the speaker says. Do not add words. Preserve punctuation and casing. If the audio is silence or non-speech noise, return an empty transcript.";

-    const stt = this.stt.createSession({
-      logger: this.logger.child({ dictationId }),
-      language: "en",
-      prompt: transcriptionPrompt,
-    });
+    let stt: ReturnType<SpeechToTextProvider["createSession"]>;
+    try {
+      stt = sttProvider.createSession({
+        logger: this.logger.child({ dictationId }),
+        language: "en",
+        prompt: transcriptionPrompt,
+      });
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      this.failDictationStream(dictationId, message, false);
+      return;
+    }

    stt.on("committed", ({ segmentId }) => {
      const state = this.streams.get(dictationId);
@@ -221,7 +234,18 @@ export class DictationStreamManager {
      void this.failAndCleanupDictationStream(dictationId, message, true);
    });

-    await stt.connect();
+    try {
+      await stt.connect();
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      this.failDictationStream(dictationId, message, true);
+      try {
+        stt.close();
+      } catch {
+        // no-op
+      }
+      return;
+    }

    const inputRate = parsePcmRateFromFormat(format, 16000) ?? 16000;
    if (!Number.isFinite(inputRate) || inputRate <= 0) {
--- a/packages/server/src/server/session.ts
+++ b/packages/server/src/server/session.ts
@@ -130,6 +130,7 @@ import {
  listLocalSpeechModels,
  type LocalSpeechModelId,
 } from "./speech/providers/local/models.js";
+import type { SpeechReadinessSnapshot } from "./speech/speech-runtime.js";
 import type pino from "pino";

 const execAsync = promisify(exec);
@@ -323,8 +324,8 @@ export type SessionOptions = {
  agentManager: AgentManager;
  agentStorage: AgentStorage;
  createAgentMcpTransport: AgentMcpTransportFactory;
-  stt: SpeechToTextProvider | null;
-  tts: TextToSpeechProvider | null;
+  stt: SpeechToTextProvider | null | (() => SpeechToTextProvider | null);
+  tts: TextToSpeechProvider | null | (() => TextToSpeechProvider | null);
  terminalManager: TerminalManager | null;
  voice?: {
    voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
@@ -339,15 +340,37 @@ export type SessionOptions = {
  };
  dictation?: {
    finalTimeoutMs?: number;
-    stt?: SpeechToTextProvider | null;
+    stt?: SpeechToTextProvider | null | (() => SpeechToTextProvider | null);
    localModels?: {
      modelsDir: string;
      defaultModelIds: LocalSpeechModelId[];
    };
+    getSpeechReadiness?: () => SpeechReadinessSnapshot;
  };
  agentProviderRuntimeSettings?: AgentProviderRuntimeSettingsMap;
 };

+type VoiceFeatureUnavailableContext = {
+  reasonCode: SpeechReadinessSnapshot["voiceFeature"]["reasonCode"];
+  message: string;
+  retryable: boolean;
+  missingModelIds: LocalSpeechModelId[];
+};
+
+class VoiceFeatureUnavailableError extends Error {
+  readonly reasonCode: SpeechReadinessSnapshot["voiceFeature"]["reasonCode"];
+  readonly retryable: boolean;
+  readonly missingModelIds: LocalSpeechModelId[];
+
+  constructor(context: VoiceFeatureUnavailableContext) {
+    super(context.message);
+    this.name = "VoiceFeatureUnavailableError";
+    this.reasonCode = context.reasonCode;
+    this.retryable = context.retryable;
+    this.missingModelIds = [...context.missingModelIds];
+  }
+}
+
 function convertPCMToWavBuffer(
  pcmBuffer: Buffer,
  sampleRate: number,
@@ -533,6 +556,7 @@ export class Session {
  private readonly unregisterVoiceCallerContext?: (agentId: string) => void;
  private readonly ensureVoiceMcpSocketForAgent?: (agentId: string) => Promise<string>;
  private readonly removeVoiceMcpSocketForAgent?: (agentId: string) => Promise<void>;
+  private readonly getSpeechReadiness?: () => SpeechReadinessSnapshot;
  private readonly agentProviderRuntimeSettings: AgentProviderRuntimeSettingsMap | undefined;
  private voiceModeAgentId: string | null = null;
  private voiceModeBaseConfig: VoiceModeBaseConfig | null = null;
@@ -584,6 +608,7 @@ export class Session {
    this.unregisterVoiceCallerContext = voiceBridge?.unregisterVoiceCallerContext;
    this.ensureVoiceMcpSocketForAgent = voiceBridge?.ensureVoiceMcpSocketForAgent;
    this.removeVoiceMcpSocketForAgent = voiceBridge?.removeVoiceMcpSocketForAgent;
+    this.getSpeechReadiness = dictation?.getSpeechReadiness;
    this.agentProviderRuntimeSettings = agentProviderRuntimeSettings;
    this.abortController = new AbortController();
    this.sessionLogger = logger.child({
@@ -1192,6 +1217,22 @@ export class Session {
          break;

        case "dictation_stream_start":
+          {
+            const unavailable = this.resolveVoiceFeatureUnavailableContext("dictation");
+            if (unavailable) {
+              this.emit({
+                type: "dictation_stream_error",
+                payload: {
+                  dictationId: msg.dictationId,
+                  error: unavailable.message,
+                  retryable: unavailable.retryable,
+                  reasonCode: unavailable.reasonCode,
+                  missingModelIds: unavailable.missingModelIds,
+                },
+              });
+              break;
+            }
+          }
          await this.dictationStreamManager.handleStart(msg.dictationId, msg.format);
          break;

@@ -1716,6 +1757,50 @@ export class Session {
    }
  }

+  private toVoiceFeatureUnavailableContext(
+    state: SpeechReadinessSnapshot["voiceFeature"]
+  ): VoiceFeatureUnavailableContext {
+    return {
+      reasonCode: state.reasonCode,
+      message: state.message,
+      retryable: state.retryable,
+      missingModelIds: [...state.missingModelIds],
+    };
+  }
+
+  private resolveVoiceFeatureUnavailableContext(
+    mode: "voice_mode" | "dictation"
+  ): VoiceFeatureUnavailableContext | null {
+    const readiness = this.getSpeechReadiness?.();
+    if (!readiness) {
+      return null;
+    }
+
+    if (mode === "voice_mode") {
+      if (!readiness.realtimeVoice.enabled) {
+        return this.toVoiceFeatureUnavailableContext(readiness.realtimeVoice);
+      }
+      if (!readiness.voiceFeature.available) {
+        return this.toVoiceFeatureUnavailableContext(readiness.voiceFeature);
+      }
+      if (!readiness.realtimeVoice.available) {
+        return this.toVoiceFeatureUnavailableContext(readiness.realtimeVoice);
+      }
+      return null;
+    }
+
+    if (!readiness.dictation.enabled) {
+      return this.toVoiceFeatureUnavailableContext(readiness.dictation);
+    }
+    if (!readiness.voiceFeature.available) {
+      return this.toVoiceFeatureUnavailableContext(readiness.voiceFeature);
+    }
+    if (!readiness.dictation.available) {
+      return this.toVoiceFeatureUnavailableContext(readiness.dictation);
+    }
+    return null;
+  }
+
  /**
   * Handle voice mode toggle
   */
@@ -1726,6 +1811,11 @@ export class Session {
  ): Promise<void> {
    try {
      if (enabled) {
+        const unavailable = this.resolveVoiceFeatureUnavailableContext("voice_mode");
+        if (unavailable) {
+          throw new VoiceFeatureUnavailableError(unavailable);
+        }
+
        const normalizedAgentId = this.parseVoiceTargetAgentId(
          agentId ?? "",
          "set_voice_mode"
@@ -1784,6 +1874,14 @@ export class Session {
    } catch (error) {
      const errorMessage =
        error instanceof Error ? error.message : "Failed to set voice mode";
+      const unavailable =
+        error instanceof VoiceFeatureUnavailableError
+          ? {
+              reasonCode: error.reasonCode,
+              retryable: error.retryable,
+              missingModelIds: error.missingModelIds,
+            }
+          : null;
      this.sessionLogger.error(
        {
          err: error,
@@ -1801,6 +1899,9 @@ export class Session {
            agentId: this.voiceModeAgentId,
            accepted: false,
            error: errorMessage,
+            ...(unavailable ? { reasonCode: unavailable.reasonCode } : {}),
+            ...(unavailable ? { retryable: unavailable.retryable } : {}),
+            ...(unavailable ? { missingModelIds: unavailable.missingModelIds } : {}),
          },
        });
        return;
--- a/packages/server/src/server/speech/providers/local/runtime.ts
+++ b/packages/server/src/server/speech/providers/local/runtime.ts
@@ -186,10 +186,12 @@ export async function initializeLocalSpeechServices(params: {
  providers: RequestedSpeechProviders;
  speechConfig: PaseoSpeechConfig | null;
  logger: Logger;
+  modelEnsureAutoDownload?: boolean;
 }): Promise<InitializedLocalSpeech> {
  const { providers, logger, speechConfig } = params;
  const localConfig = speechConfig?.local ?? null;
  const localModels = resolveConfiguredLocalModels(speechConfig);
+  const modelEnsureAutoDownload = params.modelEnsureAutoDownload ?? (localConfig?.autoDownload ?? true);

  let sttService: SpeechToTextProvider | null = null;
  let ttsService: TextToSpeechProvider | null = null;
@@ -207,14 +209,14 @@ export async function initializeLocalSpeechServices(params: {
        {
          modelsDir: localConfig.modelsDir,
          modelIds: requiredLocalModelIds,
-          autoDownload: localConfig.autoDownload ?? true,
+          autoDownload: modelEnsureAutoDownload,
        },
        "Ensuring local speech models"
      );
      await ensureLocalSpeechModels({
        modelsDir: localConfig.modelsDir,
        modelIds: requiredLocalModelIds,
-        autoDownload: localConfig.autoDownload ?? true,
+        autoDownload: modelEnsureAutoDownload,
        logger,
      });
    } catch (err) {
@@ -223,7 +225,7 @@ export async function initializeLocalSpeechServices(params: {
          err,
          modelsDir: localConfig.modelsDir,
          modelIds: requiredLocalModelIds,
-          autoDownload: localConfig.autoDownload ?? true,
+          autoDownload: modelEnsureAutoDownload,
          hint:
            "Use `paseo speech models` to inspect status and " +
            "`paseo speech download --model <MODEL_ID>` to fetch missing models.",
--- a/packages/server/src/server/speech/speech-runtime.test.ts
+++ b/packages/server/src/server/speech/speech-runtime.test.ts
@@ -0,0 +1,140 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import pino from "pino";
+
+import type { PaseoSpeechConfig } from "../bootstrap.js";
+import type { InitializedLocalSpeech } from "./providers/local/runtime.js";
+import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech-provider.js";
+import { initializeSpeechRuntime } from "./speech-runtime.js";
+
+const { initializeLocalSpeechServicesMock } = vi.hoisted(() => ({
+  initializeLocalSpeechServicesMock: vi.fn<
+    (args: unknown) => Promise<InitializedLocalSpeech>
+  >(),
+}));
+
+vi.mock("./providers/local/runtime.js", () => ({
+  initializeLocalSpeechServices: initializeLocalSpeechServicesMock,
+}));
+
+vi.mock("./providers/openai/runtime.js", () => ({
+  getOpenAiSpeechAvailability: () => ({ configured: false }),
+  initializeOpenAiSpeechServices: (args: {
+    existing: {
+      sttService: SpeechToTextProvider | null;
+      ttsService: TextToSpeechProvider | null;
+      dictationSttService: SpeechToTextProvider | null;
+    };
+  }) => ({
+    sttService: args.existing.sttService,
+    ttsService: args.existing.ttsService,
+    dictationSttService: args.existing.dictationSttService,
+  }),
+  validateOpenAiCredentialRequirements: () => {},
+}));
+
+vi.mock("./providers/local/models.js", () => ({
+  ensureLocalSpeechModels: vi.fn(async () => {}),
+  getLocalSpeechModelDir: vi.fn(() => ""),
+  listLocalSpeechModels: vi.fn(() => []),
+}));
+
+function createStubStt(id: string): SpeechToTextProvider {
+  return {
+    id,
+    createSession: vi.fn(() => {
+      throw new Error("not used in this test");
+    }),
+  };
+}
+
+function createStubTts(id: string): TextToSpeechProvider {
+  return {
+    id,
+    synthesizeSpeech: vi.fn(async () => {
+      throw new Error("not used in this test");
+    }),
+  };
+}
+
+function createSpeechConfig(
+  providers: PaseoSpeechConfig["providers"]
+): PaseoSpeechConfig {
+  return { providers };
+}
+
+describe("initializeSpeechRuntime readiness", () => {
+  beforeEach(() => {
+    initializeLocalSpeechServicesMock.mockReset();
+  });
+
+  it("keeps voice feature available when only dictation is enabled and ready", async () => {
+    const dictationStt = createStubStt("dictation-local");
+
+    initializeLocalSpeechServicesMock.mockResolvedValue({
+      sttService: null,
+      ttsService: null,
+      dictationSttService: dictationStt,
+      localVoiceTtsProvider: null,
+      localModelConfig: null,
+      availability: {
+        configured: false,
+        modelsDir: null,
+        autoDownload: null,
+      },
+      cleanup: () => {},
+    });
+
+    const runtime = await initializeSpeechRuntime({
+      logger: pino({ level: "silent" }),
+      speechConfig: createSpeechConfig({
+        dictationStt: { provider: "local", enabled: true, explicit: true },
+        voiceStt: { provider: "local", enabled: false, explicit: true },
+        voiceTts: { provider: "local", enabled: false, explicit: true },
+      }),
+    });
+
+    const readiness = runtime.getSpeechReadiness();
+    expect(readiness.dictation.available).toBe(true);
+    expect(readiness.realtimeVoice.reasonCode).toBe("disabled");
+    expect(readiness.voiceFeature.available).toBe(true);
+    expect(readiness.voiceFeature.reasonCode).toBe("ready");
+
+    runtime.cleanup();
+  });
+
+  it("keeps voice feature available when only realtime voice is enabled and ready", async () => {
+    const voiceStt = createStubStt("voice-local");
+    const voiceTts = createStubTts("tts-local");
+
+    initializeLocalSpeechServicesMock.mockResolvedValue({
+      sttService: voiceStt,
+      ttsService: voiceTts,
+      dictationSttService: null,
+      localVoiceTtsProvider: voiceTts,
+      localModelConfig: null,
+      availability: {
+        configured: false,
+        modelsDir: null,
+        autoDownload: null,
+      },
+      cleanup: () => {},
+    });
+
+    const runtime = await initializeSpeechRuntime({
+      logger: pino({ level: "silent" }),
+      speechConfig: createSpeechConfig({
+        dictationStt: { provider: "local", enabled: false, explicit: true },
+        voiceStt: { provider: "local", enabled: true, explicit: true },
+        voiceTts: { provider: "local", enabled: true, explicit: true },
+      }),
+    });
+
+    const readiness = runtime.getSpeechReadiness();
+    expect(readiness.realtimeVoice.available).toBe(true);
+    expect(readiness.dictation.reasonCode).toBe("disabled");
+    expect(readiness.voiceFeature.available).toBe(true);
+    expect(readiness.voiceFeature.reasonCode).toBe("ready");
+
+    runtime.cleanup();
+  });
+});
--- a/packages/server/src/server/speech/speech-runtime.ts
+++ b/packages/server/src/server/speech/speech-runtime.ts
@@ -1,11 +1,15 @@
+import { stat } from "node:fs/promises";
+import { join } from "node:path";
 import type { Logger } from "pino";

 import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js";
-import {
-  getLocalSpeechAvailability,
-  initializeLocalSpeechServices,
-} from "./providers/local/runtime.js";
 import type { LocalSpeechModelId } from "./providers/local/config.js";
+import {
+  ensureLocalSpeechModels,
+  getLocalSpeechModelDir,
+  listLocalSpeechModels,
+} from "./providers/local/models.js";
+import { initializeLocalSpeechServices } from "./providers/local/runtime.js";
 import {
  getOpenAiSpeechAvailability,
  initializeOpenAiSpeechServices,
@@ -14,6 +18,39 @@ import {
 import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech-provider.js";
 import type { RequestedSpeechProviders } from "./speech-types.js";

+const SPEECH_RUNTIME_MONITOR_INTERVAL_MS = 3000;
+
+export type SpeechReadinessReasonCode =
+  | "ready"
+  | "disabled"
+  | "model_download_in_progress"
+  | "models_missing"
+  | "model_download_failed"
+  | "stt_unavailable"
+  | "tts_unavailable";
+
+export type SpeechReadinessState = {
+  enabled: boolean;
+  available: boolean;
+  reasonCode: SpeechReadinessReasonCode;
+  message: string;
+  retryable: boolean;
+  missingModelIds: LocalSpeechModelId[];
+};
+
+export type SpeechReadinessSnapshot = {
+  generatedAt: string;
+  requiredLocalModelIds: LocalSpeechModelId[];
+  missingLocalModelIds: LocalSpeechModelId[];
+  download: {
+    inProgress: boolean;
+    error: string | null;
+  };
+  realtimeVoice: SpeechReadinessState;
+  dictation: SpeechReadinessState;
+  voiceFeature: SpeechReadinessState;
+};
+
 function resolveRequestedSpeechProviders(
  speechConfig: PaseoSpeechConfig | null
 ): RequestedSpeechProviders {
@@ -29,10 +66,248 @@ function resolveRequestedSpeechProviders(
  };
 }

-export type InitializedSpeechRuntime = {
+async function hasRequiredLocalModelFile(filePath: string): Promise<boolean> {
+  try {
+    const fileStat = await stat(filePath);
+    if (fileStat.isDirectory()) {
+      return true;
+    }
+    return fileStat.isFile() && fileStat.size > 0;
+  } catch {
+    return false;
+  }
+}
+
+async function findMissingRequiredLocalModels(params: {
+  modelsDir: string | null;
+  requiredModelIds: LocalSpeechModelId[];
+}): Promise<LocalSpeechModelId[]> {
+  const { modelsDir, requiredModelIds } = params;
+  if (!modelsDir || requiredModelIds.length === 0) {
+    return [];
+  }
+
+  const specsById = new Map(listLocalSpeechModels().map((model) => [model.id, model]));
+  const missing = new Set<LocalSpeechModelId>();
+
+  for (const modelId of requiredModelIds) {
+    const spec = specsById.get(modelId);
+    if (!spec) {
+      missing.add(modelId);
+      continue;
+    }
+    const modelDir = getLocalSpeechModelDir(modelsDir, modelId);
+    for (const relPath of spec.requiredFiles) {
+      const filePath = join(modelDir, relPath);
+      if (!(await hasRequiredLocalModelFile(filePath))) {
+        missing.add(modelId);
+        break;
+      }
+    }
+  }
+
+  return Array.from(missing);
+}
+
+function joinModelIds(modelIds: LocalSpeechModelId[]): string {
+  if (modelIds.length === 0) {
+    return "none";
+  }
+  return modelIds.join(", ");
+}
+
+function buildRealtimeVoiceReadiness(params: {
+  providers: RequestedSpeechProviders;
+  sttService: SpeechToTextProvider | null;
+  ttsService: TextToSpeechProvider | null;
+}): SpeechReadinessState {
+  const voiceSttEnabled = params.providers.voiceStt.enabled !== false;
+  const voiceTtsEnabled = params.providers.voiceTts.enabled !== false;
+  const enabled = voiceSttEnabled || voiceTtsEnabled;
+  if (!enabled) {
+    return {
+      enabled: false,
+      available: false,
+      reasonCode: "disabled",
+      message: "Realtime voice is disabled in daemon config.",
+      retryable: false,
+      missingModelIds: [],
+    };
+  }
+  if (voiceSttEnabled && !params.sttService) {
+    return {
+      enabled: true,
+      available: false,
+      reasonCode: "stt_unavailable",
+      message: "Realtime voice is unavailable: speech-to-text service is not ready.",
+      retryable: false,
+      missingModelIds: [],
+    };
+  }
+  if (voiceTtsEnabled && !params.ttsService) {
+    return {
+      enabled: true,
+      available: false,
+      reasonCode: "tts_unavailable",
+      message: "Realtime voice is unavailable: text-to-speech service is not ready.",
+      retryable: false,
+      missingModelIds: [],
+    };
+  }
+  return {
+    enabled: true,
+    available: true,
+    reasonCode: "ready",
+    message: "Realtime voice is ready.",
+    retryable: false,
+    missingModelIds: [],
+  };
+}
+
+function buildDictationReadiness(params: {
+  providers: RequestedSpeechProviders;
+  dictationSttService: SpeechToTextProvider | null;
+}): SpeechReadinessState {
+  const enabled = params.providers.dictationStt.enabled !== false;
+  if (!enabled) {
+    return {
+      enabled: false,
+      available: false,
+      reasonCode: "disabled",
+      message: "Dictation is disabled in daemon config.",
+      retryable: false,
+      missingModelIds: [],
+    };
+  }
+  if (!params.dictationSttService) {
+    return {
+      enabled: true,
+      available: false,
+      reasonCode: "stt_unavailable",
+      message: "Dictation is unavailable: speech-to-text service is not ready.",
+      retryable: false,
+      missingModelIds: [],
+    };
+  }
+  return {
+    enabled: true,
+    available: true,
+    reasonCode: "ready",
+    message: "Dictation is ready.",
+    retryable: false,
+    missingModelIds: [],
+  };
+}
+
+function buildVoiceFeatureReadiness(params: {
+  realtimeVoice: SpeechReadinessState;
+  dictation: SpeechReadinessState;
+  missingLocalModelIds: LocalSpeechModelId[];
+  backgroundDownloadInProgress: boolean;
+  backgroundDownloadError: string | null;
+  localAutoDownloadEnabled: boolean;
+}): SpeechReadinessState {
+  const enabled = params.realtimeVoice.enabled || params.dictation.enabled;
+  if (!enabled) {
+    return {
+      enabled: false,
+      available: false,
+      reasonCode: "disabled",
+      message: "Voice features are disabled in daemon config.",
+      retryable: false,
+      missingModelIds: [],
+    };
+  }
+
+  if (params.missingLocalModelIds.length > 0) {
+    const missingModelIds = [...params.missingLocalModelIds];
+    if (params.backgroundDownloadInProgress) {
+      return {
+        enabled: true,
+        available: false,
+        reasonCode: "model_download_in_progress",
+        message: `Voice features are unavailable while models download in the background (${joinModelIds(missingModelIds)}).`,
+        retryable: true,
+        missingModelIds,
+      };
+    }
+    if (params.backgroundDownloadError) {
+      return {
+        enabled: true,
+        available: false,
+        reasonCode: "model_download_failed",
+        message: `Voice features are unavailable: model download failed (${params.backgroundDownloadError}).`,
+        retryable: false,
+        missingModelIds,
+      };
+    }
+    return {
+      enabled: true,
+      available: false,
+      reasonCode: "models_missing",
+      message: `Voice features are unavailable: missing local models (${joinModelIds(missingModelIds)}).`,
+      retryable: params.localAutoDownloadEnabled,
+      missingModelIds,
+    };
+  }
+
+  return {
+    enabled: true,
+    available: true,
+    reasonCode: "ready",
+    message: "Voice features are ready.",
+    retryable: false,
+    missingModelIds: [],
+  };
+}
+
+function describeRequestedProviders(providers: RequestedSpeechProviders): {
+  dictationStt: { provider: string; enabled: boolean; explicit: boolean };
+  voiceStt: { provider: string; enabled: boolean; explicit: boolean };
+  voiceTts: { provider: string; enabled: boolean; explicit: boolean };
+} {
+  return {
+    dictationStt: {
+      provider: providers.dictationStt.provider,
+      enabled: providers.dictationStt.enabled !== false,
+      explicit: providers.dictationStt.explicit,
+    },
+    voiceStt: {
+      provider: providers.voiceStt.provider,
+      enabled: providers.voiceStt.enabled !== false,
+      explicit: providers.voiceStt.explicit,
+    },
+    voiceTts: {
+      provider: providers.voiceTts.provider,
+      enabled: providers.voiceTts.enabled !== false,
+      explicit: providers.voiceTts.explicit,
+    },
+  };
+}
+
+function resolveEffectiveProviderIds(params: {
  sttService: SpeechToTextProvider | null;
  ttsService: TextToSpeechProvider | null;
  dictationSttService: SpeechToTextProvider | null;
+  localVoiceTtsProvider: TextToSpeechProvider | null;
+}): { dictationStt: string; voiceStt: string; voiceTts: string } {
+  return {
+    dictationStt: params.dictationSttService?.id ?? "unavailable",
+    voiceStt: params.sttService?.id ?? "unavailable",
+    voiceTts:
+      !params.ttsService
+        ? "unavailable"
+        : params.ttsService === params.localVoiceTtsProvider
+          ? "local"
+          : "openai",
+  };
+}
+
+export type InitializedSpeechRuntime = {
+  resolveVoiceStt: () => SpeechToTextProvider | null;
+  resolveVoiceTts: () => TextToSpeechProvider | null;
+  resolveDictationStt: () => SpeechToTextProvider | null;
+  getSpeechReadiness: () => SpeechReadinessSnapshot;
  cleanup: () => void;
  localModelConfig: {
    modelsDir: string;
@@ -45,10 +320,12 @@ export async function initializeSpeechRuntime(params: {
  openaiConfig?: PaseoOpenAIConfig;
  speechConfig?: PaseoSpeechConfig;
 }): Promise<InitializedSpeechRuntime> {
-  const logger = params.logger;
+  const logger = params.logger.child({ module: "speech-runtime" });
  const speechConfig = params.speechConfig ?? null;
  const openaiConfig = params.openaiConfig;
  const providers = resolveRequestedSpeechProviders(speechConfig);
+  const requestedProviders = describeRequestedProviders(providers);
+  const localAutoDownloadEnabled = speechConfig?.local?.autoDownload ?? true;

  validateOpenAiCredentialRequirements({
    providers,
@@ -58,145 +335,264 @@ export async function initializeSpeechRuntime(params: {

  logger.info(
    {
-      requestedProviders: {
-        dictationStt: {
-          provider: providers.dictationStt.provider,
-          enabled: providers.dictationStt.enabled !== false,
-        },
-        voiceStt: {
-          provider: providers.voiceStt.provider,
-          enabled: providers.voiceStt.enabled !== false,
-        },
-        voiceTts: {
-          provider: providers.voiceTts.provider,
-          enabled: providers.voiceTts.enabled !== false,
-        },
-      },
+      requestedProviders,
      availability: {
        openai: getOpenAiSpeechAvailability(openaiConfig),
-        local: getLocalSpeechAvailability(speechConfig),
      },
    },
    "Speech provider reconciliation started"
  );

-  const localSpeech = await initializeLocalSpeechServices({
-    providers,
-    speechConfig,
-    logger,
-  });
+  let sttService: SpeechToTextProvider | null = null;
+  let ttsService: TextToSpeechProvider | null = null;
+  let dictationSttService: SpeechToTextProvider | null = null;
+  let localModelConfig: {
+    modelsDir: string;
+    defaultModelIds: LocalSpeechModelId[];
+  } | null = null;
+  let localCleanup = () => {};
+  let localVoiceTtsProvider: TextToSpeechProvider | null = null;

-  const openAiSpeech = initializeOpenAiSpeechServices({
-    providers,
-    openaiConfig,
-    existing: {
-      sttService: localSpeech.sttService,
-      ttsService: localSpeech.ttsService,
-      dictationSttService: localSpeech.dictationSttService,
-    },
-    logger,
-  });
+  let missingLocalModelIds: LocalSpeechModelId[] = [];
+  let backgroundDownloadInProgress = false;
+  let backgroundDownloadError: string | null = null;
+  let stopped = false;
+  let monitorTimeout: ReturnType<typeof setTimeout> | null = null;
+  let reconcileInFlight: Promise<void> | null = null;

-  const effectiveProviders = {
-    dictationStt: openAiSpeech.dictationSttService?.id ?? "unavailable",
-    voiceStt: openAiSpeech.sttService?.id ?? "unavailable",
-    voiceTts:
-      !openAiSpeech.ttsService
-        ? "unavailable"
-        : openAiSpeech.ttsService === localSpeech.localVoiceTtsProvider
-          ? "local"
-          : "openai",
+  const computeReadinessSnapshot = (): SpeechReadinessSnapshot => {
+    const realtimeVoice = buildRealtimeVoiceReadiness({
+      providers,
+      sttService,
+      ttsService,
+    });
+    const dictation = buildDictationReadiness({
+      providers,
+      dictationSttService,
+    });
+    const voiceFeature = buildVoiceFeatureReadiness({
+      realtimeVoice,
+      dictation,
+      missingLocalModelIds,
+      backgroundDownloadInProgress,
+      backgroundDownloadError,
+      localAutoDownloadEnabled,
+    });
+    return {
+      generatedAt: new Date().toISOString(),
+      requiredLocalModelIds: localModelConfig?.defaultModelIds ?? [],
+      missingLocalModelIds: [...missingLocalModelIds],
+      download: {
+        inProgress: backgroundDownloadInProgress,
+        error: backgroundDownloadError,
+      },
+      realtimeVoice: {
+        ...realtimeVoice,
+      },
+      dictation: {
+        ...dictation,
+      },
+      voiceFeature: {
+        ...voiceFeature,
+      },
+    };
  };
-  const unavailableFeatures = [
-    providers.dictationStt.enabled !== false && !openAiSpeech.dictationSttService
-      ? "dictation.stt"
-      : null,
-    providers.voiceStt.enabled !== false && !openAiSpeech.sttService ? "voice.stt" : null,
-    providers.voiceTts.enabled !== false && !openAiSpeech.ttsService ? "voice.tts" : null,
-  ].filter((feature): feature is string => feature !== null);
-  const explicitlyConfiguredUnavailableFeatures = unavailableFeatures.filter((feature) => {
-    if (feature === "dictation.stt") {
-      return providers.dictationStt.explicit;
-    }
-    if (feature === "voice.stt") {
-      return providers.voiceStt.explicit;
-    }
-    return providers.voiceTts.explicit;
-  });

-  if (explicitlyConfiguredUnavailableFeatures.length > 0) {
-    logger.error(
-      {
-        requestedProviders: {
-          dictationStt: {
-            provider: providers.dictationStt.provider,
-            enabled: providers.dictationStt.enabled !== false,
-          },
-          voiceStt: {
-            provider: providers.voiceStt.provider,
-            enabled: providers.voiceStt.enabled !== false,
-          },
-          voiceTts: {
-            provider: providers.voiceTts.provider,
-            enabled: providers.voiceTts.enabled !== false,
-          },
-        },
-        explicitProviders: {
-          dictationStt: providers.dictationStt.explicit,
-          voiceStt: providers.voiceStt.explicit,
-          voiceTts: providers.voiceTts.explicit,
-        },
-        effectiveProviders,
-        unavailableFeatures: explicitlyConfiguredUnavailableFeatures,
-      },
-      "Speech provider reconciliation failed: configured features are unavailable"
-    );
-    throw new Error(
-      `Configured speech features unavailable: ${explicitlyConfiguredUnavailableFeatures.join(", ")}`
-    );
-  }
+  const refreshMissingLocalModels = async (): Promise<void> => {
+    missingLocalModelIds = await findMissingRequiredLocalModels({
+      modelsDir: localModelConfig?.modelsDir ?? null,
+      requiredModelIds: localModelConfig?.defaultModelIds ?? [],
+    });
+  };

-  if (unavailableFeatures.length > 0) {
-    logger.warn(
-      {
-        requestedProviders: {
-          dictationStt: {
-            provider: providers.dictationStt.provider,
-            enabled: providers.dictationStt.enabled !== false,
-          },
-          voiceStt: {
-            provider: providers.voiceStt.provider,
-            enabled: providers.voiceStt.enabled !== false,
-          },
-          voiceTts: {
-            provider: providers.voiceTts.provider,
-            enabled: providers.voiceTts.enabled !== false,
-          },
-        },
-        explicitProviders: {
-          dictationStt: providers.dictationStt.explicit,
-          voiceStt: providers.voiceStt.explicit,
-          voiceTts: providers.voiceTts.explicit,
-        },
-        effectiveProviders,
-        unavailableFeatures,
+  const reconcileServices = async (modelEnsureAutoDownload: boolean): Promise<void> => {
+    const nextLocalSpeech = await initializeLocalSpeechServices({
+      providers,
+      speechConfig,
+      logger,
+      modelEnsureAutoDownload,
+    });
+    const nextOpenAiSpeech = initializeOpenAiSpeechServices({
+      providers,
+      openaiConfig,
+      existing: {
+        sttService: nextLocalSpeech.sttService,
+        ttsService: nextLocalSpeech.ttsService,
+        dictationSttService: nextLocalSpeech.dictationSttService,
      },
-      "Speech provider reconciliation completed with unavailable default features"
-    );
-  } else {
+      logger,
+    });
+
+    const previousLocalCleanup = localCleanup;
+    sttService = nextOpenAiSpeech.sttService;
+    ttsService = nextOpenAiSpeech.ttsService;
+    dictationSttService = nextOpenAiSpeech.dictationSttService;
+    localModelConfig = nextLocalSpeech.localModelConfig;
+    localVoiceTtsProvider = nextLocalSpeech.localVoiceTtsProvider;
+    localCleanup = nextLocalSpeech.cleanup;
+    previousLocalCleanup();
+
+    await refreshMissingLocalModels();
+
+    const effectiveProviders = resolveEffectiveProviderIds({
+      sttService,
+      ttsService,
+      dictationSttService,
+      localVoiceTtsProvider,
+    });
+    const unavailableFeatures = [
+      providers.dictationStt.enabled !== false && !dictationSttService ? "dictation.stt" : null,
+      providers.voiceStt.enabled !== false && !sttService ? "voice.stt" : null,
+      providers.voiceTts.enabled !== false && !ttsService ? "voice.tts" : null,
+    ].filter((feature): feature is string => feature !== null);
+
+    if (unavailableFeatures.length > 0) {
+      logger.warn(
+        {
+          requestedProviders,
+          effectiveProviders,
+          unavailableFeatures,
+          missingLocalModelIds,
+        },
+        "Speech provider reconciliation completed with unavailable features"
+      );
+    } else {
+      logger.info(
+        {
+          requestedProviders,
+          effectiveProviders,
+        },
+        "Speech provider reconciliation completed"
+      );
+    }
+  };
+
+  const runReconcile = async (params: { modelEnsureAutoDownload: boolean }): Promise<void> => {
+    if (reconcileInFlight) {
+      await reconcileInFlight;
+      return;
+    }
+    reconcileInFlight = reconcileServices(params.modelEnsureAutoDownload).finally(() => {
+      reconcileInFlight = null;
+    });
+    await reconcileInFlight;
+  };
+
+  const scheduleMonitor = (): void => {
+    if (stopped || monitorTimeout) {
+      return;
+    }
+    monitorTimeout = setTimeout(() => {
+      monitorTimeout = null;
+      void runMonitorTick();
+    }, SPEECH_RUNTIME_MONITOR_INTERVAL_MS);
+  };
+
+  const startBackgroundDownload = (): void => {
+    if (stopped || backgroundDownloadInProgress) {
+      return;
+    }
+    const modelsDir = localModelConfig?.modelsDir ?? null;
+    const modelIds = [...missingLocalModelIds];
+    if (!modelsDir || modelIds.length === 0 || !localAutoDownloadEnabled) {
+      return;
+    }
+
+    backgroundDownloadInProgress = true;
+    backgroundDownloadError = null;
+
    logger.info(
      {
-        effectiveProviders,
+        modelsDir,
+        modelIds,
      },
-      "Speech provider reconciliation completed"
+      "Starting background download for missing local speech models"
    );
+
+    void (async () => {
+      try {
+        await ensureLocalSpeechModels({
+          modelsDir,
+          modelIds,
+          autoDownload: true,
+          logger,
+        });
+        await runReconcile({ modelEnsureAutoDownload: false });
+        backgroundDownloadError = null;
+      } catch (error) {
+        backgroundDownloadError = error instanceof Error ? error.message : String(error);
+        logger.error(
+          {
+            err: error,
+            modelIds,
+          },
+          "Background local speech model download failed"
+        );
+      } finally {
+        backgroundDownloadInProgress = false;
+        await refreshMissingLocalModels().catch((error) => {
+          logger.warn({ err: error }, "Failed to refresh local speech model status after download");
+        });
+        scheduleMonitor();
+      }
+    })();
+  };
+
+  const runMonitorTick = async (): Promise<void> => {
+    if (stopped) {
+      return;
+    }
+    try {
+      await refreshMissingLocalModels();
+      const snapshot = computeReadinessSnapshot();
+      if (
+        snapshot.voiceFeature.enabled &&
+        !snapshot.voiceFeature.available &&
+        missingLocalModelIds.length === 0 &&
+        !backgroundDownloadInProgress
+      ) {
+        await runReconcile({ modelEnsureAutoDownload: false });
+      }
+
+      if (
+        missingLocalModelIds.length > 0 &&
+        localAutoDownloadEnabled &&
+        !backgroundDownloadInProgress &&
+        !backgroundDownloadError
+      ) {
+        startBackgroundDownload();
+      }
+    } catch (error) {
+      logger.warn({ err: error }, "Speech runtime monitor tick failed");
+    } finally {
+      scheduleMonitor();
+    }
+  };
+
+  await runReconcile({ modelEnsureAutoDownload: false });
+  const snapshot = computeReadinessSnapshot();
+  if (snapshot.voiceFeature.enabled && !snapshot.voiceFeature.available) {
+    if (missingLocalModelIds.length > 0 && localAutoDownloadEnabled) {
+      startBackgroundDownload();
+    }
+    scheduleMonitor();
  }

+  const cleanup = (): void => {
+    stopped = true;
+    if (monitorTimeout) {
+      clearTimeout(monitorTimeout);
+      monitorTimeout = null;
+    }
+    localCleanup();
+  };
+
  return {
-    sttService: openAiSpeech.sttService,
-    ttsService: openAiSpeech.ttsService,
-    dictationSttService: openAiSpeech.dictationSttService,
-    cleanup: localSpeech.cleanup,
-    localModelConfig: localSpeech.localModelConfig,
+    resolveVoiceStt: () => sttService,
+    resolveVoiceTts: () => ttsService,
+    resolveDictationStt: () => dictationSttService,
+    getSpeechReadiness: () => computeReadinessSnapshot(),
+    cleanup,
+    localModelConfig,
  };
 }
--- a/packages/server/src/server/websocket-server.ts
+++ b/packages/server/src/server/websocket-server.ts
@@ -26,6 +26,7 @@ import type { AgentProviderRuntimeSettingsMap } from "./agent/provider-launch-co
 import { PushTokenStore } from "./push/token-store.js";
 import { PushService } from "./push/push-service.js";
 import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js";
+import type { SpeechReadinessSnapshot } from "./speech/speech-runtime.js";
 import type { LocalSpeechModelId } from "./speech/providers/local/models.js";
 import type {
  VoiceCallerContext,
@@ -93,16 +94,23 @@ export class VoiceAssistantWebSocketServer {
  private readonly pushTokenStore: PushTokenStore;
  private readonly pushService: PushService;
  private readonly createAgentMcpTransport: AgentMcpTransportFactory;
-  private readonly stt: SpeechToTextProvider | null;
-  private readonly tts: TextToSpeechProvider | null;
+  private readonly stt:
+    | SpeechToTextProvider
+    | null
+    | (() => SpeechToTextProvider | null);
+  private readonly tts:
+    | TextToSpeechProvider
+    | null
+    | (() => TextToSpeechProvider | null);
  private readonly terminalManager: TerminalManager | null;
  private readonly dictation: {
    finalTimeoutMs?: number;
-    stt?: SpeechToTextProvider | null;
+    stt?: SpeechToTextProvider | null | (() => SpeechToTextProvider | null);
    localModels?: {
      modelsDir: string;
      defaultModelIds: LocalSpeechModelId[];
    };
+    getSpeechReadiness?: () => SpeechReadinessSnapshot;
  } | null;
  private readonly voice: {
    voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
@@ -126,7 +134,10 @@ export class VoiceAssistantWebSocketServer {
    paseoHome: string,
    createAgentMcpTransport: AgentMcpTransportFactory,
    wsConfig: WebSocketServerConfig,
-    speech?: { stt: SpeechToTextProvider | null; tts: TextToSpeechProvider | null },
+    speech?: {
+      stt: SpeechToTextProvider | null | (() => SpeechToTextProvider | null);
+      tts: TextToSpeechProvider | null | (() => TextToSpeechProvider | null);
+    },
    terminalManager?: TerminalManager | null,
    voice?: {
      voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
@@ -135,11 +146,12 @@ export class VoiceAssistantWebSocketServer {
    },
    dictation?: {
      finalTimeoutMs?: number;
-      stt?: SpeechToTextProvider | null;
+      stt?: SpeechToTextProvider | null | (() => SpeechToTextProvider | null);
      localModels?: {
        modelsDir: string;
        defaultModelIds: LocalSpeechModelId[];
      };
+      getSpeechReadiness?: () => SpeechReadinessSnapshot;
    },
    agentProviderRuntimeSettings?: AgentProviderRuntimeSettingsMap
  ) {
--- a/packages/server/src/shared/messages.ts
+++ b/packages/server/src/shared/messages.ts
@@ -692,6 +692,9 @@ export const SetVoiceModeResponseMessageSchema = z.object({
    agentId: z.string().nullable(),
    accepted: z.boolean(),
    error: z.string().nullable(),
+    reasonCode: z.string().optional(),
+    retryable: z.boolean().optional(),
+    missingModelIds: z.array(z.string()).optional(),
  }),
 });

@@ -1160,6 +1163,8 @@ export const DictationStreamErrorMessageSchema = z.object({
    dictationId: z.string(),
    error: z.string(),
    retryable: z.boolean(),
+    reasonCode: z.string().optional(),
+    missingModelIds: z.array(z.string()).optional(),
    debugRecordingPath: z.string().optional(),
  }),
 });