Refactor speech provider selection model and harden CI startup

2026-07-29 12:01:31 +00:00 · 2026-02-06 22:48:52 +07:00
parent cdb7ef12ae
commit bc4e2daf67
16 changed files with 368 additions and 175 deletions
--- a/packages/server/src/server/bootstrap.smoke.test.ts
+++ b/packages/server/src/server/bootstrap.smoke.test.ts
@@ -13,9 +13,16 @@ describe("paseo daemon bootstrap", () => {
    const daemonHandle = await createTestPaseoDaemon({
      openai: { apiKey: "test-openai-api-key" },
      speech: {
-        dictationSttProvider: "openai",
-        voiceSttProvider: "openai",
-        voiceTtsProvider: "openai",
+        providers: {
+          dictationStt: { provider: "openai", explicit: true },
+          voiceStt: { provider: "openai", explicit: true },
+          voiceTts: { provider: "openai", explicit: true },
+        },
+        localModels: {
+          dictationStt: "parakeet-tdt-0.6b-v3-int8",
+          voiceStt: "parakeet-tdt-0.6b-v3-int8",
+          voiceTts: "pocket-tts-onnx-int8",
+        },
      },
    });
    try {
@@ -56,9 +63,16 @@ describe("paseo daemon bootstrap", () => {
      appBaseUrl: "https://app.paseo.sh",
      openai: undefined,
      speech: {
-        dictationSttProvider: "openai",
-        voiceSttProvider: "openai",
-        voiceTtsProvider: "openai",
+        providers: {
+          dictationStt: { provider: "openai", explicit: true },
+          voiceStt: { provider: "openai", explicit: true },
+          voiceTts: { provider: "openai", explicit: true },
+        },
+        localModels: {
+          dictationStt: "parakeet-tdt-0.6b-v3-int8",
+          voiceStt: "parakeet-tdt-0.6b-v3-int8",
+          voiceTts: "pocket-tts-onnx-int8",
+        },
      },
    };

--- a/packages/server/src/server/bootstrap.ts
+++ b/packages/server/src/server/bootstrap.ts
@@ -43,6 +43,7 @@ import { VoiceAssistantWebSocketServer } from "./websocket-server.js";
 import { DownloadTokenStore } from "./file-download/token-store.js";
 import type { OpenAiSpeechProviderConfig } from "./speech/providers/openai/config.js";
 import type { LocalSpeechProviderConfig } from "./speech/providers/local/config.js";
+import type { RequestedSpeechProviders } from "./speech/speech-types.js";
 import { initializeSpeechRuntime } from "./speech/speech-runtime.js";
 import { AgentManager } from "./agent/agent-manager.js";
 import { AgentStorage } from "./agent/agent-storage.js";
@@ -105,15 +106,15 @@ export type PaseoOpenAIConfig = OpenAiSpeechProviderConfig;
 export type PaseoLocalSpeechConfig = LocalSpeechProviderConfig;

 export type PaseoSpeechConfig = {
-  dictationSttProvider?: "openai" | "local";
-  voiceSttProvider?: "openai" | "local";
-  voiceTtsProvider?: "openai" | "local";
+  providers: RequestedSpeechProviders;
  local?: PaseoLocalSpeechConfig;
-  dictationLocalSttModel?: string;
-  voiceLocalSttModel?: string;
-  voiceLocalTtsModel?: string;
-  voiceLocalTtsSpeakerId?: number;
-  voiceLocalTtsSpeed?: number;
+  localModels: {
+    dictationStt: string;
+    voiceStt: string;
+    voiceTts: string;
+    voiceTtsSpeakerId?: number;
+    voiceTtsSpeed?: number;
+  };
 };

 export type PaseoDaemonConfig = {
@@ -347,25 +348,34 @@ export async function createPaseoDaemon(
      voiceEnabledProviders.find((provider) => voiceLlmAvailability[provider]) ?? null;
  }

-  if (!resolvedVoiceLlmProvider) {
-    logger.error(
-      { requestedVoiceLlmProvider, voiceLlmAvailability },
-      "No voice LLM provider available"
-    );
-    throw new Error("No voice LLM provider available");
-  }
+  let resolvedVoiceLlmModeId: string | null = null;
+  let resolvedVoiceLlmModel: string | null = null;

-  const resolvedVoiceProviderDefinition = AGENT_PROVIDER_DEFINITIONS.find(
-    (definition) => definition.id === resolvedVoiceLlmProvider
-  );
-  if (!resolvedVoiceProviderDefinition?.voice?.enabled) {
-    throw new Error(
-      `Provider '${resolvedVoiceLlmProvider}' is missing voice metadata in agent registry`
+  if (!resolvedVoiceLlmProvider) {
+    if (voiceLlmProviderExplicit) {
+      logger.error(
+        { requestedVoiceLlmProvider, voiceLlmAvailability },
+        "No voice LLM provider available"
+      );
+      throw new Error("No voice LLM provider available");
+    }
+    logger.warn(
+      { requestedVoiceLlmProvider, voiceLlmAvailability },
+      "No default voice LLM provider available; voice mode will be disabled until a provider is configured"
    );
+  } else {
+    const resolvedVoiceProviderDefinition = AGENT_PROVIDER_DEFINITIONS.find(
+      (definition) => definition.id === resolvedVoiceLlmProvider
+    );
+    if (!resolvedVoiceProviderDefinition?.voice?.enabled) {
+      throw new Error(
+        `Provider '${resolvedVoiceLlmProvider}' is missing voice metadata in agent registry`
+      );
+    }
+    resolvedVoiceLlmModeId = resolvedVoiceProviderDefinition.voice.defaultModeId;
+    resolvedVoiceLlmModel =
+      config.voiceLlmModel ?? resolvedVoiceProviderDefinition.voice.defaultModel ?? null;
  }
-  const resolvedVoiceLlmModeId = resolvedVoiceProviderDefinition.voice.defaultModeId;
-  const resolvedVoiceLlmModel =
-    config.voiceLlmModel ?? resolvedVoiceProviderDefinition.voice.defaultModel ?? null;

  logger.info(
    {
--- a/packages/server/src/server/daemon-client.e2e.test.ts
+++ b/packages/server/src/server/daemon-client.e2e.test.ts
@@ -93,33 +93,47 @@ describe("daemon client E2E", () => {
  let ctx: DaemonTestContext;

  beforeAll(async () => {
-    const speechConfig = openaiApiKey
-      ? {
-          dictationSttProvider: "openai" as const,
-          voiceSttProvider: "openai" as const,
-          voiceTtsProvider: "openai" as const,
-        }
-      : {
-          dictationSttProvider: "local" as const,
-          voiceSttProvider: "local" as const,
-          voiceTtsProvider: "local" as const,
-          local: {
-            modelsDir: localModelsDir,
-          },
-          dictationLocalSttModel:
-            process.env.PASEO_DICTATION_LOCAL_STT_MODEL ??
-            "zipformer-bilingual-zh-en-2023-02-20",
-          voiceLocalSttModel:
-            process.env.PASEO_VOICE_LOCAL_STT_MODEL ??
-            "zipformer-bilingual-zh-en-2023-02-20",
-          voiceLocalTtsModel:
-            process.env.PASEO_VOICE_LOCAL_TTS_MODEL ?? "kitten-nano-en-v0_1-fp16",
-        };
+    const speechConfig =
+      openaiApiKey
+        ? {
+            providers: {
+              dictationStt: { provider: "openai" as const, explicit: true },
+              voiceStt: { provider: "openai" as const, explicit: true },
+              voiceTts: { provider: "openai" as const, explicit: true },
+            },
+            localModels: {
+              dictationStt: "parakeet-tdt-0.6b-v3-int8",
+              voiceStt: "parakeet-tdt-0.6b-v3-int8",
+              voiceTts: "pocket-tts-onnx-int8",
+            },
+          }
+        : hasLocalSpeech
+          ? {
+              providers: {
+                dictationStt: { provider: "local" as const, explicit: true },
+                voiceStt: { provider: "local" as const, explicit: true },
+                voiceTts: { provider: "local" as const, explicit: true },
+              },
+              local: {
+                modelsDir: localModelsDir,
+              },
+              localModels: {
+                dictationStt:
+                  process.env.PASEO_DICTATION_LOCAL_STT_MODEL ??
+                  "zipformer-bilingual-zh-en-2023-02-20",
+                voiceStt:
+                  process.env.PASEO_VOICE_LOCAL_STT_MODEL ??
+                  "zipformer-bilingual-zh-en-2023-02-20",
+                voiceTts:
+                  process.env.PASEO_VOICE_LOCAL_TTS_MODEL ?? "kitten-nano-en-v0_1-fp16",
+              },
+            }
+          : undefined;

    ctx = await createDaemonTestContext({
      dictationFinalTimeoutMs: 5000,
      ...(openaiApiKey ? { openai: { apiKey: openaiApiKey } } : {}),
-      speech: speechConfig,
+      ...(speechConfig ? { speech: speechConfig } : {}),
    });
  }, 60000);

--- a/packages/server/src/server/daemon-e2e/connection-offer.e2e.test.ts
+++ b/packages/server/src/server/daemon-e2e/connection-offer.e2e.test.ts
@@ -186,6 +186,7 @@ describe("ConnectionOfferV2 (daemon E2E)", () => {
        PASEO_HOME: tempHome,
        PASEO_LISTEN: `0.0.0.0:${port}`,
        OPENAI_API_KEY: "",
+        PASEO_LOCAL_AUTO_DOWNLOAD: "0",
        PASEO_LOG_FORMAT: "json",
      };

--- a/packages/server/src/server/persisted-config.ts
+++ b/packages/server/src/server/persisted-config.ts
@@ -32,15 +32,11 @@ const ProvidersSchema = z
  })
  .strict();

-const SpeechProviderIdSchema = z.preprocess(
-  (value) => {
-    if (typeof value !== "string") {
-      return value;
-    }
-    return value.trim().toLowerCase();
-  },
-  z.enum(["openai", "local"])
-);
+const SpeechProviderIdSchema = z
+  .string()
+  .trim()
+  .toLowerCase()
+  .pipe(z.enum(["openai", "local"]));

 const FeatureDictationSchema = z
  .object({
--- a/packages/server/src/server/speech/providers/local/config.ts
+++ b/packages/server/src/server/speech/providers/local/config.ts
@@ -68,7 +68,7 @@ const LocalSpeechResolutionSchema = z.object({
 });

 function persistedLocalFeatureModel(
-  provider: RequestedSpeechProviders[keyof RequestedSpeechProviders],
+  provider: RequestedSpeechProviders[keyof RequestedSpeechProviders]["provider"],
  model: string | undefined
 ): string | undefined {
  if (provider !== "local") {
@@ -83,9 +83,9 @@ function shouldIncludeLocalProviderConfig(params: {
  persisted: PersistedConfig;
 }): boolean {
  const localRequestedByFeature =
-    params.providers.dictationSttProvider === "local" ||
-    params.providers.voiceSttProvider === "local" ||
-    params.providers.voiceTtsProvider === "local";
+    params.providers.dictationStt.provider === "local" ||
+    params.providers.voiceStt.provider === "local" ||
+    params.providers.voiceTts.provider === "local";

  return (
    localRequestedByFeature ||
@@ -114,21 +114,21 @@ export function resolveLocalSpeechConfig(params: {
    dictationLocalSttModel:
      params.env.PASEO_DICTATION_LOCAL_STT_MODEL ??
      persistedLocalFeatureModel(
-        params.providers.dictationSttProvider,
+        params.providers.dictationStt.provider,
        params.persisted.features?.dictation?.stt?.model
      ) ??
      DEFAULT_LOCAL_STT_MODEL,
    voiceLocalSttModel:
      params.env.PASEO_VOICE_LOCAL_STT_MODEL ??
      persistedLocalFeatureModel(
-        params.providers.voiceSttProvider,
+        params.providers.voiceStt.provider,
        params.persisted.features?.voiceMode?.stt?.model
      ) ??
      DEFAULT_LOCAL_STT_MODEL,
    voiceLocalTtsModel:
      params.env.PASEO_VOICE_LOCAL_TTS_MODEL ??
      persistedLocalFeatureModel(
-        params.providers.voiceTtsProvider,
+        params.providers.voiceTts.provider,
        params.persisted.features?.voiceMode?.tts?.model
      ) ??
      DEFAULT_LOCAL_TTS_MODEL,
--- a/packages/server/src/server/speech/providers/local/runtime.ts
+++ b/packages/server/src/server/speech/providers/local/runtime.ts
@@ -63,13 +63,13 @@ function resolveConfiguredLocalModels(
 ): ResolvedLocalModels {
  return {
    dictationLocalSttModel: LocalSttModelIdSchema.parse(
-      speechConfig?.dictationLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL
+      speechConfig?.localModels.dictationStt ?? DEFAULT_LOCAL_STT_MODEL
    ),
    voiceLocalSttModel: LocalSttModelIdSchema.parse(
-      speechConfig?.voiceLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL
+      speechConfig?.localModels.voiceStt ?? DEFAULT_LOCAL_STT_MODEL
    ),
    voiceLocalTtsModel: LocalTtsModelIdSchema.parse(
-      speechConfig?.voiceLocalTtsModel ?? DEFAULT_LOCAL_TTS_MODEL
+      speechConfig?.localModels.voiceTts ?? DEFAULT_LOCAL_TTS_MODEL
    ),
  };
 }
@@ -90,13 +90,13 @@ function computeRequiredLocalModelIds(params: {
  models: ResolvedLocalModels;
 }): LocalSpeechModelId[] {
  const ids = new Set<LocalSpeechModelId>();
-  if (params.providers.dictationSttProvider === "local") {
+  if (params.providers.dictationStt.provider === "local") {
    ids.add(params.models.dictationLocalSttModel);
  }
-  if (params.providers.voiceSttProvider === "local") {
+  if (params.providers.voiceStt.provider === "local") {
    ids.add(params.models.voiceLocalSttModel);
  }
-  if (params.providers.voiceTtsProvider === "local") {
+  if (params.providers.voiceTts.provider === "local") {
    ids.add(params.models.voiceLocalTtsModel);
  }
  return Array.from(ids);
@@ -260,7 +260,7 @@ export async function initializeLocalSpeechServices(params: {
    }
  };

-  if (providers.voiceSttProvider === "local") {
+  if (providers.voiceStt.provider === "local") {
    if (!localConfig) {
      logger.warn(
        { configured: false },
@@ -276,7 +276,7 @@ export async function initializeLocalSpeechServices(params: {
    }
  }

-  if (providers.dictationSttProvider === "local") {
+  if (providers.dictationStt.provider === "local") {
    if (!localConfig) {
      logger.warn(
        { configured: false },
@@ -299,7 +299,7 @@ export async function initializeLocalSpeechServices(params: {
    }
  }

-  if (providers.voiceTtsProvider === "local") {
+  if (providers.voiceTts.provider === "local") {
    if (!localConfig) {
      logger.warn(
        { configured: false },
@@ -323,8 +323,8 @@ export async function initializeLocalSpeechServices(params: {
            {
              preset: localModels.voiceLocalTtsModel,
              modelDir,
-              speakerId: speechConfig?.voiceLocalTtsSpeakerId,
-              speed: speechConfig?.voiceLocalTtsSpeed,
+              speakerId: speechConfig?.localModels.voiceTtsSpeakerId,
+              speed: speechConfig?.localModels.voiceTtsSpeed,
            },
            logger
          );
--- a/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts
+++ b/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts
@@ -115,23 +115,27 @@ describe("speech models (download E2E)", () => {
        paseoHomeRoot,
        dictationFinalTimeoutMs: 8000,
        speech: {
-          dictationSttProvider: "local",
-          voiceSttProvider: "local",
-          voiceTtsProvider: "local",
+          providers: {
+            dictationStt: { provider: "local", explicit: true },
+            voiceStt: { provider: "local", explicit: true },
+            voiceTts: { provider: "local", explicit: true },
+          },
          local: {
            modelsDir,
            autoDownload: false,
          },
-          dictationLocalSttModel:
-            set === "parakeet-pocket"
-              ? "parakeet-tdt-0.6b-v3-int8"
-              : "zipformer-bilingual-zh-en-2023-02-20",
-          voiceLocalSttModel:
-            set === "parakeet-pocket"
-              ? "parakeet-tdt-0.6b-v3-int8"
-              : "zipformer-bilingual-zh-en-2023-02-20",
-          voiceLocalTtsModel:
-            set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16",
+          localModels: {
+            dictationStt:
+              set === "parakeet-pocket"
+                ? "parakeet-tdt-0.6b-v3-int8"
+                : "zipformer-bilingual-zh-en-2023-02-20",
+            voiceStt:
+              set === "parakeet-pocket"
+                ? "parakeet-tdt-0.6b-v3-int8"
+                : "zipformer-bilingual-zh-en-2023-02-20",
+            voiceTts:
+              set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16",
+          },
        },
      });

--- a/packages/server/src/server/speech/providers/openai/config.test.ts
+++ b/packages/server/src/server/speech/providers/openai/config.test.ts
@@ -0,0 +1,46 @@
+import { describe, expect, test } from "vitest";
+
+import { PersistedConfigSchema } from "../../../persisted-config.js";
+import { resolveOpenAiSpeechConfig } from "./config.js";
+
+describe("resolveOpenAiSpeechConfig", () => {
+  test("treats empty OPENAI_API_KEY as unset", () => {
+    const persisted = PersistedConfigSchema.parse({});
+    const env = {
+      OPENAI_API_KEY: "",
+    } as NodeJS.ProcessEnv;
+
+    const resolved = resolveOpenAiSpeechConfig({
+      env,
+      persisted,
+      providers: {
+        dictationStt: { provider: "local", explicit: false },
+        voiceStt: { provider: "local", explicit: false },
+        voiceTts: { provider: "local", explicit: false },
+      },
+    });
+
+    expect(resolved).toBeUndefined();
+  });
+
+  test("uses trimmed OPENAI_API_KEY when configured", () => {
+    const persisted = PersistedConfigSchema.parse({});
+    const env = {
+      OPENAI_API_KEY: "  sk-test  ",
+    } as NodeJS.ProcessEnv;
+
+    const resolved = resolveOpenAiSpeechConfig({
+      env,
+      persisted,
+      providers: {
+        dictationStt: { provider: "openai", explicit: true },
+        voiceStt: { provider: "openai", explicit: true },
+        voiceTts: { provider: "openai", explicit: true },
+      },
+    });
+
+    expect(resolved?.apiKey).toBe("sk-test");
+    expect(resolved?.stt?.apiKey).toBe("sk-test");
+    expect(resolved?.tts?.apiKey).toBe("sk-test");
+  });
+});
--- a/packages/server/src/server/speech/providers/openai/config.ts
+++ b/packages/server/src/server/speech/providers/openai/config.ts
@@ -35,7 +35,11 @@ const OptionalFiniteNumberSchema = NumberLikeSchema
  .pipe(z.coerce.number().finite())
  .optional();

-const OptionalTrimmedStringSchema = z.string().trim().min(1).optional();
+const OptionalTrimmedStringSchema = z
+  .string()
+  .trim()
+  .optional()
+  .transform((value) => (value && value.length > 0 ? value : undefined));

 const OpenAiSpeechResolutionSchema = z.object({
  apiKey: OptionalTrimmedStringSchema,
@@ -70,27 +74,27 @@ export function resolveOpenAiSpeechConfig(params: {
      params.persisted.features?.dictation?.stt?.confidenceThreshold,
    sttModel:
      params.env.STT_MODEL ??
-      (params.providers.voiceSttProvider === "openai"
+      (params.providers.voiceStt.provider === "openai"
        ? params.persisted.features?.voiceMode?.stt?.model
        : undefined) ??
-      (params.providers.dictationSttProvider === "openai"
+      (params.providers.dictationStt.provider === "openai"
        ? params.persisted.features?.dictation?.stt?.model
        : undefined),
    ttsVoice:
      params.env.TTS_VOICE ??
-      (params.providers.voiceTtsProvider === "openai"
+      (params.providers.voiceTts.provider === "openai"
        ? params.persisted.features?.voiceMode?.tts?.voice
        : undefined) ??
      "alloy",
    ttsModel:
      params.env.TTS_MODEL ??
-      (params.providers.voiceTtsProvider === "openai"
+      (params.providers.voiceTts.provider === "openai"
        ? params.persisted.features?.voiceMode?.tts?.model
        : undefined) ??
      DEFAULT_OPENAI_TTS_MODEL,
    realtimeTranscriptionModel:
      params.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ??
-      (params.providers.dictationSttProvider === "openai"
+      (params.providers.dictationStt.provider === "openai"
        ? params.persisted.features?.dictation?.stt?.model
        : undefined) ??
      DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
--- a/packages/server/src/server/speech/providers/openai/runtime.ts
+++ b/packages/server/src/server/speech/providers/openai/runtime.ts
@@ -60,13 +60,16 @@ export function validateOpenAiCredentialRequirements(params: {
  const openAiCredentials = resolveOpenAiCredentials(openaiConfig);

  const missingOpenAiCredentialsFor: string[] = [];
-  if (providers.voiceSttProvider === "openai" && !openAiCredentials.openaiSttApiKey) {
+  if (providers.voiceStt.provider === "openai" && !openAiCredentials.openaiSttApiKey) {
    missingOpenAiCredentialsFor.push("voice.stt");
  }
-  if (providers.voiceTtsProvider === "openai" && !openAiCredentials.openaiTtsApiKey) {
+  if (providers.voiceTts.provider === "openai" && !openAiCredentials.openaiTtsApiKey) {
    missingOpenAiCredentialsFor.push("voice.tts");
  }
-  if (providers.dictationSttProvider === "openai" && !openAiCredentials.openaiDictationApiKey) {
+  if (
+    providers.dictationStt.provider === "openai" &&
+    !openAiCredentials.openaiDictationApiKey
+  ) {
    missingOpenAiCredentialsFor.push("dictation.stt");
  }

@@ -74,9 +77,9 @@ export function validateOpenAiCredentialRequirements(params: {
    logger.error(
      {
        requestedProviders: {
-          dictationStt: providers.dictationSttProvider,
-          voiceStt: providers.voiceSttProvider,
-          voiceTts: providers.voiceTtsProvider,
+          dictationStt: providers.dictationStt.provider,
+          voiceStt: providers.voiceStt.provider,
+          voiceTts: providers.voiceTts.provider,
        },
        missingOpenAiCredentialsFor,
      },
@@ -101,9 +104,10 @@ export function initializeOpenAiSpeechServices(params: {
  let ttsService = existing.ttsService;
  let dictationSttService = existing.dictationSttService;

-  const needsOpenAiStt = !sttService && providers.voiceSttProvider === "openai";
-  const needsOpenAiTts = !ttsService && providers.voiceTtsProvider === "openai";
-  const needsOpenAiDictation = !dictationSttService && providers.dictationSttProvider === "openai";
+  const needsOpenAiStt = !sttService && providers.voiceStt.provider === "openai";
+  const needsOpenAiTts = !ttsService && providers.voiceTts.provider === "openai";
+  const needsOpenAiDictation =
+    !dictationSttService && providers.dictationStt.provider === "openai";

  if (
    (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) &&
--- a/packages/server/src/server/speech/speech-config-resolver.test.ts
+++ b/packages/server/src/server/speech/speech-config-resolver.test.ts
@@ -18,16 +18,25 @@ describe("resolveSpeechConfig", () => {
    });

    expect(result.openai).toBeUndefined();
-    expect(result.speech.dictationSttProvider).toBe("local");
-    expect(result.speech.voiceSttProvider).toBe("local");
-    expect(result.speech.voiceTtsProvider).toBe("local");
+    expect(result.speech.providers.dictationStt).toEqual({
+      provider: "local",
+      explicit: false,
+    });
+    expect(result.speech.providers.voiceStt).toEqual({
+      provider: "local",
+      explicit: false,
+    });
+    expect(result.speech.providers.voiceTts).toEqual({
+      provider: "local",
+      explicit: false,
+    });
    expect(result.speech.local).toEqual({
      modelsDir: path.join(paseoHome, "models", "local-speech"),
      autoDownload: true,
    });
-    expect(result.speech.dictationLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8");
-    expect(result.speech.voiceLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8");
-    expect(result.speech.voiceLocalTtsModel).toBe("pocket-tts-onnx-int8");
+    expect(result.speech.localModels.dictationStt).toBe("parakeet-tdt-0.6b-v3-int8");
+    expect(result.speech.localModels.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8");
+    expect(result.speech.localModels.voiceTts).toBe("pocket-tts-onnx-int8");
  });

  test("resolves feature-scoped local model env vars", () => {
@@ -65,11 +74,23 @@ describe("resolveSpeechConfig", () => {
      modelsDir: "/tmp/models",
      autoDownload: false,
    });
-    expect(result.speech.dictationLocalSttModel).toBe("zipformer-bilingual-zh-en-2023-02-20");
-    expect(result.speech.voiceLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8");
-    expect(result.speech.voiceLocalTtsModel).toBe("kitten-nano-en-v0_1-fp16");
-    expect(result.speech.voiceLocalTtsSpeakerId).toBe(5);
-    expect(result.speech.voiceLocalTtsSpeed).toBe(1.35);
+    expect(result.speech.providers.dictationStt).toEqual({
+      provider: "local",
+      explicit: true,
+    });
+    expect(result.speech.providers.voiceStt).toEqual({
+      provider: "openai",
+      explicit: true,
+    });
+    expect(result.speech.providers.voiceTts).toEqual({
+      provider: "local",
+      explicit: true,
+    });
+    expect(result.speech.localModels.dictationStt).toBe("zipformer-bilingual-zh-en-2023-02-20");
+    expect(result.speech.localModels.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8");
+    expect(result.speech.localModels.voiceTts).toBe("kitten-nano-en-v0_1-fp16");
+    expect(result.speech.localModels.voiceTtsSpeakerId).toBe(5);
+    expect(result.speech.localModels.voiceTtsSpeed).toBe(1.35);
    expect(result.openai?.apiKey).toBe("env-key");
    expect(result.openai?.stt?.model).toBe("gpt-4o-transcribe");
  });
@@ -87,8 +108,8 @@ describe("resolveSpeechConfig", () => {
      persisted,
    });

-    expect(result.speech.dictationLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8");
-    expect(result.speech.voiceLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8");
-    expect(result.speech.voiceLocalTtsModel).toBe("pocket-tts-onnx-int8");
+    expect(result.speech.localModels.dictationStt).toBe("parakeet-tdt-0.6b-v3-int8");
+    expect(result.speech.localModels.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8");
+    expect(result.speech.localModels.voiceTts).toBe("pocket-tts-onnx-int8");
  });
 });
--- a/packages/server/src/server/speech/speech-config-resolver.ts
+++ b/packages/server/src/server/speech/speech-config-resolver.ts
@@ -6,6 +6,7 @@ import { resolveLocalSpeechConfig } from "./providers/local/config.js";
 import { resolveOpenAiSpeechConfig } from "./providers/openai/config.js";
 import {
  SpeechProviderIdSchema,
+  type RequestedSpeechProvider,
  type RequestedSpeechProviders,
 } from "./speech-types.js";

@@ -17,29 +18,53 @@ const OptionalSpeechProviderSchema = z
  .optional();

 const RequestedSpeechProvidersSchema = z.object({
-  dictationSttProvider: OptionalSpeechProviderSchema.default("local"),
-  voiceSttProvider: OptionalSpeechProviderSchema.default("local"),
-  voiceTtsProvider: OptionalSpeechProviderSchema.default("local"),
+  dictationStt: OptionalSpeechProviderSchema.default("local"),
+  voiceStt: OptionalSpeechProviderSchema.default("local"),
+  voiceTts: OptionalSpeechProviderSchema.default("local"),
 });

 function resolveRequestedSpeechProviders(params: {
  env: NodeJS.ProcessEnv;
  persisted: PersistedConfig;
 }): RequestedSpeechProviders {
-  return RequestedSpeechProvidersSchema.parse({
-    dictationSttProvider:
-      params.env.PASEO_DICTATION_STT_PROVIDER ??
-      params.persisted.features?.dictation?.stt?.provider ??
-      "local",
-    voiceSttProvider:
-      params.env.PASEO_VOICE_STT_PROVIDER ??
-      params.persisted.features?.voiceMode?.stt?.provider ??
-      "local",
-    voiceTtsProvider:
-      params.env.PASEO_VOICE_TTS_PROVIDER ??
-      params.persisted.features?.voiceMode?.tts?.provider ??
-      "local",
+  const resolveFeatureProvider = (
+    configuredValue: string | undefined,
+    parsedValue: z.infer<typeof SpeechProviderIdSchema>
+  ): RequestedSpeechProvider => ({
+    provider: parsedValue,
+    explicit: configuredValue !== undefined,
  });
+
+  const dictationSttProviderFromConfig =
+    params.env.PASEO_DICTATION_STT_PROVIDER ??
+    params.persisted.features?.dictation?.stt?.provider;
+  const voiceSttProviderFromConfig =
+    params.env.PASEO_VOICE_STT_PROVIDER ??
+    params.persisted.features?.voiceMode?.stt?.provider;
+  const voiceTtsProviderFromConfig =
+    params.env.PASEO_VOICE_TTS_PROVIDER ??
+    params.persisted.features?.voiceMode?.tts?.provider;
+
+  const parsed = RequestedSpeechProvidersSchema.parse({
+    dictationStt: dictationSttProviderFromConfig ?? "local",
+    voiceStt: voiceSttProviderFromConfig ?? "local",
+    voiceTts: voiceTtsProviderFromConfig ?? "local",
+  });
+
+  return {
+    dictationStt: resolveFeatureProvider(
+      dictationSttProviderFromConfig,
+      parsed.dictationStt
+    ),
+    voiceStt: resolveFeatureProvider(
+      voiceSttProviderFromConfig,
+      parsed.voiceStt
+    ),
+    voiceTts: resolveFeatureProvider(
+      voiceTtsProviderFromConfig,
+      parsed.voiceTts
+    ),
+  };
 }

 export function resolveSpeechConfig(params: {
@@ -71,21 +96,21 @@ export function resolveSpeechConfig(params: {
  return {
    openai,
    speech: {
-      dictationSttProvider: providers.dictationSttProvider,
-      voiceSttProvider: providers.voiceSttProvider,
-      voiceTtsProvider: providers.voiceTtsProvider,
+      providers,
      ...(local.local
        ? { local: local.local }
        : {}),
-      dictationLocalSttModel: local.dictationLocalSttModel,
-      voiceLocalSttModel: local.voiceLocalSttModel,
-      voiceLocalTtsModel: local.voiceLocalTtsModel,
-      ...(local.voiceLocalTtsSpeakerId !== undefined
-        ? { voiceLocalTtsSpeakerId: local.voiceLocalTtsSpeakerId }
-        : {}),
-      ...(local.voiceLocalTtsSpeed !== undefined
-        ? { voiceLocalTtsSpeed: local.voiceLocalTtsSpeed }
-        : {}),
+      localModels: {
+        dictationStt: local.dictationLocalSttModel,
+        voiceStt: local.voiceLocalSttModel,
+        voiceTts: local.voiceLocalTtsModel,
+        ...(local.voiceLocalTtsSpeakerId !== undefined
+          ? { voiceTtsSpeakerId: local.voiceLocalTtsSpeakerId }
+          : {}),
+        ...(local.voiceLocalTtsSpeed !== undefined
+          ? { voiceTtsSpeed: local.voiceLocalTtsSpeed }
+          : {}),
+      },
    },
  };
 }
--- a/packages/server/src/server/speech/speech-runtime.ts
+++ b/packages/server/src/server/speech/speech-runtime.ts
@@ -17,10 +17,15 @@ import type { RequestedSpeechProviders } from "./speech-types.js";
 function resolveRequestedSpeechProviders(
  speechConfig: PaseoSpeechConfig | null
 ): RequestedSpeechProviders {
+  const fromConfig = speechConfig?.providers;
+  if (fromConfig) {
+    return fromConfig;
+  }
+
  return {
-    dictationSttProvider: speechConfig?.dictationSttProvider ?? "local",
-    voiceSttProvider: speechConfig?.voiceSttProvider ?? "local",
-    voiceTtsProvider: speechConfig?.voiceTtsProvider ?? "local",
+    dictationStt: { provider: "local", explicit: false },
+    voiceStt: { provider: "local", explicit: false },
+    voiceTts: { provider: "local", explicit: false },
  };
 }

@@ -54,9 +59,9 @@ export async function initializeSpeechRuntime(params: {
  logger.info(
    {
      requestedProviders: {
-        dictationStt: providers.dictationSttProvider,
-        voiceStt: providers.voiceSttProvider,
-        voiceTts: providers.voiceTtsProvider,
+        dictationStt: providers.dictationStt.provider,
+        voiceStt: providers.voiceStt.provider,
+        voiceTts: providers.voiceTts.provider,
      },
      availability: {
        openai: getOpenAiSpeechAvailability(openaiConfig),
@@ -98,30 +103,66 @@ export async function initializeSpeechRuntime(params: {
    !openAiSpeech.sttService ? "voice.stt" : null,
    !openAiSpeech.ttsService ? "voice.tts" : null,
  ].filter((feature): feature is string => feature !== null);
+  const explicitlyConfiguredUnavailableFeatures = unavailableFeatures.filter((feature) => {
+    if (feature === "dictation.stt") {
+      return providers.dictationStt.explicit;
+    }
+    if (feature === "voice.stt") {
+      return providers.voiceStt.explicit;
+    }
+    return providers.voiceTts.explicit;
+  });

-  if (unavailableFeatures.length > 0) {
+  if (explicitlyConfiguredUnavailableFeatures.length > 0) {
    logger.error(
      {
        requestedProviders: {
-          dictationStt: providers.dictationSttProvider,
-          voiceStt: providers.voiceSttProvider,
-          voiceTts: providers.voiceTtsProvider,
+          dictationStt: providers.dictationStt.provider,
+          voiceStt: providers.voiceStt.provider,
+          voiceTts: providers.voiceTts.provider,
+        },
+        explicitProviders: {
+          dictationStt: providers.dictationStt.explicit,
+          voiceStt: providers.voiceStt.explicit,
+          voiceTts: providers.voiceTts.explicit,
+        },
+        effectiveProviders,
+        unavailableFeatures: explicitlyConfiguredUnavailableFeatures,
+      },
+      "Speech provider reconciliation failed: configured features are unavailable"
+    );
+    throw new Error(
+      `Configured speech features unavailable: ${explicitlyConfiguredUnavailableFeatures.join(", ")}`
+    );
+  }
+
+  if (unavailableFeatures.length > 0) {
+    logger.warn(
+      {
+        requestedProviders: {
+          dictationStt: providers.dictationStt.provider,
+          voiceStt: providers.voiceStt.provider,
+          voiceTts: providers.voiceTts.provider,
+        },
+        explicitProviders: {
+          dictationStt: providers.dictationStt.explicit,
+          voiceStt: providers.voiceStt.explicit,
+          voiceTts: providers.voiceTts.explicit,
        },
        effectiveProviders,
        unavailableFeatures,
      },
-      "Speech provider reconciliation failed: configured features are unavailable"
+      "Speech provider reconciliation completed with unavailable default features"
+    );
+  } else {
+    logger.info(
+      {
+        effectiveProviders,
+      },
+      "Speech provider reconciliation completed"
    );
-    throw new Error(`Configured speech features unavailable: ${unavailableFeatures.join(", ")}`);
  }

-  logger.info(
-    {
-      effectiveProviders,
-    },
-    "Speech provider reconciliation completed"
-  );
-
  return {
    sttService: openAiSpeech.sttService,
    ttsService: openAiSpeech.ttsService,
--- a/packages/server/src/server/speech/speech-types.ts
+++ b/packages/server/src/server/speech/speech-types.ts
@@ -3,8 +3,14 @@ import { z } from "zod";
 export const SpeechProviderIdSchema = z.enum(["openai", "local"]);
 export type SpeechProviderId = z.infer<typeof SpeechProviderIdSchema>;

+export const RequestedSpeechProviderSchema = z.object({
+  provider: SpeechProviderIdSchema,
+  explicit: z.boolean(),
+});
+export type RequestedSpeechProvider = z.infer<typeof RequestedSpeechProviderSchema>;
+
 export type RequestedSpeechProviders = {
-  dictationSttProvider: SpeechProviderId;
-  voiceSttProvider: SpeechProviderId;
-  voiceTtsProvider: SpeechProviderId;
+  dictationStt: RequestedSpeechProvider;
+  voiceStt: RequestedSpeechProvider;
+  voiceTts: RequestedSpeechProvider;
 };
--- a/packages/server/src/server/voice-local-agent.e2e.test.ts
+++ b/packages/server/src/server/voice-local-agent.e2e.test.ts
@@ -48,9 +48,16 @@ function waitForSignal<T>(
        agentClients: {},
        openai: { apiKey: openaiApiKey! },
        speech: {
-          dictationSttProvider: "openai",
-          voiceSttProvider: "openai",
-          voiceTtsProvider: "openai",
+          providers: {
+            dictationStt: { provider: "openai", explicit: true },
+            voiceStt: { provider: "openai", explicit: true },
+            voiceTts: { provider: "openai", explicit: true },
+          },
+          localModels: {
+            dictationStt: "parakeet-tdt-0.6b-v3-int8",
+            voiceStt: "parakeet-tdt-0.6b-v3-int8",
+            voiceTts: "pocket-tts-onnx-int8",
+          },
        },
        voiceLlmProvider: "codex",
        voiceLlmProviderExplicit: true,