diff --git a/packages/server/src/server/agent/agent-management-mcp.ts b/packages/server/src/server/agent/agent-management-mcp.ts index e2e26d0b6..4f46e0882 100644 --- a/packages/server/src/server/agent/agent-management-mcp.ts +++ b/packages/server/src/server/agent/agent-management-mcp.ts @@ -23,8 +23,6 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { z } from "zod"; -import { homedir } from "node:os"; -import { resolve } from "node:path"; import { ensureValidJson } from "../json-utils.js"; import type { Logger } from "pino"; @@ -51,6 +49,7 @@ import { AgentStorage } from "./agent-storage.js"; import { createWorktree } from "../../utils/worktree.js"; import { WaitForAgentTracker } from "./wait-for-agent-tracker.js"; import { scheduleAgentMetadataGeneration } from "./agent-metadata-generator.js"; +import { expandUserPath } from "../path-utils.js"; export interface AgentManagementMcpOptions { agentManager: AgentManager; @@ -77,13 +76,6 @@ const AgentStatusEnum = z.enum([ // 50 seconds - surface friendly message before SDK tool timeout (~60s) const AGENT_WAIT_TIMEOUT_MS = 50000; -function expandPath(path: string): string { - if (path.startsWith("~/") || path === "~") { - return resolve(homedir(), path.slice(2)); - } - return resolve(path); -} - async function waitForAgentWithTimeout( agentManager: AgentManager, agentId: string, @@ -320,7 +312,7 @@ export async function createAgentManagementMcpServer( title: string; }; - let resolvedCwd = expandPath(cwd); + let resolvedCwd = expandUserPath(cwd); if (worktreeName) { if (!baseBranch) { diff --git a/packages/server/src/server/agent/mcp-server.ts b/packages/server/src/server/agent/mcp-server.ts index 3c83aae76..b8c62667e 100644 --- a/packages/server/src/server/agent/mcp-server.ts +++ b/packages/server/src/server/agent/mcp-server.ts @@ -1,9 +1,12 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { z } from "zod"; -import { homedir } from "node:os"; -import { resolve } from "node:path"; import { ensureValidJson } from "../json-utils.js"; import type { Logger } from "pino"; +import type { RequestHandlerExtra } from "@modelcontextprotocol/sdk/shared/protocol.js"; +import type { + ServerNotification, + ServerRequest, +} from "@modelcontextprotocol/sdk/types.js"; import type { AgentPromptInput, @@ -32,6 +35,7 @@ import type { VoiceCallerContext, VoiceSpeakHandler, } from "../voice-types.js"; +import { expandUserPath, resolvePathFromBase } from "../path-utils.js"; export interface AgentMcpServerOptions { agentManager: AgentManager; @@ -115,11 +119,25 @@ const AgentStatusEnum = z.enum([ // 50 seconds - surface friendly message before SDK tool timeout (~60s) const AGENT_WAIT_TIMEOUT_MS = 50000; -function expandPath(path: string): string { - if (path.startsWith("~/") || path === "~") { - return resolve(homedir(), path.slice(2)); +type McpToolContext = RequestHandlerExtra; + +function resolveChildAgentCwd(params: { + parentCwd: string; + requestedCwd?: string; + lockedCwd?: string; + allowCustomCwd: boolean; +}): string { + const lockedCwd = params.lockedCwd?.trim(); + if (lockedCwd) { + return expandUserPath(lockedCwd); } - return resolve(path); + + const requestedCwd = params.requestedCwd?.trim(); + if (!requestedCwd || !params.allowCustomCwd) { + return params.parentCwd; + } + + return resolvePathFromBase(params.parentCwd, requestedCwd); } /** @@ -369,6 +387,11 @@ export async function createAgentMcpServer( const createAgentInputSchema = callerAgentId ? agentToAgentInputSchema : topLevelInputSchema; + const agentToAgentCreateAgentArgsSchema = z.object(agentToAgentInputSchema); + const topLevelCreateAgentArgsSchema = z.object({ + ...topLevelInputSchema, + initialMode: topLevelInputSchema.initialMode.optional(), + }); if (options.enableVoiceTools || callerContext?.enableVoiceTools) { server.registerTool( @@ -388,7 +411,7 @@ export async function createAgentMcpServer( ok: z.boolean(), }, }, - async (args, context) => { + async (args, context?: McpToolContext) => { if (!callerAgentId) { throw new Error("speak is only available to agent-scoped MCP sessions"); } @@ -399,7 +422,7 @@ export async function createAgentMcpServer( await handler({ text: args.text, callerAgentId, - signal: (context as { signal?: AbortSignal } | undefined)?.signal, + signal: context?.signal, }); return { content: [], @@ -434,44 +457,31 @@ export async function createAgentMcpServer( }, }, async (args: unknown) => { - const { - agentType, - initialPrompt, - background = false, - title, - } = args as { - cwd?: string; - agentType?: AgentProvider; - initialPrompt: string; - initialMode?: string; - worktreeName?: string; - background?: boolean; - title: string; - }; + let provider: AgentProvider; + let initialPrompt: string; + let background = false; + let normalizedTitle: string | null; let resolvedCwd: string; let resolvedMode: string | undefined; if (callerAgentId) { + const callerArgs = agentToAgentCreateAgentArgsSchema.parse(args); + provider = callerArgs.agentType ?? "claude"; + initialPrompt = callerArgs.initialPrompt; + background = callerArgs.background ?? false; + normalizedTitle = callerArgs.title.trim(); + const parentAgent = agentManager.getAgent(callerAgentId); if (!parentAgent) { throw new Error(`Parent agent ${callerAgentId} not found`); } - const callerArgs = args as unknown as { cwd?: string }; - const requestedCwd = callerArgs.cwd?.trim(); - const lockedCwd = callerContext?.lockedCwd?.trim(); - if (lockedCwd) { - resolvedCwd = expandPath(lockedCwd); - } else if (requestedCwd && (callerContext?.allowCustomCwd ?? true)) { - resolvedCwd = - requestedCwd.startsWith("/") || requestedCwd.startsWith("~") - ? expandPath(requestedCwd) - : resolve(parentAgent.cwd, requestedCwd); - } else { - resolvedCwd = parentAgent.cwd; - } - - const provider: AgentProvider = agentType ?? "claude"; + resolvedCwd = resolveChildAgentCwd({ + parentCwd: parentAgent.cwd, + requestedCwd: callerArgs.cwd, + lockedCwd: callerContext?.lockedCwd, + allowCustomCwd: callerContext?.allowCustomCwd ?? true, + }); const parentMode = parentAgent.currentModeId; if (parentMode) { resolvedMode = mapModeAcrossProviders( @@ -481,12 +491,11 @@ export async function createAgentMcpServer( ); } } else { - const topLevelArgs = args as unknown as { - cwd: string; - initialMode: string; - worktreeName?: string; - baseBranch?: string; - }; + const topLevelArgs = topLevelCreateAgentArgsSchema.parse(args); + provider = topLevelArgs.agentType ?? "claude"; + initialPrompt = topLevelArgs.initialPrompt; + background = topLevelArgs.background ?? false; + normalizedTitle = topLevelArgs.title.trim(); const { cwd, initialMode, @@ -494,7 +503,7 @@ export async function createAgentMcpServer( baseBranch, } = topLevelArgs; - resolvedCwd = expandPath(cwd); + resolvedCwd = expandUserPath(cwd); if (worktreeName) { if (!baseBranch) { @@ -513,8 +522,6 @@ export async function createAgentMcpServer( resolvedMode = initialMode; } - const provider: AgentProvider = agentType ?? "claude"; - const normalizedTitle = title?.trim() ?? null; const childAgentDefaultLabels = callerAgentId && callerContext?.childAgentDefaultLabels ? callerContext.childAgentDefaultLabels diff --git a/packages/server/src/server/path-utils.ts b/packages/server/src/server/path-utils.ts new file mode 100644 index 000000000..76403ef36 --- /dev/null +++ b/packages/server/src/server/path-utils.ts @@ -0,0 +1,22 @@ +import { homedir } from "node:os"; +import { isAbsolute, resolve } from "node:path"; + +function hasHomePrefix(value: string): boolean { + return value === "~" || value.startsWith("~/"); +} + +export function expandUserPath(value: string): string { + const trimmed = value.trim(); + if (hasHomePrefix(trimmed)) { + return resolve(homedir(), trimmed.slice(2)); + } + return resolve(trimmed); +} + +export function resolvePathFromBase(baseCwd: string, requestedPath: string): string { + const trimmed = requestedPath.trim(); + if (hasHomePrefix(trimmed) || isAbsolute(trimmed)) { + return expandUserPath(trimmed); + } + return resolve(baseCwd, trimmed); +} diff --git a/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts b/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts index 6e679d6db..8012fd911 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts @@ -2,57 +2,75 @@ import { z } from "zod"; export type SherpaOnnxModelKind = "stt-online" | "stt-offline" | "tts"; -export type SherpaOnnxModelId = - | "zipformer-bilingual-zh-en-2023-02-20" - | "paraformer-bilingual-zh-en" - | "parakeet-tdt-0.6b-v3-int8" - | "kitten-nano-en-v0_1-fp16" - | "kokoro-en-v0_19" - | "pocket-tts-onnx-int8"; +const MODEL_IDS = { + ZIPFORMER_BILINGUAL_ZH_EN_2023_02_20: "zipformer-bilingual-zh-en-2023-02-20", + PARAFORMER_BILINGUAL_ZH_EN: "paraformer-bilingual-zh-en", + PARAKEET_TDT_0_6B_V3_INT8: "parakeet-tdt-0.6b-v3-int8", + KITTEN_NANO_EN_V0_1_FP16: "kitten-nano-en-v0_1-fp16", + KOKORO_EN_V0_19: "kokoro-en-v0_19", + POCKET_TTS_ONNX_INT8: "pocket-tts-onnx-int8", +} as const; + +export type SherpaOnnxModelId = (typeof MODEL_IDS)[keyof typeof MODEL_IDS]; export const LOCAL_STT_MODEL_IDS = [ - "zipformer-bilingual-zh-en-2023-02-20", - "paraformer-bilingual-zh-en", - "parakeet-tdt-0.6b-v3-int8", + MODEL_IDS.ZIPFORMER_BILINGUAL_ZH_EN_2023_02_20, + MODEL_IDS.PARAFORMER_BILINGUAL_ZH_EN, + MODEL_IDS.PARAKEET_TDT_0_6B_V3_INT8, ] as const; export type LocalSttModelId = (typeof LOCAL_STT_MODEL_IDS)[number]; export const LOCAL_TTS_MODEL_IDS = [ - "kitten-nano-en-v0_1-fp16", - "kokoro-en-v0_19", - "pocket-tts-onnx-int8", + MODEL_IDS.KITTEN_NANO_EN_V0_1_FP16, + MODEL_IDS.KOKORO_EN_V0_19, + MODEL_IDS.POCKET_TTS_ONNX_INT8, ] as const; export type LocalTtsModelId = (typeof LOCAL_TTS_MODEL_IDS)[number]; -const STT_MODEL_ALIASES: Record = { - zipformer: "zipformer-bilingual-zh-en-2023-02-20", - "zipformer-bilingual": "zipformer-bilingual-zh-en-2023-02-20", - paraformer: "paraformer-bilingual-zh-en", - parakeet: "parakeet-tdt-0.6b-v3-int8", - "parakeet-v3": "parakeet-tdt-0.6b-v3-int8", - "parakeet-tdt": "parakeet-tdt-0.6b-v3-int8", +export const DEFAULT_LOCAL_STT_MODEL: LocalSttModelId = MODEL_IDS.PARAKEET_TDT_0_6B_V3_INT8; +export const DEFAULT_LOCAL_TTS_MODEL: LocalTtsModelId = MODEL_IDS.POCKET_TTS_ONNX_INT8; + +const STT_MODEL_ALIASES: Record = { + zipformer: MODEL_IDS.ZIPFORMER_BILINGUAL_ZH_EN_2023_02_20, + "zipformer-bilingual": MODEL_IDS.ZIPFORMER_BILINGUAL_ZH_EN_2023_02_20, + paraformer: MODEL_IDS.PARAFORMER_BILINGUAL_ZH_EN, + parakeet: MODEL_IDS.PARAKEET_TDT_0_6B_V3_INT8, + "parakeet-v3": MODEL_IDS.PARAKEET_TDT_0_6B_V3_INT8, + "parakeet-tdt": MODEL_IDS.PARAKEET_TDT_0_6B_V3_INT8, }; -const TTS_MODEL_ALIASES: Record = { - pocket: "pocket-tts-onnx-int8", - "pocket-tts": "pocket-tts-onnx-int8", - kitten: "kitten-nano-en-v0_1-fp16", - kokoro: "kokoro-en-v0_19", +const TTS_MODEL_ALIASES: Record = { + pocket: MODEL_IDS.POCKET_TTS_ONNX_INT8, + "pocket-tts": MODEL_IDS.POCKET_TTS_ONNX_INT8, + kitten: MODEL_IDS.KITTEN_NANO_EN_V0_1_FP16, + kokoro: MODEL_IDS.KOKORO_EN_V0_19, }; -export const LocalSttModelIdSchema = z.preprocess((value) => { - if (typeof value !== "string") return value; - const normalized = value.trim().toLowerCase(); - if (!normalized) return value; - return STT_MODEL_ALIASES[normalized] ?? normalized; -}, z.enum(LOCAL_STT_MODEL_IDS)); +function createAliasedModelIdSchema( + values: T, + aliases: Record +) { + return z.preprocess((value) => { + if (typeof value !== "string") { + return value; + } + const normalized = value.trim().toLowerCase(); + if (!normalized) { + return value; + } + return aliases[normalized] ?? normalized; + }, z.enum(values)); +} -export const LocalTtsModelIdSchema = z.preprocess((value) => { - if (typeof value !== "string") return value; - const normalized = value.trim().toLowerCase(); - if (!normalized) return value; - return TTS_MODEL_ALIASES[normalized] ?? normalized; -}, z.enum(LOCAL_TTS_MODEL_IDS)); +export const LocalSttModelIdSchema = createAliasedModelIdSchema( + LOCAL_STT_MODEL_IDS, + STT_MODEL_ALIASES +); + +export const LocalTtsModelIdSchema = createAliasedModelIdSchema( + LOCAL_TTS_MODEL_IDS, + TTS_MODEL_ALIASES +); export type SherpaOnnxModelSpec = { id: SherpaOnnxModelId; @@ -65,8 +83,8 @@ export type SherpaOnnxModelSpec = { }; export const SHERPA_ONNX_MODEL_CATALOG: Record = { - "zipformer-bilingual-zh-en-2023-02-20": { - id: "zipformer-bilingual-zh-en-2023-02-20", + [MODEL_IDS.ZIPFORMER_BILINGUAL_ZH_EN_2023_02_20]: { + id: MODEL_IDS.ZIPFORMER_BILINGUAL_ZH_EN_2023_02_20, kind: "stt-online", archiveUrl: "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2", @@ -79,8 +97,8 @@ export const SHERPA_ONNX_MODEL_CATALOG: Record any; @@ -10,39 +8,12 @@ export type SherpaOnnxModule = { let cached: SherpaOnnxModule | null = null; -function ensureSherpaNativeLibraryPath(requireFn: NodeRequire): void { - const platform = os.platform(); - if (platform !== "darwin" && platform !== "linux") { - return; - } - - const platformArch = `${platform}-${os.arch()}`; - const packageName = `sherpa-onnx-${platformArch}`; - - let nativeDir: string; - try { - const binaryPath = requireFn.resolve(`${packageName}/sherpa-onnx.node`); - nativeDir = path.dirname(binaryPath); - } catch { - return; - } - - const envKey = platform === "darwin" ? "DYLD_LIBRARY_PATH" : "LD_LIBRARY_PATH"; - const current = process.env[envKey]?.trim() ?? ""; - const entries = current.length > 0 ? current.split(":").filter(Boolean) : []; - if (entries.includes(nativeDir)) { - return; - } - - process.env[envKey] = entries.length > 0 ? `${nativeDir}:${entries.join(":")}` : nativeDir; -} - export function loadSherpaOnnx(): SherpaOnnxModule { if (cached) { return cached; } + const require = createRequire(import.meta.url); - ensureSherpaNativeLibraryPath(require); cached = require("sherpa-onnx") as SherpaOnnxModule; return cached; } diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-node-loader.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-node-loader.ts index 8318b8673..81498c079 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-node-loader.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-node-loader.ts @@ -14,13 +14,16 @@ function platformArch(): string { return `${platform}-${process.arch}`; } -function prependLibraryPath(envKey: "DYLD_LIBRARY_PATH" | "LD_LIBRARY_PATH", dir: string): void { +function prependLibraryPath( + envKey: "DYLD_LIBRARY_PATH" | "LD_LIBRARY_PATH" | "PATH", + dir: string +): void { const current = process.env[envKey] ?? ""; - const parts = current.split(":").filter(Boolean); + const parts = current.split(path.delimiter).filter(Boolean); if (parts.includes(dir)) { return; } - process.env[envKey] = [dir, ...parts].join(":"); + process.env[envKey] = [dir, ...parts].join(path.delimiter); } export function loadSherpaOnnxNode(): SherpaOnnxNodeModule { @@ -31,8 +34,7 @@ export function loadSherpaOnnxNode(): SherpaOnnxNodeModule { const require = createRequire(import.meta.url); // sherpa-onnx-node depends on a platform-specific package (e.g. sherpa-onnx-darwin-arm64) - // that contains the native addon + its shared libraries. On macOS/Linux we need to ensure - // the appropriate library path env var includes that directory before requiring the addon. + // that contains the native addon and shared libraries. Ensure the OS loader path includes it. const arch = platformArch(); const pkgName = `sherpa-onnx-${arch}`; @@ -43,6 +45,8 @@ export function loadSherpaOnnxNode(): SherpaOnnxNodeModule { prependLibraryPath("DYLD_LIBRARY_PATH", pkgDir); } else if (process.platform === "linux") { prependLibraryPath("LD_LIBRARY_PATH", pkgDir); + } else if (process.platform === "win32") { + prependLibraryPath("PATH", pkgDir); } } catch { // Best effort - if the platform package isn't present, require() below will throw a useful error. diff --git a/packages/server/src/server/speech/speech-config-resolver.ts b/packages/server/src/server/speech/speech-config-resolver.ts index 921427d08..e4a54f9e3 100644 --- a/packages/server/src/server/speech/speech-config-resolver.ts +++ b/packages/server/src/server/speech/speech-config-resolver.ts @@ -1,104 +1,172 @@ import path from "node:path"; -import type { STTConfig } from "./providers/openai/stt.js"; -import type { TTSConfig } from "./providers/openai/tts.js"; +import { z } from "zod"; + import type { PersistedConfig } from "../persisted-config.js"; import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js"; -import { LocalSttModelIdSchema, LocalTtsModelIdSchema } from "./providers/local/sherpa/model-catalog.js"; +import { + LocalSttModelIdSchema, + LocalTtsModelIdSchema, +} from "./providers/local/sherpa/model-catalog.js"; import { DEFAULT_LOCAL_STT_MODEL, DEFAULT_LOCAL_TTS_MODEL, DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL, - type SpeechProviderId, -} from "./speech-types.js"; + DEFAULT_OPENAI_TTS_MODEL, +} from "./speech-defaults.js"; +import { SpeechProviderIdSchema } from "./speech-types.js"; + const DEFAULT_LOCAL_MODELS_SUBDIR = path.join("models", "local-speech"); -const DEFAULT_OPENAI_TTS_MODEL: TTSConfig["model"] = "tts-1"; -function parseSpeechProviderId(value: unknown): SpeechProviderId | null { - if (typeof value !== "string") return null; +const OpenAiTtsVoiceSchema = z.enum([ + "alloy", + "echo", + "fable", + "onyx", + "nova", + "shimmer", +]); + +const OpenAiTtsModelSchema = z.enum(["tts-1", "tts-1-hd"]); + +const OptionalSpeechProviderSchema = z.preprocess((value) => { + if (typeof value !== "string") { + return value; + } const normalized = value.trim().toLowerCase(); - if (!normalized) return null; - if (normalized === "openai") return "openai"; - if (normalized === "local") return "local"; - return null; -} + return normalized.length > 0 ? normalized : undefined; +}, SpeechProviderIdSchema.optional()); -function parseBooleanFlag(value: string | undefined): boolean | null { - if (value === undefined) return null; +const OptionalBooleanFlagSchema = z.preprocess((value) => { + if (typeof value === "boolean") { + return value; + } + if (typeof value !== "string") { + return value; + } const normalized = value.trim().toLowerCase(); - if (normalized === "1" || normalized === "true" || normalized === "yes") return true; - if (normalized === "0" || normalized === "false" || normalized === "no") return false; - return null; -} + if (normalized === "1" || normalized === "true" || normalized === "yes") { + return true; + } + if (normalized === "0" || normalized === "false" || normalized === "no") { + return false; + } + return undefined; +}, z.boolean().optional()); -function parseNumberOrUndefined(value: string | undefined): number | undefined { - if (value === undefined) return undefined; +const OptionalFiniteNumberSchema = z.preprocess((value) => { + if (typeof value === "number") { + return Number.isFinite(value) ? value : undefined; + } + if (typeof value !== "string") { + return value; + } const parsed = Number.parseFloat(value); return Number.isFinite(parsed) ? parsed : undefined; -} +}, z.number().optional()); -function parseIntOrUndefined(value: string | undefined): number | undefined { - if (value === undefined) return undefined; +const OptionalIntegerSchema = z.preprocess((value) => { + if (typeof value === "number") { + return Number.isInteger(value) ? value : undefined; + } + if (typeof value !== "string") { + return value; + } const parsed = Number.parseInt(value, 10); return Number.isFinite(parsed) ? parsed : undefined; -} +}, z.number().int().optional()); -function parseOpenAiConfig( - env: NodeJS.ProcessEnv, - persisted: PersistedConfig, - providers: { - dictationSttProvider: SpeechProviderId; - voiceSttProvider: SpeechProviderId; - voiceTtsProvider: SpeechProviderId; +const OptionalTrimmedStringSchema = z.preprocess((value) => { + if (typeof value !== "string") { + return value; } -): PaseoOpenAIConfig | undefined { - const apiKey = env.OPENAI_API_KEY ?? persisted.providers?.openai?.apiKey; - if (!apiKey) return undefined; + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : undefined; +}, z.string().optional()); - const sttConfidenceThreshold = parseNumberOrUndefined(env.STT_CONFIDENCE_THRESHOLD) - ?? persisted.features?.dictation?.stt?.confidenceThreshold; +const ResolvedSpeechConfigSchema = z + .object({ + dictationSttProvider: OptionalSpeechProviderSchema.default("local"), + voiceSttProvider: OptionalSpeechProviderSchema.default("local"), + voiceTtsProvider: OptionalSpeechProviderSchema.default("local"), + localModelsDir: z.string().trim().min(1), + localAutoDownload: OptionalBooleanFlagSchema.default(true), + dictationLocalSttModel: LocalSttModelIdSchema.default(DEFAULT_LOCAL_STT_MODEL), + voiceLocalSttModel: LocalSttModelIdSchema.default(DEFAULT_LOCAL_STT_MODEL), + voiceLocalTtsModel: LocalTtsModelIdSchema.default(DEFAULT_LOCAL_TTS_MODEL), + voiceLocalTtsSpeakerId: OptionalIntegerSchema, + voiceLocalTtsSpeed: OptionalFiniteNumberSchema, + anyLocalRequested: z.boolean(), + openaiApiKey: OptionalTrimmedStringSchema, + openaiSttConfidenceThreshold: OptionalFiniteNumberSchema, + openaiSttModel: OptionalTrimmedStringSchema, + openaiTtsVoice: z.preprocess((value) => { + if (typeof value !== "string") { + return value; + } + const normalized = value.trim().toLowerCase(); + return normalized.length > 0 ? normalized : undefined; + }, OpenAiTtsVoiceSchema.default("alloy")), + openaiTtsModel: z.preprocess((value) => { + if (typeof value !== "string") { + return value; + } + const normalized = value.trim().toLowerCase(); + return normalized.length > 0 ? normalized : undefined; + }, OpenAiTtsModelSchema.default(DEFAULT_OPENAI_TTS_MODEL)), + openaiRealtimeTranscriptionModel: OptionalTrimmedStringSchema.default( + DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL + ), + }) + .transform((input): { openai: PaseoOpenAIConfig | undefined; speech: PaseoSpeechConfig } => { + const openai = input.openaiApiKey + ? { + apiKey: input.openaiApiKey, + stt: { + apiKey: input.openaiApiKey, + ...(input.openaiSttConfidenceThreshold !== undefined + ? { confidenceThreshold: input.openaiSttConfidenceThreshold } + : {}), + ...(input.openaiSttModel + ? { model: input.openaiSttModel } + : {}), + }, + tts: { + apiKey: input.openaiApiKey, + voice: input.openaiTtsVoice, + model: input.openaiTtsModel, + responseFormat: "pcm" as const, + }, + realtimeTranscriptionModel: input.openaiRealtimeTranscriptionModel, + } + : undefined; - const sttModel = ( - env.STT_MODEL - ?? (providers.voiceSttProvider === "openai" ? persisted.features?.voiceMode?.stt?.model : undefined) - ?? (providers.dictationSttProvider === "openai" ? persisted.features?.dictation?.stt?.model : undefined) - ) as STTConfig["model"] | undefined; - - const ttsVoice = ( - env.TTS_VOICE - || (providers.voiceTtsProvider === "openai" ? persisted.features?.voiceMode?.tts?.voice : undefined) - || "alloy" - ) as TTSConfig["voice"]; - const ttsModelRaw = - env.TTS_MODEL - || (providers.voiceTtsProvider === "openai" ? persisted.features?.voiceMode?.tts?.model : undefined) - || DEFAULT_OPENAI_TTS_MODEL; - const ttsModel: TTSConfig["model"] = - ttsModelRaw === "tts-1" || ttsModelRaw === "tts-1-hd" ? ttsModelRaw : DEFAULT_OPENAI_TTS_MODEL; - - const realtimeTranscriptionModel = - env.OPENAI_REALTIME_TRANSCRIPTION_MODEL - || (providers.dictationSttProvider === "openai" - ? persisted.features?.dictation?.stt?.model - : undefined) - || DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL; - - return { - apiKey, - stt: { - apiKey, - ...(sttConfidenceThreshold !== undefined ? { confidenceThreshold: sttConfidenceThreshold } : {}), - ...(sttModel ? { model: sttModel } : {}), - }, - tts: { - apiKey, - voice: ttsVoice, - model: ttsModel, - responseFormat: "pcm", - }, - realtimeTranscriptionModel, - }; -} + return { + openai, + speech: { + dictationSttProvider: input.dictationSttProvider, + voiceSttProvider: input.voiceSttProvider, + voiceTtsProvider: input.voiceTtsProvider, + ...(input.anyLocalRequested + ? { + local: { + modelsDir: input.localModelsDir, + autoDownload: input.localAutoDownload, + }, + } + : {}), + dictationLocalSttModel: input.dictationLocalSttModel, + voiceLocalSttModel: input.voiceLocalSttModel, + voiceLocalTtsModel: input.voiceLocalTtsModel, + ...(input.voiceLocalTtsSpeakerId !== undefined + ? { voiceLocalTtsSpeakerId: input.voiceLocalTtsSpeakerId } + : {}), + ...(input.voiceLocalTtsSpeed !== undefined + ? { voiceLocalTtsSpeed: input.voiceLocalTtsSpeed } + : {}), + }, + }; + }); export function resolveSpeechConfig(params: { paseoHome: string; @@ -111,18 +179,18 @@ export function resolveSpeechConfig(params: { const { paseoHome, env, persisted } = params; const dictationSttProvider = - parseSpeechProviderId(env.PASEO_DICTATION_STT_PROVIDER) - ?? parseSpeechProviderId(persisted.features?.dictation?.stt?.provider) + env.PASEO_DICTATION_STT_PROVIDER + ?? persisted.features?.dictation?.stt?.provider ?? "local"; const voiceSttProvider = - parseSpeechProviderId(env.PASEO_VOICE_STT_PROVIDER) - ?? parseSpeechProviderId(persisted.features?.voiceMode?.stt?.provider) + env.PASEO_VOICE_STT_PROVIDER + ?? persisted.features?.voiceMode?.stt?.provider ?? "local"; const voiceTtsProvider = - parseSpeechProviderId(env.PASEO_VOICE_TTS_PROVIDER) - ?? parseSpeechProviderId(persisted.features?.voiceMode?.tts?.provider) + env.PASEO_VOICE_TTS_PROVIDER + ?? persisted.features?.voiceMode?.tts?.provider ?? "local"; const anyLocalRequested = @@ -132,65 +200,65 @@ export function resolveSpeechConfig(params: { env.PASEO_LOCAL_MODELS_DIR !== undefined || persisted.providers?.local !== undefined; - const localModelsDir = - env.PASEO_LOCAL_MODELS_DIR - ?? persisted.providers?.local?.modelsDir - ?? path.join(paseoHome, DEFAULT_LOCAL_MODELS_SUBDIR); - - const localAutoDownload = - parseBooleanFlag(env.PASEO_LOCAL_AUTO_DOWNLOAD) - ?? persisted.providers?.local?.autoDownload - ?? true; - - const dictationLocalSttModel = LocalSttModelIdSchema.parse( - env.PASEO_DICTATION_LOCAL_STT_MODEL - ?? persisted.features?.dictation?.stt?.model - ?? DEFAULT_LOCAL_STT_MODEL - ); - - const voiceLocalSttModel = LocalSttModelIdSchema.parse( - env.PASEO_VOICE_LOCAL_STT_MODEL - ?? persisted.features?.voiceMode?.stt?.model - ?? DEFAULT_LOCAL_STT_MODEL - ); - - const voiceLocalTtsModel = LocalTtsModelIdSchema.parse( - env.PASEO_VOICE_LOCAL_TTS_MODEL - ?? persisted.features?.voiceMode?.tts?.model - ?? DEFAULT_LOCAL_TTS_MODEL - ); - - const voiceLocalTtsSpeakerId = - parseIntOrUndefined(env.PASEO_VOICE_LOCAL_TTS_SPEAKER_ID) - ?? persisted.features?.voiceMode?.tts?.speakerId; - - const voiceLocalTtsSpeed = - parseNumberOrUndefined(env.PASEO_VOICE_LOCAL_TTS_SPEED) - ?? persisted.features?.voiceMode?.tts?.speed; - - return { - openai: parseOpenAiConfig(env, persisted, { - dictationSttProvider, - voiceSttProvider, - voiceTtsProvider, - }), - speech: { - dictationSttProvider, - voiceSttProvider, - voiceTtsProvider, - ...(anyLocalRequested - ? { - local: { - modelsDir: localModelsDir.trim(), - autoDownload: localAutoDownload, - }, - } - : {}), - dictationLocalSttModel, - voiceLocalSttModel, - voiceLocalTtsModel, - ...(voiceLocalTtsSpeakerId !== undefined ? { voiceLocalTtsSpeakerId } : {}), - ...(voiceLocalTtsSpeed !== undefined ? { voiceLocalTtsSpeed } : {}), - }, - }; + return ResolvedSpeechConfigSchema.parse({ + dictationSttProvider, + voiceSttProvider, + voiceTtsProvider, + localModelsDir: + env.PASEO_LOCAL_MODELS_DIR + ?? persisted.providers?.local?.modelsDir + ?? path.join(paseoHome, DEFAULT_LOCAL_MODELS_SUBDIR), + localAutoDownload: + env.PASEO_LOCAL_AUTO_DOWNLOAD + ?? persisted.providers?.local?.autoDownload, + dictationLocalSttModel: + env.PASEO_DICTATION_LOCAL_STT_MODEL + ?? persisted.features?.dictation?.stt?.model + ?? DEFAULT_LOCAL_STT_MODEL, + voiceLocalSttModel: + env.PASEO_VOICE_LOCAL_STT_MODEL + ?? persisted.features?.voiceMode?.stt?.model + ?? DEFAULT_LOCAL_STT_MODEL, + voiceLocalTtsModel: + env.PASEO_VOICE_LOCAL_TTS_MODEL + ?? persisted.features?.voiceMode?.tts?.model + ?? DEFAULT_LOCAL_TTS_MODEL, + voiceLocalTtsSpeakerId: + env.PASEO_VOICE_LOCAL_TTS_SPEAKER_ID + ?? persisted.features?.voiceMode?.tts?.speakerId, + voiceLocalTtsSpeed: + env.PASEO_VOICE_LOCAL_TTS_SPEED + ?? persisted.features?.voiceMode?.tts?.speed, + anyLocalRequested, + openaiApiKey: env.OPENAI_API_KEY ?? persisted.providers?.openai?.apiKey, + openaiSttConfidenceThreshold: + env.STT_CONFIDENCE_THRESHOLD + ?? persisted.features?.dictation?.stt?.confidenceThreshold, + openaiSttModel: + env.STT_MODEL + ?? (voiceSttProvider === "openai" + ? persisted.features?.voiceMode?.stt?.model + : undefined) + ?? (dictationSttProvider === "openai" + ? persisted.features?.dictation?.stt?.model + : undefined), + openaiTtsVoice: + env.TTS_VOICE + ?? (voiceTtsProvider === "openai" + ? persisted.features?.voiceMode?.tts?.voice + : undefined) + ?? "alloy", + openaiTtsModel: + env.TTS_MODEL + ?? (voiceTtsProvider === "openai" + ? persisted.features?.voiceMode?.tts?.model + : undefined) + ?? DEFAULT_OPENAI_TTS_MODEL, + openaiRealtimeTranscriptionModel: + env.OPENAI_REALTIME_TRANSCRIPTION_MODEL + ?? (dictationSttProvider === "openai" + ? persisted.features?.dictation?.stt?.model + : undefined) + ?? DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL, + }); } diff --git a/packages/server/src/server/speech/speech-defaults.ts b/packages/server/src/server/speech/speech-defaults.ts new file mode 100644 index 000000000..c249734b9 --- /dev/null +++ b/packages/server/src/server/speech/speech-defaults.ts @@ -0,0 +1,9 @@ +import { + DEFAULT_LOCAL_STT_MODEL, + DEFAULT_LOCAL_TTS_MODEL, +} from "./providers/local/sherpa/model-catalog.js"; + +export { DEFAULT_LOCAL_STT_MODEL, DEFAULT_LOCAL_TTS_MODEL }; + +export const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe"; +export const DEFAULT_OPENAI_TTS_MODEL = "tts-1"; diff --git a/packages/server/src/server/speech/speech-runtime.ts b/packages/server/src/server/speech/speech-runtime.ts index d06f953fa..938f81f59 100644 --- a/packages/server/src/server/speech/speech-runtime.ts +++ b/packages/server/src/server/speech/speech-runtime.ts @@ -29,24 +29,48 @@ import { DEFAULT_LOCAL_STT_MODEL, DEFAULT_LOCAL_TTS_MODEL, DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL, - type SpeechProviderId, -} from "./speech-types.js"; + DEFAULT_OPENAI_TTS_MODEL, +} from "./speech-defaults.js"; +import type { SpeechProviderId } from "./speech-types.js"; type LocalSttEngine = | { kind: "offline"; engine: SherpaOfflineRecognizerEngine } | { kind: "online"; engine: SherpaOnlineRecognizerEngine }; -function buildModelDownloadHint(modelsDir: string, modelId: SherpaOnnxModelId): string { - return `Run: tsx packages/server/scripts/download-speech-models.ts --models-dir '${modelsDir}' --model '${modelId}'`; -} - -function resolveSpeechProviders( - speechConfig: PaseoSpeechConfig | null -): { +type RequestedSpeechProviders = { dictationSttProvider: SpeechProviderId; voiceSttProvider: SpeechProviderId; voiceTtsProvider: SpeechProviderId; -} { +}; + +type ResolvedLocalModels = { + dictationLocalSttModel: LocalSttModelId; + voiceLocalSttModel: LocalSttModelId; + voiceLocalTtsModel: LocalTtsModelId; +}; + +type OpenAiCredentialState = { + openaiSttApiKey: string | undefined; + openaiTtsApiKey: string | undefined; + openaiDictationApiKey: string | undefined; +}; + +type InitializedLocalSpeech = { + sttService: SpeechToTextProvider | null; + ttsService: TextToSpeechProvider | null; + dictationSttService: SpeechToTextProvider | null; + localVoiceTtsProvider: TextToSpeechProvider | null; + localSttEngines: Map; + requiredLocalModelIds: SherpaOnnxModelId[]; +}; + +function buildModelDownloadHint(modelId: SherpaOnnxModelId): string { + return `Use 'paseo speech download --model ${modelId}' to download this model.`; +} + +function resolveRequestedSpeechProviders( + speechConfig: PaseoSpeechConfig | null +): RequestedSpeechProviders { return { dictationSttProvider: speechConfig?.dictationSttProvider ?? "local", voiceSttProvider: speechConfig?.voiceSttProvider ?? "local", @@ -54,13 +78,9 @@ function resolveSpeechProviders( }; } -function resolveLocalModels( +function resolveConfiguredLocalModels( speechConfig: PaseoSpeechConfig | null -): { - dictationLocalSttModel: LocalSttModelId; - voiceLocalSttModel: LocalSttModelId; - voiceLocalTtsModel: LocalTtsModelId; -} { +): ResolvedLocalModels { return { dictationLocalSttModel: LocalSttModelIdSchema.parse( speechConfig?.dictationLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL @@ -74,12 +94,8 @@ function resolveLocalModels( }; } -function computeDefaultLocalModelIds(params: { - providers: { - dictationSttProvider: SpeechProviderId; - voiceSttProvider: SpeechProviderId; - voiceTtsProvider: SpeechProviderId; - }; +function computeRequiredLocalModelIds(params: { + providers: RequestedSpeechProviders; models: { dictationLocalSttModel: SherpaOnnxModelId; voiceLocalSttModel: SherpaOnnxModelId; @@ -99,6 +115,50 @@ function computeDefaultLocalModelIds(params: { return Array.from(ids); } +function resolveOpenAiCredentials(openaiConfig?: PaseoOpenAIConfig): OpenAiCredentialState { + const openaiApiKey = openaiConfig?.apiKey; + return { + openaiSttApiKey: openaiConfig?.stt?.apiKey ?? openaiApiKey, + openaiTtsApiKey: openaiConfig?.tts?.apiKey ?? openaiApiKey, + openaiDictationApiKey: openaiApiKey, + }; +} + +function validateOpenAiCredentialRequirements(params: { + providers: RequestedSpeechProviders; + openAiCredentials: OpenAiCredentialState; + logger: Logger; +}): void { + const { providers, openAiCredentials, logger } = params; + const missingOpenAiCredentialsFor: string[] = []; + if (providers.voiceSttProvider === "openai" && !openAiCredentials.openaiSttApiKey) { + missingOpenAiCredentialsFor.push("voice.stt"); + } + if (providers.voiceTtsProvider === "openai" && !openAiCredentials.openaiTtsApiKey) { + missingOpenAiCredentialsFor.push("voice.tts"); + } + if (providers.dictationSttProvider === "openai" && !openAiCredentials.openaiDictationApiKey) { + missingOpenAiCredentialsFor.push("dictation.stt"); + } + + if (missingOpenAiCredentialsFor.length > 0) { + logger.error( + { + requestedProviders: { + dictationStt: providers.dictationSttProvider, + voiceStt: providers.voiceSttProvider, + voiceTts: providers.voiceTtsProvider, + }, + missingOpenAiCredentialsFor, + }, + "Invalid speech configuration: OpenAI provider selected but credentials are missing" + ); + throw new Error( + `Missing OpenAI credentials for configured speech features: ${missingOpenAiCredentialsFor.join(", ")}` + ); + } +} + async function createLocalSttEngine(params: { modelId: LocalSttModelId; modelsDir: string; @@ -172,95 +232,25 @@ async function createLocalSttEngine(params: { throw new Error(`Unsupported local STT model '${modelId}'`); } -export type InitializedSpeechRuntime = { - sttService: SpeechToTextProvider | null; - ttsService: TextToSpeechProvider | null; - dictationSttService: SpeechToTextProvider | null; - cleanup: () => void; - localModelConfig: { - modelsDir: string; - defaultModelIds: SherpaOnnxModelId[]; - } | null; -}; - -export async function initializeSpeechRuntime(params: { +async function initializeLocalSpeechServices(params: { + providers: RequestedSpeechProviders; + localConfig: NonNullable | null; + localModels: ResolvedLocalModels; + speechConfig: PaseoSpeechConfig | null; logger: Logger; - openaiConfig?: PaseoOpenAIConfig; - speechConfig?: PaseoSpeechConfig; -}): Promise { - const logger = params.logger; - const speechConfig = params.speechConfig ?? null; - const localConfig = speechConfig?.local ?? null; - const openaiConfig = params.openaiConfig; +}): Promise { + const { providers, localConfig, localModels, speechConfig, logger } = params; - const providers = resolveSpeechProviders(speechConfig); - const localModels = resolveLocalModels(speechConfig); + const sttServices = { + sttService: null as SpeechToTextProvider | null, + ttsService: null as TextToSpeechProvider | null, + dictationSttService: null as SpeechToTextProvider | null, + }; - const wantsLocalDictation = providers.dictationSttProvider === "local"; - const wantsLocalVoiceStt = providers.voiceSttProvider === "local"; - const wantsLocalVoiceTts = providers.voiceTtsProvider === "local"; - - const openaiApiKey = openaiConfig?.apiKey; - const openaiSttApiKey = openaiConfig?.stt?.apiKey ?? openaiApiKey; - const openaiTtsApiKey = openaiConfig?.tts?.apiKey ?? openaiApiKey; - const openaiDictationApiKey = openaiApiKey; - - const missingOpenAiCredentialsFor: string[] = []; - if (providers.voiceSttProvider === "openai" && !openaiSttApiKey) { - missingOpenAiCredentialsFor.push("voice.stt"); - } - if (providers.voiceTtsProvider === "openai" && !openaiTtsApiKey) { - missingOpenAiCredentialsFor.push("voice.tts"); - } - if (providers.dictationSttProvider === "openai" && !openaiDictationApiKey) { - missingOpenAiCredentialsFor.push("dictation.stt"); - } - if (missingOpenAiCredentialsFor.length > 0) { - logger.error( - { - requestedProviders: { - dictationStt: providers.dictationSttProvider, - voiceStt: providers.voiceSttProvider, - voiceTts: providers.voiceTtsProvider, - }, - missingOpenAiCredentialsFor, - }, - "Invalid speech configuration: OpenAI provider selected but credentials are missing" - ); - throw new Error( - `Missing OpenAI credentials for configured speech features: ${missingOpenAiCredentialsFor.join(", ")}` - ); - } - - logger.info( - { - requestedProviders: { - dictationStt: providers.dictationSttProvider, - voiceStt: providers.voiceSttProvider, - voiceTts: providers.voiceTtsProvider, - }, - availability: { - openai: { - stt: Boolean(openaiSttApiKey), - tts: Boolean(openaiTtsApiKey), - dictationStt: Boolean(openaiDictationApiKey), - }, - local: { - configured: Boolean(localConfig), - modelsDir: localConfig?.modelsDir ?? null, - autoDownload: localConfig?.autoDownload ?? null, - }, - }, - }, - "Speech provider reconciliation started" - ); - - let sttService: SpeechToTextProvider | null = null; - let ttsService: TextToSpeechProvider | null = null; - let dictationSttService: SpeechToTextProvider | null = null; + const localSttEngines = new Map(); let localVoiceTtsProvider: TextToSpeechProvider | null = null; - const requiredLocalModelIds = computeDefaultLocalModelIds({ + const requiredLocalModelIds = computeRequiredLocalModelIds({ providers, models: localModels, }); @@ -289,15 +279,14 @@ export async function initializeSpeechRuntime(params: { modelIds: requiredLocalModelIds, autoDownload: localConfig.autoDownload ?? true, hint: - "Run: npm run dev --workspace=@getpaseo/server, then run: " + - "`tsx packages/server/scripts/download-speech-models.ts --models-dir --model `", + "Use `paseo speech models` to inspect status and " + + "`paseo speech download --model ` to fetch missing models.", }, "Failed to ensure local speech models" ); } } - const localSttEngines = new Map(); const getLocalSttEngine = async ( modelId: LocalSttModelId ): Promise => { @@ -322,7 +311,7 @@ export async function initializeSpeechRuntime(params: { err, modelsDir: localConfig.modelsDir, modelId, - hint: buildModelDownloadHint(localConfig.modelsDir, modelId), + hint: buildModelDownloadHint(modelId), }, "Failed to initialize local STT engine (models missing or invalid)" ); @@ -330,7 +319,7 @@ export async function initializeSpeechRuntime(params: { } }; - if (wantsLocalVoiceStt) { + if (providers.voiceSttProvider === "local") { if (!localConfig) { logger.warn( { configured: false }, @@ -339,14 +328,14 @@ export async function initializeSpeechRuntime(params: { } else { const voiceEngine = await getLocalSttEngine(localModels.voiceLocalSttModel); if (voiceEngine?.kind === "offline") { - sttService = new SherpaOnnxParakeetSTT({ engine: voiceEngine.engine }, logger); + sttServices.sttService = new SherpaOnnxParakeetSTT({ engine: voiceEngine.engine }, logger); } else if (voiceEngine?.kind === "online") { - sttService = new SherpaOnnxSTT({ engine: voiceEngine.engine }, logger); + sttServices.sttService = new SherpaOnnxSTT({ engine: voiceEngine.engine }, logger); } } } - if (wantsLocalDictation) { + if (providers.dictationSttProvider === "local") { if (!localConfig) { logger.warn( { configured: false }, @@ -355,13 +344,13 @@ export async function initializeSpeechRuntime(params: { } else { const dictationEngine = await getLocalSttEngine(localModels.dictationLocalSttModel); if (dictationEngine?.kind === "offline") { - dictationSttService = { + sttServices.dictationSttService = { id: "local", createSession: () => new SherpaParakeetRealtimeTranscriptionSession({ engine: dictationEngine.engine }), }; } else if (dictationEngine?.kind === "online") { - dictationSttService = { + sttServices.dictationSttService = { id: "local", createSession: () => new SherpaRealtimeTranscriptionSession({ engine: dictationEngine.engine }), }; @@ -369,7 +358,7 @@ export async function initializeSpeechRuntime(params: { } } - if (wantsLocalVoiceTts) { + if (providers.voiceTtsProvider === "local") { if (!localConfig) { logger.warn( { configured: false }, @@ -399,14 +388,14 @@ export async function initializeSpeechRuntime(params: { logger ); } - ttsService = localVoiceTtsProvider; + sttServices.ttsService = localVoiceTtsProvider; } catch (err) { logger.error( { err, modelsDir: localConfig.modelsDir, modelId: localModels.voiceLocalTtsModel, - hint: buildModelDownloadHint(localConfig.modelsDir, localModels.voiceLocalTtsModel), + hint: buildModelDownloadHint(localModels.voiceLocalTtsModel), }, "Failed to initialize local TTS engine (models missing or invalid)" ); @@ -414,34 +403,65 @@ export async function initializeSpeechRuntime(params: { } } + return { + ...sttServices, + localVoiceTtsProvider, + localSttEngines, + requiredLocalModelIds, + }; +} + +function initializeOpenAiSpeechServices(params: { + providers: RequestedSpeechProviders; + openaiConfig?: PaseoOpenAIConfig; + openAiCredentials: OpenAiCredentialState; + existing: { + sttService: SpeechToTextProvider | null; + ttsService: TextToSpeechProvider | null; + dictationSttService: SpeechToTextProvider | null; + }; + logger: Logger; +}): { + sttService: SpeechToTextProvider | null; + ttsService: TextToSpeechProvider | null; + dictationSttService: SpeechToTextProvider | null; +} { + const { providers, openaiConfig, openAiCredentials, existing, logger } = params; + + let sttService = existing.sttService; + let ttsService = existing.ttsService; + let dictationSttService = existing.dictationSttService; + const needsOpenAiStt = !sttService && providers.voiceSttProvider === "openai"; const needsOpenAiTts = !ttsService && providers.voiceTtsProvider === "openai"; const needsOpenAiDictation = !dictationSttService && providers.dictationSttProvider === "openai"; if ( (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) && - (openaiSttApiKey || openaiTtsApiKey || openaiDictationApiKey) + (openAiCredentials.openaiSttApiKey || + openAiCredentials.openaiTtsApiKey || + openAiCredentials.openaiDictationApiKey) ) { logger.info("OpenAI speech provider initialized"); - if (needsOpenAiStt && openaiSttApiKey) { + if (needsOpenAiStt && openAiCredentials.openaiSttApiKey) { const { apiKey: _sttApiKey, ...sttConfig } = openaiConfig?.stt ?? {}; sttService = new OpenAISTT( { - apiKey: openaiSttApiKey, + apiKey: openAiCredentials.openaiSttApiKey, ...sttConfig, }, logger ); } - if (needsOpenAiTts && openaiTtsApiKey) { + if (needsOpenAiTts && openAiCredentials.openaiTtsApiKey) { const { apiKey: _ttsApiKey, ...ttsConfig } = openaiConfig?.tts ?? {}; ttsService = new OpenAITTS( { - apiKey: openaiTtsApiKey, + apiKey: openAiCredentials.openaiTtsApiKey, voice: "alloy", - model: "tts-1", + model: DEFAULT_OPENAI_TTS_MODEL, responseFormat: "pcm", ...ttsConfig, }, @@ -449,12 +469,13 @@ export async function initializeSpeechRuntime(params: { ); } - if (needsOpenAiDictation && openaiDictationApiKey) { + const dictationApiKey = openAiCredentials.openaiDictationApiKey; + if (needsOpenAiDictation && dictationApiKey) { dictationSttService = { id: "openai", createSession: ({ logger: sessionLogger, language, prompt }) => new OpenAIRealtimeTranscriptionSession({ - apiKey: openaiDictationApiKey, + apiKey: dictationApiKey, logger: sessionLogger, transcriptionModel: openaiConfig?.realtimeTranscriptionModel @@ -469,15 +490,101 @@ export async function initializeSpeechRuntime(params: { logger.warn("OpenAI speech providers are configured but credentials are missing"); } + return { + sttService, + ttsService, + dictationSttService, + }; +} + +export type InitializedSpeechRuntime = { + sttService: SpeechToTextProvider | null; + ttsService: TextToSpeechProvider | null; + dictationSttService: SpeechToTextProvider | null; + cleanup: () => void; + localModelConfig: { + modelsDir: string; + defaultModelIds: SherpaOnnxModelId[]; + } | null; +}; + +export async function initializeSpeechRuntime(params: { + logger: Logger; + openaiConfig?: PaseoOpenAIConfig; + speechConfig?: PaseoSpeechConfig; +}): Promise { + const logger = params.logger; + const speechConfig = params.speechConfig ?? null; + const localConfig = speechConfig?.local ?? null; + const openaiConfig = params.openaiConfig; + + const providers = resolveRequestedSpeechProviders(speechConfig); + const localModels = resolveConfiguredLocalModels(speechConfig); + const openAiCredentials = resolveOpenAiCredentials(openaiConfig); + + validateOpenAiCredentialRequirements({ + providers, + openAiCredentials, + logger, + }); + + logger.info( + { + requestedProviders: { + dictationStt: providers.dictationSttProvider, + voiceStt: providers.voiceSttProvider, + voiceTts: providers.voiceTtsProvider, + }, + availability: { + openai: { + stt: Boolean(openAiCredentials.openaiSttApiKey), + tts: Boolean(openAiCredentials.openaiTtsApiKey), + dictationStt: Boolean(openAiCredentials.openaiDictationApiKey), + }, + local: { + configured: Boolean(localConfig), + modelsDir: localConfig?.modelsDir ?? null, + autoDownload: localConfig?.autoDownload ?? null, + }, + }, + }, + "Speech provider reconciliation started" + ); + + const localSpeech = await initializeLocalSpeechServices({ + providers, + localConfig, + localModels, + speechConfig, + logger, + }); + + const openAiSpeech = initializeOpenAiSpeechServices({ + providers, + openaiConfig, + openAiCredentials, + existing: { + sttService: localSpeech.sttService, + ttsService: localSpeech.ttsService, + dictationSttService: localSpeech.dictationSttService, + }, + logger, + }); + const effectiveProviders = { - dictationStt: dictationSttService?.id ?? "unavailable", - voiceStt: sttService?.id ?? "unavailable", - voiceTts: !ttsService ? "unavailable" : ttsService === localVoiceTtsProvider ? "local" : "openai", + dictationStt: openAiSpeech.dictationSttService?.id ?? "unavailable", + voiceStt: openAiSpeech.sttService?.id ?? "unavailable", + voiceTts: + !openAiSpeech.ttsService + ? "unavailable" + : openAiSpeech.ttsService === localSpeech.localVoiceTtsProvider + ? "local" + : "openai", }; const unavailableFeatures = [ - !dictationSttService ? "dictation.stt" : null, - !sttService ? "voice.stt" : null, - !ttsService ? "voice.tts" : null, + !openAiSpeech.dictationSttService ? "dictation.stt" : null, + !openAiSpeech.sttService ? "voice.stt" : null, + !openAiSpeech.ttsService ? "voice.tts" : null, ].filter((feature): feature is string => feature !== null); if (unavailableFeatures.length > 0) { @@ -504,25 +611,25 @@ export async function initializeSpeechRuntime(params: { ); const cleanup = () => { - const maybeFreeable = localVoiceTtsProvider as unknown as { free?: () => void } | null; + const maybeFreeable = localSpeech.localVoiceTtsProvider as unknown as { free?: () => void } | null; if (typeof maybeFreeable?.free === "function") { maybeFreeable.free(); } - for (const engine of localSttEngines.values()) { + for (const engine of localSpeech.localSttEngines.values()) { engine.engine.free(); } }; return { - sttService, - ttsService, - dictationSttService, + sttService: openAiSpeech.sttService, + ttsService: openAiSpeech.ttsService, + dictationSttService: openAiSpeech.dictationSttService, cleanup, localModelConfig: localConfig ? { modelsDir: localConfig.modelsDir, - defaultModelIds: requiredLocalModelIds, + defaultModelIds: localSpeech.requiredLocalModelIds, } : null, }; diff --git a/packages/server/src/server/speech/speech-types.ts b/packages/server/src/server/speech/speech-types.ts index 459d48eb2..ff44f9126 100644 --- a/packages/server/src/server/speech/speech-types.ts +++ b/packages/server/src/server/speech/speech-types.ts @@ -1,10 +1,4 @@ -import type { - LocalSttModelId, - LocalTtsModelId, -} from "./providers/local/sherpa/model-catalog.js"; +import { z } from "zod"; -export type SpeechProviderId = "openai" | "local"; - -export const DEFAULT_LOCAL_STT_MODEL: LocalSttModelId = "parakeet-tdt-0.6b-v3-int8"; -export const DEFAULT_LOCAL_TTS_MODEL: LocalTtsModelId = "pocket-tts-onnx-int8"; -export const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe"; +export const SpeechProviderIdSchema = z.enum(["openai", "local"]); +export type SpeechProviderId = z.infer;