mirror of
https://github.com/getpaseo/paseo.git
synced 2026-07-29 12:01:31 +00:00
speech: move defaults and runtime into provider modules
This commit is contained in:
Mohamed Boudra
@@ -41,8 +41,8 @@ function parseListenString(listen: string): ListenTarget {
|
||||
|
||||
import { VoiceAssistantWebSocketServer } from "./websocket-server.js";
|
||||
import { DownloadTokenStore } from "./file-download/token-store.js";
|
||||
import type { STTConfig } from "./speech/providers/openai/stt.js";
|
||||
import type { TTSConfig } from "./speech/providers/openai/tts.js";
|
||||
import type { OpenAiSpeechProviderConfig } from "./speech/providers/openai/config.js";
|
||||
import type { LocalSpeechProviderConfig } from "./speech/providers/local/config.js";
|
||||
import { initializeSpeechRuntime } from "./speech/speech-runtime.js";
|
||||
import { AgentManager } from "./agent/agent-manager.js";
|
||||
import { AgentStorage } from "./agent/agent-storage.js";
|
||||
@@ -101,17 +101,8 @@ function resolveVoiceMcpBridgeCommand(logger: Logger): { command: string; baseAr
|
||||
return resolved;
|
||||
}
|
||||
|
||||
export type PaseoOpenAIConfig = {
|
||||
apiKey?: string;
|
||||
stt?: Partial<STTConfig> & { apiKey?: string };
|
||||
tts?: Partial<TTSConfig> & { apiKey?: string };
|
||||
realtimeTranscriptionModel?: string;
|
||||
};
|
||||
|
||||
export type PaseoLocalSpeechConfig = {
|
||||
modelsDir: string;
|
||||
autoDownload?: boolean;
|
||||
};
|
||||
export type PaseoOpenAIConfig = OpenAiSpeechProviderConfig;
|
||||
export type PaseoLocalSpeechConfig = LocalSpeechProviderConfig;
|
||||
|
||||
export type PaseoSpeechConfig = {
|
||||
dictationSttProvider?: "openai" | "local";
|
||||
|
||||
176
packages/server/src/server/speech/providers/local/config.ts
Normal file
176
packages/server/src/server/speech/providers/local/config.ts
Normal file
@@ -0,0 +1,176 @@
|
||||
import path from "node:path";
|
||||
|
||||
import { z } from "zod";
|
||||
|
||||
import type { PersistedConfig } from "../../../persisted-config.js";
|
||||
import type { RequestedSpeechProviders } from "../../speech-types.js";
|
||||
import {
|
||||
DEFAULT_LOCAL_STT_MODEL,
|
||||
DEFAULT_LOCAL_TTS_MODEL,
|
||||
LocalSttModelIdSchema,
|
||||
LocalTtsModelIdSchema,
|
||||
type LocalSpeechModelId,
|
||||
type LocalSttModelId,
|
||||
type LocalTtsModelId,
|
||||
} from "./sherpa/model-catalog.js";
|
||||
|
||||
export type LocalSpeechProviderConfig = {
|
||||
modelsDir: string;
|
||||
autoDownload?: boolean;
|
||||
};
|
||||
|
||||
export type ResolvedLocalSpeechConfig = {
|
||||
local: LocalSpeechProviderConfig | undefined;
|
||||
dictationLocalSttModel: LocalSttModelId;
|
||||
voiceLocalSttModel: LocalSttModelId;
|
||||
voiceLocalTtsModel: LocalTtsModelId;
|
||||
voiceLocalTtsSpeakerId?: number;
|
||||
voiceLocalTtsSpeed?: number;
|
||||
};
|
||||
|
||||
export type { LocalSpeechModelId, LocalSttModelId, LocalTtsModelId };
|
||||
|
||||
const DEFAULT_LOCAL_MODELS_SUBDIR = path.join("models", "local-speech");
|
||||
|
||||
const OptionalBooleanFlagSchema = z.preprocess((value) => {
|
||||
if (typeof value === "boolean") {
|
||||
return value;
|
||||
}
|
||||
if (typeof value !== "string") {
|
||||
return value;
|
||||
}
|
||||
const normalized = value.trim().toLowerCase();
|
||||
if (normalized === "1" || normalized === "true" || normalized === "yes") {
|
||||
return true;
|
||||
}
|
||||
if (normalized === "0" || normalized === "false" || normalized === "no") {
|
||||
return false;
|
||||
}
|
||||
return undefined;
|
||||
}, z.boolean().optional());
|
||||
|
||||
const OptionalFiniteNumberSchema = z.preprocess((value) => {
|
||||
if (typeof value === "number") {
|
||||
return Number.isFinite(value) ? value : undefined;
|
||||
}
|
||||
if (typeof value !== "string") {
|
||||
return value;
|
||||
}
|
||||
const parsed = Number.parseFloat(value);
|
||||
return Number.isFinite(parsed) ? parsed : undefined;
|
||||
}, z.number().optional());
|
||||
|
||||
const OptionalIntegerSchema = z.preprocess((value) => {
|
||||
if (typeof value === "number") {
|
||||
return Number.isInteger(value) ? value : undefined;
|
||||
}
|
||||
if (typeof value !== "string") {
|
||||
return value;
|
||||
}
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
return Number.isFinite(parsed) ? parsed : undefined;
|
||||
}, z.number().int().optional());
|
||||
|
||||
const LocalSpeechResolutionSchema = z.object({
|
||||
includeProviderConfig: z.boolean(),
|
||||
modelsDir: z.string().trim().min(1),
|
||||
autoDownload: OptionalBooleanFlagSchema.default(true),
|
||||
dictationLocalSttModel: LocalSttModelIdSchema.default(DEFAULT_LOCAL_STT_MODEL),
|
||||
voiceLocalSttModel: LocalSttModelIdSchema.default(DEFAULT_LOCAL_STT_MODEL),
|
||||
voiceLocalTtsModel: LocalTtsModelIdSchema.default(DEFAULT_LOCAL_TTS_MODEL),
|
||||
voiceLocalTtsSpeakerId: OptionalIntegerSchema,
|
||||
voiceLocalTtsSpeed: OptionalFiniteNumberSchema,
|
||||
});
|
||||
|
||||
function persistedLocalFeatureModel(
|
||||
provider: RequestedSpeechProviders[keyof RequestedSpeechProviders],
|
||||
model: string | undefined
|
||||
): string | undefined {
|
||||
if (provider !== "local") {
|
||||
return undefined;
|
||||
}
|
||||
return model;
|
||||
}
|
||||
|
||||
function shouldIncludeLocalProviderConfig(params: {
|
||||
providers: RequestedSpeechProviders;
|
||||
env: NodeJS.ProcessEnv;
|
||||
persisted: PersistedConfig;
|
||||
}): boolean {
|
||||
const localRequestedByFeature =
|
||||
params.providers.dictationSttProvider === "local" ||
|
||||
params.providers.voiceSttProvider === "local" ||
|
||||
params.providers.voiceTtsProvider === "local";
|
||||
|
||||
return (
|
||||
localRequestedByFeature ||
|
||||
params.env.PASEO_LOCAL_MODELS_DIR !== undefined ||
|
||||
params.persisted.providers?.local !== undefined
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveLocalSpeechConfig(params: {
|
||||
paseoHome: string;
|
||||
env: NodeJS.ProcessEnv;
|
||||
persisted: PersistedConfig;
|
||||
providers: RequestedSpeechProviders;
|
||||
}): ResolvedLocalSpeechConfig {
|
||||
const includeProviderConfig = shouldIncludeLocalProviderConfig(params);
|
||||
|
||||
const parsed = LocalSpeechResolutionSchema.parse({
|
||||
includeProviderConfig,
|
||||
modelsDir:
|
||||
params.env.PASEO_LOCAL_MODELS_DIR ??
|
||||
params.persisted.providers?.local?.modelsDir ??
|
||||
path.join(params.paseoHome, DEFAULT_LOCAL_MODELS_SUBDIR),
|
||||
autoDownload:
|
||||
params.env.PASEO_LOCAL_AUTO_DOWNLOAD ??
|
||||
params.persisted.providers?.local?.autoDownload,
|
||||
dictationLocalSttModel:
|
||||
params.env.PASEO_DICTATION_LOCAL_STT_MODEL ??
|
||||
persistedLocalFeatureModel(
|
||||
params.providers.dictationSttProvider,
|
||||
params.persisted.features?.dictation?.stt?.model
|
||||
) ??
|
||||
DEFAULT_LOCAL_STT_MODEL,
|
||||
voiceLocalSttModel:
|
||||
params.env.PASEO_VOICE_LOCAL_STT_MODEL ??
|
||||
persistedLocalFeatureModel(
|
||||
params.providers.voiceSttProvider,
|
||||
params.persisted.features?.voiceMode?.stt?.model
|
||||
) ??
|
||||
DEFAULT_LOCAL_STT_MODEL,
|
||||
voiceLocalTtsModel:
|
||||
params.env.PASEO_VOICE_LOCAL_TTS_MODEL ??
|
||||
persistedLocalFeatureModel(
|
||||
params.providers.voiceTtsProvider,
|
||||
params.persisted.features?.voiceMode?.tts?.model
|
||||
) ??
|
||||
DEFAULT_LOCAL_TTS_MODEL,
|
||||
voiceLocalTtsSpeakerId:
|
||||
params.env.PASEO_VOICE_LOCAL_TTS_SPEAKER_ID ??
|
||||
params.persisted.features?.voiceMode?.tts?.speakerId,
|
||||
voiceLocalTtsSpeed:
|
||||
params.env.PASEO_VOICE_LOCAL_TTS_SPEED ??
|
||||
params.persisted.features?.voiceMode?.tts?.speed,
|
||||
});
|
||||
|
||||
return {
|
||||
local:
|
||||
parsed.includeProviderConfig
|
||||
? {
|
||||
modelsDir: parsed.modelsDir,
|
||||
autoDownload: parsed.autoDownload,
|
||||
}
|
||||
: undefined,
|
||||
dictationLocalSttModel: parsed.dictationLocalSttModel,
|
||||
voiceLocalSttModel: parsed.voiceLocalSttModel,
|
||||
voiceLocalTtsModel: parsed.voiceLocalTtsModel,
|
||||
...(parsed.voiceLocalTtsSpeakerId !== undefined
|
||||
? { voiceLocalTtsSpeakerId: parsed.voiceLocalTtsSpeakerId }
|
||||
: {}),
|
||||
...(parsed.voiceLocalTtsSpeed !== undefined
|
||||
? { voiceLocalTtsSpeed: parsed.voiceLocalTtsSpeed }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
372
packages/server/src/server/speech/providers/local/runtime.ts
Normal file
372
packages/server/src/server/speech/providers/local/runtime.ts
Normal file
@@ -0,0 +1,372 @@
|
||||
import type { Logger } from "pino";
|
||||
|
||||
import type { PaseoSpeechConfig } from "../../../bootstrap.js";
|
||||
import type { SpeechToTextProvider, TextToSpeechProvider } from "../../speech-provider.js";
|
||||
import type { RequestedSpeechProviders } from "../../speech-types.js";
|
||||
import { PocketTtsOnnxTTS } from "./pocket/pocket-tts-onnx.js";
|
||||
import {
|
||||
ensureSherpaOnnxModels,
|
||||
getSherpaOnnxModelDir,
|
||||
} from "./sherpa/model-downloader.js";
|
||||
import {
|
||||
DEFAULT_LOCAL_STT_MODEL,
|
||||
DEFAULT_LOCAL_TTS_MODEL,
|
||||
LocalSttModelIdSchema,
|
||||
LocalTtsModelIdSchema,
|
||||
type LocalSpeechModelId,
|
||||
type LocalSttModelId,
|
||||
type LocalTtsModelId,
|
||||
} from "./sherpa/model-catalog.js";
|
||||
import { SherpaOfflineRecognizerEngine } from "./sherpa/sherpa-offline-recognizer.js";
|
||||
import { SherpaOnlineRecognizerEngine } from "./sherpa/sherpa-online-recognizer.js";
|
||||
import { SherpaOnnxParakeetSTT } from "./sherpa/sherpa-parakeet-stt.js";
|
||||
import { SherpaParakeetRealtimeTranscriptionSession } from "./sherpa/sherpa-parakeet-realtime-session.js";
|
||||
import { SherpaRealtimeTranscriptionSession } from "./sherpa/sherpa-realtime-session.js";
|
||||
import { SherpaOnnxSTT } from "./sherpa/sherpa-stt.js";
|
||||
import { SherpaOnnxTTS } from "./sherpa/sherpa-tts.js";
|
||||
|
||||
type LocalSttEngine =
|
||||
| { kind: "offline"; engine: SherpaOfflineRecognizerEngine }
|
||||
| { kind: "online"; engine: SherpaOnlineRecognizerEngine };
|
||||
|
||||
type ResolvedLocalModels = {
|
||||
dictationLocalSttModel: LocalSttModelId;
|
||||
voiceLocalSttModel: LocalSttModelId;
|
||||
voiceLocalTtsModel: LocalTtsModelId;
|
||||
};
|
||||
|
||||
type LocalSpeechAvailability = {
|
||||
configured: boolean;
|
||||
modelsDir: string | null;
|
||||
autoDownload: boolean | null;
|
||||
};
|
||||
|
||||
export type InitializedLocalSpeech = {
|
||||
sttService: SpeechToTextProvider | null;
|
||||
ttsService: TextToSpeechProvider | null;
|
||||
dictationSttService: SpeechToTextProvider | null;
|
||||
localVoiceTtsProvider: TextToSpeechProvider | null;
|
||||
localModelConfig: {
|
||||
modelsDir: string;
|
||||
defaultModelIds: LocalSpeechModelId[];
|
||||
} | null;
|
||||
availability: LocalSpeechAvailability;
|
||||
cleanup: () => void;
|
||||
};
|
||||
|
||||
function buildModelDownloadHint(modelId: LocalSpeechModelId): string {
|
||||
return `Use 'paseo speech download --model ${modelId}' to download this model.`;
|
||||
}
|
||||
|
||||
function resolveConfiguredLocalModels(
|
||||
speechConfig: PaseoSpeechConfig | null
|
||||
): ResolvedLocalModels {
|
||||
return {
|
||||
dictationLocalSttModel: LocalSttModelIdSchema.parse(
|
||||
speechConfig?.dictationLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL
|
||||
),
|
||||
voiceLocalSttModel: LocalSttModelIdSchema.parse(
|
||||
speechConfig?.voiceLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL
|
||||
),
|
||||
voiceLocalTtsModel: LocalTtsModelIdSchema.parse(
|
||||
speechConfig?.voiceLocalTtsModel ?? DEFAULT_LOCAL_TTS_MODEL
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
export function getLocalSpeechAvailability(
|
||||
speechConfig: PaseoSpeechConfig | null
|
||||
): LocalSpeechAvailability {
|
||||
const localConfig = speechConfig?.local ?? null;
|
||||
return {
|
||||
configured: Boolean(localConfig),
|
||||
modelsDir: localConfig?.modelsDir ?? null,
|
||||
autoDownload: localConfig?.autoDownload ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function computeRequiredLocalModelIds(params: {
|
||||
providers: RequestedSpeechProviders;
|
||||
models: ResolvedLocalModels;
|
||||
}): LocalSpeechModelId[] {
|
||||
const ids = new Set<LocalSpeechModelId>();
|
||||
if (params.providers.dictationSttProvider === "local") {
|
||||
ids.add(params.models.dictationLocalSttModel);
|
||||
}
|
||||
if (params.providers.voiceSttProvider === "local") {
|
||||
ids.add(params.models.voiceLocalSttModel);
|
||||
}
|
||||
if (params.providers.voiceTtsProvider === "local") {
|
||||
ids.add(params.models.voiceLocalTtsModel);
|
||||
}
|
||||
return Array.from(ids);
|
||||
}
|
||||
|
||||
async function createLocalSttEngine(params: {
|
||||
modelId: LocalSttModelId;
|
||||
modelsDir: string;
|
||||
logger: Logger;
|
||||
}): Promise<LocalSttEngine> {
|
||||
const { modelId, modelsDir, logger } = params;
|
||||
|
||||
if (modelId === "parakeet-tdt-0.6b-v3-int8") {
|
||||
const modelDir = getSherpaOnnxModelDir(modelsDir, modelId);
|
||||
return {
|
||||
kind: "offline",
|
||||
engine: new SherpaOfflineRecognizerEngine(
|
||||
{
|
||||
model: {
|
||||
kind: "nemo_transducer",
|
||||
encoder: `${modelDir}/encoder.int8.onnx`,
|
||||
decoder: `${modelDir}/decoder.int8.onnx`,
|
||||
joiner: `${modelDir}/joiner.int8.onnx`,
|
||||
tokens: `${modelDir}/tokens.txt`,
|
||||
},
|
||||
numThreads: 2,
|
||||
debug: 0,
|
||||
},
|
||||
logger
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
if (modelId === "paraformer-bilingual-zh-en") {
|
||||
const modelDir = getSherpaOnnxModelDir(modelsDir, modelId);
|
||||
return {
|
||||
kind: "online",
|
||||
engine: new SherpaOnlineRecognizerEngine(
|
||||
{
|
||||
model: {
|
||||
kind: "paraformer",
|
||||
encoder: `${modelDir}/encoder.int8.onnx`,
|
||||
decoder: `${modelDir}/decoder.int8.onnx`,
|
||||
tokens: `${modelDir}/tokens.txt`,
|
||||
},
|
||||
numThreads: 1,
|
||||
debug: 0,
|
||||
},
|
||||
logger
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
if (modelId === "zipformer-bilingual-zh-en-2023-02-20") {
|
||||
const modelDir = getSherpaOnnxModelDir(modelsDir, modelId);
|
||||
return {
|
||||
kind: "online",
|
||||
engine: new SherpaOnlineRecognizerEngine(
|
||||
{
|
||||
model: {
|
||||
kind: "transducer",
|
||||
encoder: `${modelDir}/encoder-epoch-99-avg-1.onnx`,
|
||||
decoder: `${modelDir}/decoder-epoch-99-avg-1.onnx`,
|
||||
joiner: `${modelDir}/joiner-epoch-99-avg-1.onnx`,
|
||||
tokens: `${modelDir}/tokens.txt`,
|
||||
modelType: "zipformer",
|
||||
},
|
||||
numThreads: 1,
|
||||
debug: 0,
|
||||
},
|
||||
logger
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error(`Unsupported local STT model '${modelId}'`);
|
||||
}
|
||||
|
||||
export async function initializeLocalSpeechServices(params: {
|
||||
providers: RequestedSpeechProviders;
|
||||
speechConfig: PaseoSpeechConfig | null;
|
||||
logger: Logger;
|
||||
}): Promise<InitializedLocalSpeech> {
|
||||
const { providers, logger, speechConfig } = params;
|
||||
const localConfig = speechConfig?.local ?? null;
|
||||
const localModels = resolveConfiguredLocalModels(speechConfig);
|
||||
|
||||
let sttService: SpeechToTextProvider | null = null;
|
||||
let ttsService: TextToSpeechProvider | null = null;
|
||||
let dictationSttService: SpeechToTextProvider | null = null;
|
||||
let localVoiceTtsProvider: TextToSpeechProvider | null = null;
|
||||
|
||||
const requiredLocalModelIds = computeRequiredLocalModelIds({
|
||||
providers,
|
||||
models: localModels,
|
||||
});
|
||||
|
||||
if (requiredLocalModelIds.length > 0 && localConfig) {
|
||||
try {
|
||||
logger.info(
|
||||
{
|
||||
modelsDir: localConfig.modelsDir,
|
||||
modelIds: requiredLocalModelIds,
|
||||
autoDownload: localConfig.autoDownload ?? true,
|
||||
},
|
||||
"Ensuring local speech models"
|
||||
);
|
||||
await ensureSherpaOnnxModels({
|
||||
modelsDir: localConfig.modelsDir,
|
||||
modelIds: requiredLocalModelIds,
|
||||
autoDownload: localConfig.autoDownload ?? true,
|
||||
logger,
|
||||
});
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
{
|
||||
err,
|
||||
modelsDir: localConfig.modelsDir,
|
||||
modelIds: requiredLocalModelIds,
|
||||
autoDownload: localConfig.autoDownload ?? true,
|
||||
hint:
|
||||
"Use `paseo speech models` to inspect status and " +
|
||||
"`paseo speech download --model <MODEL_ID>` to fetch missing models.",
|
||||
},
|
||||
"Failed to ensure local speech models"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const localSttEngines = new Map<LocalSttModelId, LocalSttEngine>();
|
||||
|
||||
const getLocalSttEngine = async (
|
||||
modelId: LocalSttModelId
|
||||
): Promise<LocalSttEngine | null> => {
|
||||
const existing = localSttEngines.get(modelId);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
if (!localConfig) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const created = await createLocalSttEngine({
|
||||
modelId,
|
||||
modelsDir: localConfig.modelsDir,
|
||||
logger,
|
||||
});
|
||||
localSttEngines.set(modelId, created);
|
||||
return created;
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
{
|
||||
err,
|
||||
modelsDir: localConfig.modelsDir,
|
||||
modelId,
|
||||
hint: buildModelDownloadHint(modelId),
|
||||
},
|
||||
"Failed to initialize local STT engine (models missing or invalid)"
|
||||
);
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
if (providers.voiceSttProvider === "local") {
|
||||
if (!localConfig) {
|
||||
logger.warn(
|
||||
{ configured: false },
|
||||
"Local STT selected for voice but local provider config is missing; STT will be unavailable"
|
||||
);
|
||||
} else {
|
||||
const voiceEngine = await getLocalSttEngine(localModels.voiceLocalSttModel);
|
||||
if (voiceEngine?.kind === "offline") {
|
||||
sttService = new SherpaOnnxParakeetSTT({ engine: voiceEngine.engine }, logger);
|
||||
} else if (voiceEngine?.kind === "online") {
|
||||
sttService = new SherpaOnnxSTT({ engine: voiceEngine.engine }, logger);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (providers.dictationSttProvider === "local") {
|
||||
if (!localConfig) {
|
||||
logger.warn(
|
||||
{ configured: false },
|
||||
"Local STT selected for dictation but local provider config is missing; dictation STT will be unavailable"
|
||||
);
|
||||
} else {
|
||||
const dictationEngine = await getLocalSttEngine(localModels.dictationLocalSttModel);
|
||||
if (dictationEngine?.kind === "offline") {
|
||||
dictationSttService = {
|
||||
id: "local",
|
||||
createSession: () =>
|
||||
new SherpaParakeetRealtimeTranscriptionSession({ engine: dictationEngine.engine }),
|
||||
};
|
||||
} else if (dictationEngine?.kind === "online") {
|
||||
dictationSttService = {
|
||||
id: "local",
|
||||
createSession: () => new SherpaRealtimeTranscriptionSession({ engine: dictationEngine.engine }),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (providers.voiceTtsProvider === "local") {
|
||||
if (!localConfig) {
|
||||
logger.warn(
|
||||
{ configured: false },
|
||||
"Local TTS selected for voice but local provider config is missing; TTS will be unavailable"
|
||||
);
|
||||
} else {
|
||||
try {
|
||||
if (localModels.voiceLocalTtsModel === "pocket-tts-onnx-int8") {
|
||||
const modelDir = getSherpaOnnxModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel);
|
||||
localVoiceTtsProvider = await PocketTtsOnnxTTS.create(
|
||||
{
|
||||
modelDir,
|
||||
precision: "int8",
|
||||
targetChunkMs: 50,
|
||||
},
|
||||
logger
|
||||
);
|
||||
} else {
|
||||
const modelDir = getSherpaOnnxModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel);
|
||||
localVoiceTtsProvider = new SherpaOnnxTTS(
|
||||
{
|
||||
preset: localModels.voiceLocalTtsModel,
|
||||
modelDir,
|
||||
speakerId: speechConfig?.voiceLocalTtsSpeakerId,
|
||||
speed: speechConfig?.voiceLocalTtsSpeed,
|
||||
},
|
||||
logger
|
||||
);
|
||||
}
|
||||
ttsService = localVoiceTtsProvider;
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
{
|
||||
err,
|
||||
modelsDir: localConfig.modelsDir,
|
||||
modelId: localModels.voiceLocalTtsModel,
|
||||
hint: buildModelDownloadHint(localModels.voiceLocalTtsModel),
|
||||
},
|
||||
"Failed to initialize local TTS engine (models missing or invalid)"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const cleanup = () => {
|
||||
const maybeFreeable = localVoiceTtsProvider as unknown as { free?: () => void } | null;
|
||||
if (typeof maybeFreeable?.free === "function") {
|
||||
maybeFreeable.free();
|
||||
}
|
||||
for (const engine of localSttEngines.values()) {
|
||||
engine.engine.free();
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
sttService,
|
||||
ttsService,
|
||||
dictationSttService,
|
||||
localVoiceTtsProvider,
|
||||
localModelConfig:
|
||||
localConfig
|
||||
? {
|
||||
modelsDir: localConfig.modelsDir,
|
||||
defaultModelIds: requiredLocalModelIds,
|
||||
}
|
||||
: null,
|
||||
availability: getLocalSpeechAvailability(speechConfig),
|
||||
cleanup,
|
||||
};
|
||||
}
|
||||
@@ -115,6 +115,7 @@ export const SHERPA_ONNX_MODEL_CATALOG = {
|
||||
} as const satisfies Record<string, SherpaOnnxCatalogEntry>;
|
||||
|
||||
export type SherpaOnnxModelId = keyof typeof SHERPA_ONNX_MODEL_CATALOG;
|
||||
export type LocalSpeechModelId = SherpaOnnxModelId;
|
||||
|
||||
type ModelIdByKind<K extends SherpaOnnxModelKind> = {
|
||||
[Id in SherpaOnnxModelId]: (typeof SHERPA_ONNX_MODEL_CATALOG)[Id]["kind"] extends K
|
||||
|
||||
132
packages/server/src/server/speech/providers/openai/config.ts
Normal file
132
packages/server/src/server/speech/providers/openai/config.ts
Normal file
@@ -0,0 +1,132 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import type { PersistedConfig } from "../../../persisted-config.js";
|
||||
import type { RequestedSpeechProviders } from "../../speech-types.js";
|
||||
import type { STTConfig } from "./stt.js";
|
||||
import type { TTSConfig } from "./tts.js";
|
||||
|
||||
export const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
|
||||
export const DEFAULT_OPENAI_TTS_MODEL = "tts-1";
|
||||
|
||||
export type OpenAiSpeechProviderConfig = {
|
||||
apiKey?: string;
|
||||
stt?: Partial<STTConfig> & { apiKey?: string };
|
||||
tts?: Partial<TTSConfig> & { apiKey?: string };
|
||||
realtimeTranscriptionModel?: string;
|
||||
};
|
||||
|
||||
const OpenAiTtsVoiceSchema = z.enum([
|
||||
"alloy",
|
||||
"echo",
|
||||
"fable",
|
||||
"onyx",
|
||||
"nova",
|
||||
"shimmer",
|
||||
]);
|
||||
|
||||
const OpenAiTtsModelSchema = z.enum(["tts-1", "tts-1-hd"]);
|
||||
|
||||
const OptionalFiniteNumberSchema = z.preprocess((value) => {
|
||||
if (typeof value === "number") {
|
||||
return Number.isFinite(value) ? value : undefined;
|
||||
}
|
||||
if (typeof value !== "string") {
|
||||
return value;
|
||||
}
|
||||
const parsed = Number.parseFloat(value);
|
||||
return Number.isFinite(parsed) ? parsed : undefined;
|
||||
}, z.number().optional());
|
||||
|
||||
const OptionalTrimmedStringSchema = z.preprocess((value) => {
|
||||
if (typeof value !== "string") {
|
||||
return value;
|
||||
}
|
||||
const trimmed = value.trim();
|
||||
return trimmed.length > 0 ? trimmed : undefined;
|
||||
}, z.string().optional());
|
||||
|
||||
const OpenAiSpeechResolutionSchema = z.object({
|
||||
apiKey: OptionalTrimmedStringSchema,
|
||||
sttConfidenceThreshold: OptionalFiniteNumberSchema,
|
||||
sttModel: OptionalTrimmedStringSchema,
|
||||
ttsVoice: z.preprocess((value) => {
|
||||
if (typeof value !== "string") {
|
||||
return value;
|
||||
}
|
||||
const normalized = value.trim().toLowerCase();
|
||||
return normalized.length > 0 ? normalized : undefined;
|
||||
}, OpenAiTtsVoiceSchema.default("alloy")),
|
||||
ttsModel: z.preprocess((value) => {
|
||||
if (typeof value !== "string") {
|
||||
return value;
|
||||
}
|
||||
const normalized = value.trim().toLowerCase();
|
||||
return normalized.length > 0 ? normalized : undefined;
|
||||
}, OpenAiTtsModelSchema.default(DEFAULT_OPENAI_TTS_MODEL)),
|
||||
realtimeTranscriptionModel: OptionalTrimmedStringSchema.default(
|
||||
DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL
|
||||
),
|
||||
});
|
||||
|
||||
export function resolveOpenAiSpeechConfig(params: {
|
||||
env: NodeJS.ProcessEnv;
|
||||
persisted: PersistedConfig;
|
||||
providers: RequestedSpeechProviders;
|
||||
}): OpenAiSpeechProviderConfig | undefined {
|
||||
const parsed = OpenAiSpeechResolutionSchema.parse({
|
||||
apiKey: params.env.OPENAI_API_KEY ?? params.persisted.providers?.openai?.apiKey,
|
||||
sttConfidenceThreshold:
|
||||
params.env.STT_CONFIDENCE_THRESHOLD ??
|
||||
params.persisted.features?.dictation?.stt?.confidenceThreshold,
|
||||
sttModel:
|
||||
params.env.STT_MODEL ??
|
||||
(params.providers.voiceSttProvider === "openai"
|
||||
? params.persisted.features?.voiceMode?.stt?.model
|
||||
: undefined) ??
|
||||
(params.providers.dictationSttProvider === "openai"
|
||||
? params.persisted.features?.dictation?.stt?.model
|
||||
: undefined),
|
||||
ttsVoice:
|
||||
params.env.TTS_VOICE ??
|
||||
(params.providers.voiceTtsProvider === "openai"
|
||||
? params.persisted.features?.voiceMode?.tts?.voice
|
||||
: undefined) ??
|
||||
"alloy",
|
||||
ttsModel:
|
||||
params.env.TTS_MODEL ??
|
||||
(params.providers.voiceTtsProvider === "openai"
|
||||
? params.persisted.features?.voiceMode?.tts?.model
|
||||
: undefined) ??
|
||||
DEFAULT_OPENAI_TTS_MODEL,
|
||||
realtimeTranscriptionModel:
|
||||
params.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ??
|
||||
(params.providers.dictationSttProvider === "openai"
|
||||
? params.persisted.features?.dictation?.stt?.model
|
||||
: undefined) ??
|
||||
DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
|
||||
});
|
||||
|
||||
if (!parsed.apiKey) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return {
|
||||
apiKey: parsed.apiKey,
|
||||
stt: {
|
||||
apiKey: parsed.apiKey,
|
||||
...(parsed.sttConfidenceThreshold !== undefined
|
||||
? { confidenceThreshold: parsed.sttConfidenceThreshold }
|
||||
: {}),
|
||||
...(parsed.sttModel
|
||||
? { model: parsed.sttModel }
|
||||
: {}),
|
||||
},
|
||||
tts: {
|
||||
apiKey: parsed.apiKey,
|
||||
voice: parsed.ttsVoice,
|
||||
model: parsed.ttsModel,
|
||||
responseFormat: "pcm",
|
||||
},
|
||||
realtimeTranscriptionModel: parsed.realtimeTranscriptionModel,
|
||||
};
|
||||
}
|
||||
167
packages/server/src/server/speech/providers/openai/runtime.ts
Normal file
167
packages/server/src/server/speech/providers/openai/runtime.ts
Normal file
@@ -0,0 +1,167 @@
|
||||
import type { Logger } from "pino";
|
||||
|
||||
import type { SpeechToTextProvider, TextToSpeechProvider } from "../../speech-provider.js";
|
||||
import type { RequestedSpeechProviders } from "../../speech-types.js";
|
||||
import {
|
||||
DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
|
||||
DEFAULT_OPENAI_TTS_MODEL,
|
||||
type OpenAiSpeechProviderConfig,
|
||||
} from "./config.js";
|
||||
import { OpenAIRealtimeTranscriptionSession } from "./realtime-transcription-session.js";
|
||||
import { OpenAISTT } from "./stt.js";
|
||||
import { OpenAITTS } from "./tts.js";
|
||||
|
||||
type OpenAiCredentialState = {
|
||||
openaiSttApiKey: string | undefined;
|
||||
openaiTtsApiKey: string | undefined;
|
||||
openaiDictationApiKey: string | undefined;
|
||||
};
|
||||
|
||||
export type OpenAiSpeechAvailability = {
|
||||
stt: boolean;
|
||||
tts: boolean;
|
||||
dictationStt: boolean;
|
||||
};
|
||||
|
||||
export type SpeechServices = {
|
||||
sttService: SpeechToTextProvider | null;
|
||||
ttsService: TextToSpeechProvider | null;
|
||||
dictationSttService: SpeechToTextProvider | null;
|
||||
};
|
||||
|
||||
function resolveOpenAiCredentials(
|
||||
openaiConfig: OpenAiSpeechProviderConfig | undefined
|
||||
): OpenAiCredentialState {
|
||||
const openaiApiKey = openaiConfig?.apiKey;
|
||||
return {
|
||||
openaiSttApiKey: openaiConfig?.stt?.apiKey ?? openaiApiKey,
|
||||
openaiTtsApiKey: openaiConfig?.tts?.apiKey ?? openaiApiKey,
|
||||
openaiDictationApiKey: openaiApiKey,
|
||||
};
|
||||
}
|
||||
|
||||
export function getOpenAiSpeechAvailability(
|
||||
openaiConfig: OpenAiSpeechProviderConfig | undefined
|
||||
): OpenAiSpeechAvailability {
|
||||
const credentials = resolveOpenAiCredentials(openaiConfig);
|
||||
return {
|
||||
stt: Boolean(credentials.openaiSttApiKey),
|
||||
tts: Boolean(credentials.openaiTtsApiKey),
|
||||
dictationStt: Boolean(credentials.openaiDictationApiKey),
|
||||
};
|
||||
}
|
||||
|
||||
export function validateOpenAiCredentialRequirements(params: {
|
||||
providers: RequestedSpeechProviders;
|
||||
openaiConfig: OpenAiSpeechProviderConfig | undefined;
|
||||
logger: Logger;
|
||||
}): void {
|
||||
const { providers, logger, openaiConfig } = params;
|
||||
const openAiCredentials = resolveOpenAiCredentials(openaiConfig);
|
||||
|
||||
const missingOpenAiCredentialsFor: string[] = [];
|
||||
if (providers.voiceSttProvider === "openai" && !openAiCredentials.openaiSttApiKey) {
|
||||
missingOpenAiCredentialsFor.push("voice.stt");
|
||||
}
|
||||
if (providers.voiceTtsProvider === "openai" && !openAiCredentials.openaiTtsApiKey) {
|
||||
missingOpenAiCredentialsFor.push("voice.tts");
|
||||
}
|
||||
if (providers.dictationSttProvider === "openai" && !openAiCredentials.openaiDictationApiKey) {
|
||||
missingOpenAiCredentialsFor.push("dictation.stt");
|
||||
}
|
||||
|
||||
if (missingOpenAiCredentialsFor.length > 0) {
|
||||
logger.error(
|
||||
{
|
||||
requestedProviders: {
|
||||
dictationStt: providers.dictationSttProvider,
|
||||
voiceStt: providers.voiceSttProvider,
|
||||
voiceTts: providers.voiceTtsProvider,
|
||||
},
|
||||
missingOpenAiCredentialsFor,
|
||||
},
|
||||
"Invalid speech configuration: OpenAI provider selected but credentials are missing"
|
||||
);
|
||||
throw new Error(
|
||||
`Missing OpenAI credentials for configured speech features: ${missingOpenAiCredentialsFor.join(", ")}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function initializeOpenAiSpeechServices(params: {
|
||||
providers: RequestedSpeechProviders;
|
||||
openaiConfig: OpenAiSpeechProviderConfig | undefined;
|
||||
existing: SpeechServices;
|
||||
logger: Logger;
|
||||
}): SpeechServices {
|
||||
const { providers, openaiConfig, existing, logger } = params;
|
||||
const openAiCredentials = resolveOpenAiCredentials(openaiConfig);
|
||||
|
||||
let sttService = existing.sttService;
|
||||
let ttsService = existing.ttsService;
|
||||
let dictationSttService = existing.dictationSttService;
|
||||
|
||||
const needsOpenAiStt = !sttService && providers.voiceSttProvider === "openai";
|
||||
const needsOpenAiTts = !ttsService && providers.voiceTtsProvider === "openai";
|
||||
const needsOpenAiDictation = !dictationSttService && providers.dictationSttProvider === "openai";
|
||||
|
||||
if (
|
||||
(needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) &&
|
||||
(openAiCredentials.openaiSttApiKey ||
|
||||
openAiCredentials.openaiTtsApiKey ||
|
||||
openAiCredentials.openaiDictationApiKey)
|
||||
) {
|
||||
logger.info("OpenAI speech provider initialized");
|
||||
|
||||
if (needsOpenAiStt && openAiCredentials.openaiSttApiKey) {
|
||||
const { apiKey: _sttApiKey, ...sttConfig } = openaiConfig?.stt ?? {};
|
||||
sttService = new OpenAISTT(
|
||||
{
|
||||
apiKey: openAiCredentials.openaiSttApiKey,
|
||||
...sttConfig,
|
||||
},
|
||||
logger
|
||||
);
|
||||
}
|
||||
|
||||
if (needsOpenAiTts && openAiCredentials.openaiTtsApiKey) {
|
||||
const { apiKey: _ttsApiKey, ...ttsConfig } = openaiConfig?.tts ?? {};
|
||||
ttsService = new OpenAITTS(
|
||||
{
|
||||
apiKey: openAiCredentials.openaiTtsApiKey,
|
||||
voice: "alloy",
|
||||
model: DEFAULT_OPENAI_TTS_MODEL,
|
||||
responseFormat: "pcm",
|
||||
...ttsConfig,
|
||||
},
|
||||
logger
|
||||
);
|
||||
}
|
||||
|
||||
const dictationApiKey = openAiCredentials.openaiDictationApiKey;
|
||||
if (needsOpenAiDictation && dictationApiKey) {
|
||||
dictationSttService = {
|
||||
id: "openai",
|
||||
createSession: ({ logger: sessionLogger, language, prompt }) =>
|
||||
new OpenAIRealtimeTranscriptionSession({
|
||||
apiKey: dictationApiKey,
|
||||
logger: sessionLogger,
|
||||
transcriptionModel:
|
||||
openaiConfig?.realtimeTranscriptionModel ??
|
||||
DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
|
||||
...(language ? { language } : {}),
|
||||
...(prompt ? { prompt } : {}),
|
||||
turnDetection: null,
|
||||
}),
|
||||
};
|
||||
}
|
||||
} else if (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) {
|
||||
logger.warn("OpenAI speech providers are configured but credentials are missing");
|
||||
}
|
||||
|
||||
return {
|
||||
sttService,
|
||||
ttsService,
|
||||
dictationSttService,
|
||||
};
|
||||
}
|
||||
@@ -1,33 +1,13 @@
|
||||
import path from "node:path";
|
||||
|
||||
import { z } from "zod";
|
||||
|
||||
import type { PersistedConfig } from "../persisted-config.js";
|
||||
import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js";
|
||||
import { resolveLocalSpeechConfig } from "./providers/local/config.js";
|
||||
import { resolveOpenAiSpeechConfig } from "./providers/openai/config.js";
|
||||
import {
|
||||
LocalSttModelIdSchema,
|
||||
LocalTtsModelIdSchema,
|
||||
} from "./providers/local/sherpa/model-catalog.js";
|
||||
import {
|
||||
DEFAULT_LOCAL_STT_MODEL,
|
||||
DEFAULT_LOCAL_TTS_MODEL,
|
||||
DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
|
||||
DEFAULT_OPENAI_TTS_MODEL,
|
||||
} from "./speech-defaults.js";
|
||||
import { SpeechProviderIdSchema } from "./speech-types.js";
|
||||
|
||||
const DEFAULT_LOCAL_MODELS_SUBDIR = path.join("models", "local-speech");
|
||||
|
||||
const OpenAiTtsVoiceSchema = z.enum([
|
||||
"alloy",
|
||||
"echo",
|
||||
"fable",
|
||||
"onyx",
|
||||
"nova",
|
||||
"shimmer",
|
||||
]);
|
||||
|
||||
const OpenAiTtsModelSchema = z.enum(["tts-1", "tts-1-hd"]);
|
||||
SpeechProviderIdSchema,
|
||||
type RequestedSpeechProviders,
|
||||
} from "./speech-types.js";
|
||||
|
||||
const OptionalSpeechProviderSchema = z.preprocess((value) => {
|
||||
if (typeof value !== "string") {
|
||||
@@ -37,136 +17,31 @@ const OptionalSpeechProviderSchema = z.preprocess((value) => {
|
||||
return normalized.length > 0 ? normalized : undefined;
|
||||
}, SpeechProviderIdSchema.optional());
|
||||
|
||||
const OptionalBooleanFlagSchema = z.preprocess((value) => {
|
||||
if (typeof value === "boolean") {
|
||||
return value;
|
||||
}
|
||||
if (typeof value !== "string") {
|
||||
return value;
|
||||
}
|
||||
const normalized = value.trim().toLowerCase();
|
||||
if (normalized === "1" || normalized === "true" || normalized === "yes") {
|
||||
return true;
|
||||
}
|
||||
if (normalized === "0" || normalized === "false" || normalized === "no") {
|
||||
return false;
|
||||
}
|
||||
return undefined;
|
||||
}, z.boolean().optional());
|
||||
const RequestedSpeechProvidersSchema = z.object({
|
||||
dictationSttProvider: OptionalSpeechProviderSchema.default("local"),
|
||||
voiceSttProvider: OptionalSpeechProviderSchema.default("local"),
|
||||
voiceTtsProvider: OptionalSpeechProviderSchema.default("local"),
|
||||
});
|
||||
|
||||
const OptionalFiniteNumberSchema = z.preprocess((value) => {
|
||||
if (typeof value === "number") {
|
||||
return Number.isFinite(value) ? value : undefined;
|
||||
}
|
||||
if (typeof value !== "string") {
|
||||
return value;
|
||||
}
|
||||
const parsed = Number.parseFloat(value);
|
||||
return Number.isFinite(parsed) ? parsed : undefined;
|
||||
}, z.number().optional());
|
||||
|
||||
const OptionalIntegerSchema = z.preprocess((value) => {
|
||||
if (typeof value === "number") {
|
||||
return Number.isInteger(value) ? value : undefined;
|
||||
}
|
||||
if (typeof value !== "string") {
|
||||
return value;
|
||||
}
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
return Number.isFinite(parsed) ? parsed : undefined;
|
||||
}, z.number().int().optional());
|
||||
|
||||
const OptionalTrimmedStringSchema = z.preprocess((value) => {
|
||||
if (typeof value !== "string") {
|
||||
return value;
|
||||
}
|
||||
const trimmed = value.trim();
|
||||
return trimmed.length > 0 ? trimmed : undefined;
|
||||
}, z.string().optional());
|
||||
|
||||
const ResolvedSpeechConfigSchema = z
|
||||
.object({
|
||||
dictationSttProvider: OptionalSpeechProviderSchema.default("local"),
|
||||
voiceSttProvider: OptionalSpeechProviderSchema.default("local"),
|
||||
voiceTtsProvider: OptionalSpeechProviderSchema.default("local"),
|
||||
localModelsDir: z.string().trim().min(1),
|
||||
localAutoDownload: OptionalBooleanFlagSchema.default(true),
|
||||
dictationLocalSttModel: LocalSttModelIdSchema.default(DEFAULT_LOCAL_STT_MODEL),
|
||||
voiceLocalSttModel: LocalSttModelIdSchema.default(DEFAULT_LOCAL_STT_MODEL),
|
||||
voiceLocalTtsModel: LocalTtsModelIdSchema.default(DEFAULT_LOCAL_TTS_MODEL),
|
||||
voiceLocalTtsSpeakerId: OptionalIntegerSchema,
|
||||
voiceLocalTtsSpeed: OptionalFiniteNumberSchema,
|
||||
anyLocalRequested: z.boolean(),
|
||||
openaiApiKey: OptionalTrimmedStringSchema,
|
||||
openaiSttConfidenceThreshold: OptionalFiniteNumberSchema,
|
||||
openaiSttModel: OptionalTrimmedStringSchema,
|
||||
openaiTtsVoice: z.preprocess((value) => {
|
||||
if (typeof value !== "string") {
|
||||
return value;
|
||||
}
|
||||
const normalized = value.trim().toLowerCase();
|
||||
return normalized.length > 0 ? normalized : undefined;
|
||||
}, OpenAiTtsVoiceSchema.default("alloy")),
|
||||
openaiTtsModel: z.preprocess((value) => {
|
||||
if (typeof value !== "string") {
|
||||
return value;
|
||||
}
|
||||
const normalized = value.trim().toLowerCase();
|
||||
return normalized.length > 0 ? normalized : undefined;
|
||||
}, OpenAiTtsModelSchema.default(DEFAULT_OPENAI_TTS_MODEL)),
|
||||
openaiRealtimeTranscriptionModel: OptionalTrimmedStringSchema.default(
|
||||
DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL
|
||||
),
|
||||
})
|
||||
.transform((input): { openai: PaseoOpenAIConfig | undefined; speech: PaseoSpeechConfig } => {
|
||||
const openai = input.openaiApiKey
|
||||
? {
|
||||
apiKey: input.openaiApiKey,
|
||||
stt: {
|
||||
apiKey: input.openaiApiKey,
|
||||
...(input.openaiSttConfidenceThreshold !== undefined
|
||||
? { confidenceThreshold: input.openaiSttConfidenceThreshold }
|
||||
: {}),
|
||||
...(input.openaiSttModel
|
||||
? { model: input.openaiSttModel }
|
||||
: {}),
|
||||
},
|
||||
tts: {
|
||||
apiKey: input.openaiApiKey,
|
||||
voice: input.openaiTtsVoice,
|
||||
model: input.openaiTtsModel,
|
||||
responseFormat: "pcm" as const,
|
||||
},
|
||||
realtimeTranscriptionModel: input.openaiRealtimeTranscriptionModel,
|
||||
}
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
openai,
|
||||
speech: {
|
||||
dictationSttProvider: input.dictationSttProvider,
|
||||
voiceSttProvider: input.voiceSttProvider,
|
||||
voiceTtsProvider: input.voiceTtsProvider,
|
||||
...(input.anyLocalRequested
|
||||
? {
|
||||
local: {
|
||||
modelsDir: input.localModelsDir,
|
||||
autoDownload: input.localAutoDownload,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
dictationLocalSttModel: input.dictationLocalSttModel,
|
||||
voiceLocalSttModel: input.voiceLocalSttModel,
|
||||
voiceLocalTtsModel: input.voiceLocalTtsModel,
|
||||
...(input.voiceLocalTtsSpeakerId !== undefined
|
||||
? { voiceLocalTtsSpeakerId: input.voiceLocalTtsSpeakerId }
|
||||
: {}),
|
||||
...(input.voiceLocalTtsSpeed !== undefined
|
||||
? { voiceLocalTtsSpeed: input.voiceLocalTtsSpeed }
|
||||
: {}),
|
||||
},
|
||||
};
|
||||
function resolveRequestedSpeechProviders(params: {
|
||||
env: NodeJS.ProcessEnv;
|
||||
persisted: PersistedConfig;
|
||||
}): RequestedSpeechProviders {
|
||||
return RequestedSpeechProvidersSchema.parse({
|
||||
dictationSttProvider:
|
||||
params.env.PASEO_DICTATION_STT_PROVIDER ??
|
||||
params.persisted.features?.dictation?.stt?.provider ??
|
||||
"local",
|
||||
voiceSttProvider:
|
||||
params.env.PASEO_VOICE_STT_PROVIDER ??
|
||||
params.persisted.features?.voiceMode?.stt?.provider ??
|
||||
"local",
|
||||
voiceTtsProvider:
|
||||
params.env.PASEO_VOICE_TTS_PROVIDER ??
|
||||
params.persisted.features?.voiceMode?.tts?.provider ??
|
||||
"local",
|
||||
});
|
||||
}
|
||||
|
||||
export function resolveSpeechConfig(params: {
|
||||
paseoHome: string;
|
||||
@@ -176,89 +51,42 @@ export function resolveSpeechConfig(params: {
|
||||
openai: PaseoOpenAIConfig | undefined;
|
||||
speech: PaseoSpeechConfig;
|
||||
} {
|
||||
const { paseoHome, env, persisted } = params;
|
||||
|
||||
const dictationSttProvider =
|
||||
env.PASEO_DICTATION_STT_PROVIDER
|
||||
?? persisted.features?.dictation?.stt?.provider
|
||||
?? "local";
|
||||
|
||||
const voiceSttProvider =
|
||||
env.PASEO_VOICE_STT_PROVIDER
|
||||
?? persisted.features?.voiceMode?.stt?.provider
|
||||
?? "local";
|
||||
|
||||
const voiceTtsProvider =
|
||||
env.PASEO_VOICE_TTS_PROVIDER
|
||||
?? persisted.features?.voiceMode?.tts?.provider
|
||||
?? "local";
|
||||
|
||||
const anyLocalRequested =
|
||||
dictationSttProvider === "local" ||
|
||||
voiceSttProvider === "local" ||
|
||||
voiceTtsProvider === "local" ||
|
||||
env.PASEO_LOCAL_MODELS_DIR !== undefined ||
|
||||
persisted.providers?.local !== undefined;
|
||||
|
||||
return ResolvedSpeechConfigSchema.parse({
|
||||
dictationSttProvider,
|
||||
voiceSttProvider,
|
||||
voiceTtsProvider,
|
||||
localModelsDir:
|
||||
env.PASEO_LOCAL_MODELS_DIR
|
||||
?? persisted.providers?.local?.modelsDir
|
||||
?? path.join(paseoHome, DEFAULT_LOCAL_MODELS_SUBDIR),
|
||||
localAutoDownload:
|
||||
env.PASEO_LOCAL_AUTO_DOWNLOAD
|
||||
?? persisted.providers?.local?.autoDownload,
|
||||
dictationLocalSttModel:
|
||||
env.PASEO_DICTATION_LOCAL_STT_MODEL
|
||||
?? persisted.features?.dictation?.stt?.model
|
||||
?? DEFAULT_LOCAL_STT_MODEL,
|
||||
voiceLocalSttModel:
|
||||
env.PASEO_VOICE_LOCAL_STT_MODEL
|
||||
?? persisted.features?.voiceMode?.stt?.model
|
||||
?? DEFAULT_LOCAL_STT_MODEL,
|
||||
voiceLocalTtsModel:
|
||||
env.PASEO_VOICE_LOCAL_TTS_MODEL
|
||||
?? persisted.features?.voiceMode?.tts?.model
|
||||
?? DEFAULT_LOCAL_TTS_MODEL,
|
||||
voiceLocalTtsSpeakerId:
|
||||
env.PASEO_VOICE_LOCAL_TTS_SPEAKER_ID
|
||||
?? persisted.features?.voiceMode?.tts?.speakerId,
|
||||
voiceLocalTtsSpeed:
|
||||
env.PASEO_VOICE_LOCAL_TTS_SPEED
|
||||
?? persisted.features?.voiceMode?.tts?.speed,
|
||||
anyLocalRequested,
|
||||
openaiApiKey: env.OPENAI_API_KEY ?? persisted.providers?.openai?.apiKey,
|
||||
openaiSttConfidenceThreshold:
|
||||
env.STT_CONFIDENCE_THRESHOLD
|
||||
?? persisted.features?.dictation?.stt?.confidenceThreshold,
|
||||
openaiSttModel:
|
||||
env.STT_MODEL
|
||||
?? (voiceSttProvider === "openai"
|
||||
? persisted.features?.voiceMode?.stt?.model
|
||||
: undefined)
|
||||
?? (dictationSttProvider === "openai"
|
||||
? persisted.features?.dictation?.stt?.model
|
||||
: undefined),
|
||||
openaiTtsVoice:
|
||||
env.TTS_VOICE
|
||||
?? (voiceTtsProvider === "openai"
|
||||
? persisted.features?.voiceMode?.tts?.voice
|
||||
: undefined)
|
||||
?? "alloy",
|
||||
openaiTtsModel:
|
||||
env.TTS_MODEL
|
||||
?? (voiceTtsProvider === "openai"
|
||||
? persisted.features?.voiceMode?.tts?.model
|
||||
: undefined)
|
||||
?? DEFAULT_OPENAI_TTS_MODEL,
|
||||
openaiRealtimeTranscriptionModel:
|
||||
env.OPENAI_REALTIME_TRANSCRIPTION_MODEL
|
||||
?? (dictationSttProvider === "openai"
|
||||
? persisted.features?.dictation?.stt?.model
|
||||
: undefined)
|
||||
?? DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
|
||||
const providers = resolveRequestedSpeechProviders({
|
||||
env: params.env,
|
||||
persisted: params.persisted,
|
||||
});
|
||||
|
||||
const local = resolveLocalSpeechConfig({
|
||||
paseoHome: params.paseoHome,
|
||||
env: params.env,
|
||||
persisted: params.persisted,
|
||||
providers,
|
||||
});
|
||||
|
||||
const openai = resolveOpenAiSpeechConfig({
|
||||
env: params.env,
|
||||
persisted: params.persisted,
|
||||
providers,
|
||||
});
|
||||
|
||||
return {
|
||||
openai,
|
||||
speech: {
|
||||
dictationSttProvider: providers.dictationSttProvider,
|
||||
voiceSttProvider: providers.voiceSttProvider,
|
||||
voiceTtsProvider: providers.voiceTtsProvider,
|
||||
...(local.local
|
||||
? { local: local.local }
|
||||
: {}),
|
||||
dictationLocalSttModel: local.dictationLocalSttModel,
|
||||
voiceLocalSttModel: local.voiceLocalSttModel,
|
||||
voiceLocalTtsModel: local.voiceLocalTtsModel,
|
||||
...(local.voiceLocalTtsSpeakerId !== undefined
|
||||
? { voiceLocalTtsSpeakerId: local.voiceLocalTtsSpeakerId }
|
||||
: {}),
|
||||
...(local.voiceLocalTtsSpeed !== undefined
|
||||
? { voiceLocalTtsSpeed: local.voiceLocalTtsSpeed }
|
||||
: {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
import {
|
||||
DEFAULT_LOCAL_STT_MODEL,
|
||||
DEFAULT_LOCAL_TTS_MODEL,
|
||||
} from "./providers/local/sherpa/model-catalog.js";
|
||||
|
||||
export { DEFAULT_LOCAL_STT_MODEL, DEFAULT_LOCAL_TTS_MODEL };
|
||||
|
||||
export const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
|
||||
export const DEFAULT_OPENAI_TTS_MODEL = "tts-1";
|
||||
@@ -2,71 +2,17 @@ import type { Logger } from "pino";
|
||||
|
||||
import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js";
|
||||
import {
|
||||
OpenAISTT,
|
||||
} from "./providers/openai/stt.js";
|
||||
getLocalSpeechAvailability,
|
||||
initializeLocalSpeechServices,
|
||||
} from "./providers/local/runtime.js";
|
||||
import type { LocalSpeechModelId } from "./providers/local/config.js";
|
||||
import {
|
||||
OpenAITTS,
|
||||
} from "./providers/openai/tts.js";
|
||||
import { OpenAIRealtimeTranscriptionSession } from "./providers/openai/realtime-transcription-session.js";
|
||||
getOpenAiSpeechAvailability,
|
||||
initializeOpenAiSpeechServices,
|
||||
validateOpenAiCredentialRequirements,
|
||||
} from "./providers/openai/runtime.js";
|
||||
import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech-provider.js";
|
||||
import { SherpaOnlineRecognizerEngine } from "./providers/local/sherpa/sherpa-online-recognizer.js";
|
||||
import { SherpaOfflineRecognizerEngine } from "./providers/local/sherpa/sherpa-offline-recognizer.js";
|
||||
import { SherpaOnnxSTT } from "./providers/local/sherpa/sherpa-stt.js";
|
||||
import { SherpaOnnxParakeetSTT } from "./providers/local/sherpa/sherpa-parakeet-stt.js";
|
||||
import { SherpaOnnxTTS } from "./providers/local/sherpa/sherpa-tts.js";
|
||||
import { SherpaRealtimeTranscriptionSession } from "./providers/local/sherpa/sherpa-realtime-session.js";
|
||||
import { SherpaParakeetRealtimeTranscriptionSession } from "./providers/local/sherpa/sherpa-parakeet-realtime-session.js";
|
||||
import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./providers/local/sherpa/model-downloader.js";
|
||||
import {
|
||||
LocalSttModelIdSchema,
|
||||
LocalTtsModelIdSchema,
|
||||
type LocalSttModelId,
|
||||
type LocalTtsModelId,
|
||||
type SherpaOnnxModelId,
|
||||
} from "./providers/local/sherpa/model-catalog.js";
|
||||
import { PocketTtsOnnxTTS } from "./providers/local/pocket/pocket-tts-onnx.js";
|
||||
import {
|
||||
DEFAULT_LOCAL_STT_MODEL,
|
||||
DEFAULT_LOCAL_TTS_MODEL,
|
||||
DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
|
||||
DEFAULT_OPENAI_TTS_MODEL,
|
||||
} from "./speech-defaults.js";
|
||||
import type { SpeechProviderId } from "./speech-types.js";
|
||||
|
||||
type LocalSttEngine =
|
||||
| { kind: "offline"; engine: SherpaOfflineRecognizerEngine }
|
||||
| { kind: "online"; engine: SherpaOnlineRecognizerEngine };
|
||||
|
||||
type RequestedSpeechProviders = {
|
||||
dictationSttProvider: SpeechProviderId;
|
||||
voiceSttProvider: SpeechProviderId;
|
||||
voiceTtsProvider: SpeechProviderId;
|
||||
};
|
||||
|
||||
type ResolvedLocalModels = {
|
||||
dictationLocalSttModel: LocalSttModelId;
|
||||
voiceLocalSttModel: LocalSttModelId;
|
||||
voiceLocalTtsModel: LocalTtsModelId;
|
||||
};
|
||||
|
||||
type OpenAiCredentialState = {
|
||||
openaiSttApiKey: string | undefined;
|
||||
openaiTtsApiKey: string | undefined;
|
||||
openaiDictationApiKey: string | undefined;
|
||||
};
|
||||
|
||||
type InitializedLocalSpeech = {
|
||||
sttService: SpeechToTextProvider | null;
|
||||
ttsService: TextToSpeechProvider | null;
|
||||
dictationSttService: SpeechToTextProvider | null;
|
||||
localVoiceTtsProvider: TextToSpeechProvider | null;
|
||||
localSttEngines: Map<LocalSttModelId, LocalSttEngine>;
|
||||
requiredLocalModelIds: SherpaOnnxModelId[];
|
||||
};
|
||||
|
||||
function buildModelDownloadHint(modelId: SherpaOnnxModelId): string {
|
||||
return `Use 'paseo speech download --model ${modelId}' to download this model.`;
|
||||
}
|
||||
import type { RequestedSpeechProviders } from "./speech-types.js";
|
||||
|
||||
function resolveRequestedSpeechProviders(
|
||||
speechConfig: PaseoSpeechConfig | null
|
||||
@@ -78,425 +24,6 @@ function resolveRequestedSpeechProviders(
|
||||
};
|
||||
}
|
||||
|
||||
function resolveConfiguredLocalModels(
|
||||
speechConfig: PaseoSpeechConfig | null
|
||||
): ResolvedLocalModels {
|
||||
return {
|
||||
dictationLocalSttModel: LocalSttModelIdSchema.parse(
|
||||
speechConfig?.dictationLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL
|
||||
),
|
||||
voiceLocalSttModel: LocalSttModelIdSchema.parse(
|
||||
speechConfig?.voiceLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL
|
||||
),
|
||||
voiceLocalTtsModel: LocalTtsModelIdSchema.parse(
|
||||
speechConfig?.voiceLocalTtsModel ?? DEFAULT_LOCAL_TTS_MODEL
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function computeRequiredLocalModelIds(params: {
|
||||
providers: RequestedSpeechProviders;
|
||||
models: {
|
||||
dictationLocalSttModel: SherpaOnnxModelId;
|
||||
voiceLocalSttModel: SherpaOnnxModelId;
|
||||
voiceLocalTtsModel: SherpaOnnxModelId;
|
||||
};
|
||||
}): SherpaOnnxModelId[] {
|
||||
const ids = new Set<SherpaOnnxModelId>();
|
||||
if (params.providers.dictationSttProvider === "local") {
|
||||
ids.add(params.models.dictationLocalSttModel);
|
||||
}
|
||||
if (params.providers.voiceSttProvider === "local") {
|
||||
ids.add(params.models.voiceLocalSttModel);
|
||||
}
|
||||
if (params.providers.voiceTtsProvider === "local") {
|
||||
ids.add(params.models.voiceLocalTtsModel);
|
||||
}
|
||||
return Array.from(ids);
|
||||
}
|
||||
|
||||
function resolveOpenAiCredentials(openaiConfig?: PaseoOpenAIConfig): OpenAiCredentialState {
|
||||
const openaiApiKey = openaiConfig?.apiKey;
|
||||
return {
|
||||
openaiSttApiKey: openaiConfig?.stt?.apiKey ?? openaiApiKey,
|
||||
openaiTtsApiKey: openaiConfig?.tts?.apiKey ?? openaiApiKey,
|
||||
openaiDictationApiKey: openaiApiKey,
|
||||
};
|
||||
}
|
||||
|
||||
function validateOpenAiCredentialRequirements(params: {
|
||||
providers: RequestedSpeechProviders;
|
||||
openAiCredentials: OpenAiCredentialState;
|
||||
logger: Logger;
|
||||
}): void {
|
||||
const { providers, openAiCredentials, logger } = params;
|
||||
const missingOpenAiCredentialsFor: string[] = [];
|
||||
if (providers.voiceSttProvider === "openai" && !openAiCredentials.openaiSttApiKey) {
|
||||
missingOpenAiCredentialsFor.push("voice.stt");
|
||||
}
|
||||
if (providers.voiceTtsProvider === "openai" && !openAiCredentials.openaiTtsApiKey) {
|
||||
missingOpenAiCredentialsFor.push("voice.tts");
|
||||
}
|
||||
if (providers.dictationSttProvider === "openai" && !openAiCredentials.openaiDictationApiKey) {
|
||||
missingOpenAiCredentialsFor.push("dictation.stt");
|
||||
}
|
||||
|
||||
if (missingOpenAiCredentialsFor.length > 0) {
|
||||
logger.error(
|
||||
{
|
||||
requestedProviders: {
|
||||
dictationStt: providers.dictationSttProvider,
|
||||
voiceStt: providers.voiceSttProvider,
|
||||
voiceTts: providers.voiceTtsProvider,
|
||||
},
|
||||
missingOpenAiCredentialsFor,
|
||||
},
|
||||
"Invalid speech configuration: OpenAI provider selected but credentials are missing"
|
||||
);
|
||||
throw new Error(
|
||||
`Missing OpenAI credentials for configured speech features: ${missingOpenAiCredentialsFor.join(", ")}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function createLocalSttEngine(params: {
|
||||
modelId: LocalSttModelId;
|
||||
modelsDir: string;
|
||||
logger: Logger;
|
||||
}): Promise<LocalSttEngine> {
|
||||
const { modelId, modelsDir, logger } = params;
|
||||
|
||||
if (modelId === "parakeet-tdt-0.6b-v3-int8") {
|
||||
const modelDir = getSherpaOnnxModelDir(modelsDir, modelId);
|
||||
return {
|
||||
kind: "offline",
|
||||
engine: new SherpaOfflineRecognizerEngine(
|
||||
{
|
||||
model: {
|
||||
kind: "nemo_transducer",
|
||||
encoder: `${modelDir}/encoder.int8.onnx`,
|
||||
decoder: `${modelDir}/decoder.int8.onnx`,
|
||||
joiner: `${modelDir}/joiner.int8.onnx`,
|
||||
tokens: `${modelDir}/tokens.txt`,
|
||||
},
|
||||
numThreads: 2,
|
||||
debug: 0,
|
||||
},
|
||||
logger
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
if (modelId === "paraformer-bilingual-zh-en") {
|
||||
const modelDir = getSherpaOnnxModelDir(modelsDir, modelId);
|
||||
return {
|
||||
kind: "online",
|
||||
engine: new SherpaOnlineRecognizerEngine(
|
||||
{
|
||||
model: {
|
||||
kind: "paraformer",
|
||||
encoder: `${modelDir}/encoder.int8.onnx`,
|
||||
decoder: `${modelDir}/decoder.int8.onnx`,
|
||||
tokens: `${modelDir}/tokens.txt`,
|
||||
},
|
||||
numThreads: 1,
|
||||
debug: 0,
|
||||
},
|
||||
logger
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
if (modelId === "zipformer-bilingual-zh-en-2023-02-20") {
|
||||
const modelDir = getSherpaOnnxModelDir(modelsDir, modelId);
|
||||
return {
|
||||
kind: "online",
|
||||
engine: new SherpaOnlineRecognizerEngine(
|
||||
{
|
||||
model: {
|
||||
kind: "transducer",
|
||||
encoder: `${modelDir}/encoder-epoch-99-avg-1.onnx`,
|
||||
decoder: `${modelDir}/decoder-epoch-99-avg-1.onnx`,
|
||||
joiner: `${modelDir}/joiner-epoch-99-avg-1.onnx`,
|
||||
tokens: `${modelDir}/tokens.txt`,
|
||||
modelType: "zipformer",
|
||||
},
|
||||
numThreads: 1,
|
||||
debug: 0,
|
||||
},
|
||||
logger
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error(`Unsupported local STT model '${modelId}'`);
|
||||
}
|
||||
|
||||
async function initializeLocalSpeechServices(params: {
|
||||
providers: RequestedSpeechProviders;
|
||||
localConfig: NonNullable<PaseoSpeechConfig["local"]> | null;
|
||||
localModels: ResolvedLocalModels;
|
||||
speechConfig: PaseoSpeechConfig | null;
|
||||
logger: Logger;
|
||||
}): Promise<InitializedLocalSpeech> {
|
||||
const { providers, localConfig, localModels, speechConfig, logger } = params;
|
||||
|
||||
const sttServices = {
|
||||
sttService: null as SpeechToTextProvider | null,
|
||||
ttsService: null as TextToSpeechProvider | null,
|
||||
dictationSttService: null as SpeechToTextProvider | null,
|
||||
};
|
||||
|
||||
const localSttEngines = new Map<LocalSttModelId, LocalSttEngine>();
|
||||
let localVoiceTtsProvider: TextToSpeechProvider | null = null;
|
||||
|
||||
const requiredLocalModelIds = computeRequiredLocalModelIds({
|
||||
providers,
|
||||
models: localModels,
|
||||
});
|
||||
|
||||
if (requiredLocalModelIds.length > 0 && localConfig) {
|
||||
try {
|
||||
logger.info(
|
||||
{
|
||||
modelsDir: localConfig.modelsDir,
|
||||
modelIds: requiredLocalModelIds,
|
||||
autoDownload: localConfig.autoDownload ?? true,
|
||||
},
|
||||
"Ensuring local speech models"
|
||||
);
|
||||
await ensureSherpaOnnxModels({
|
||||
modelsDir: localConfig.modelsDir,
|
||||
modelIds: requiredLocalModelIds,
|
||||
autoDownload: localConfig.autoDownload ?? true,
|
||||
logger,
|
||||
});
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
{
|
||||
err,
|
||||
modelsDir: localConfig.modelsDir,
|
||||
modelIds: requiredLocalModelIds,
|
||||
autoDownload: localConfig.autoDownload ?? true,
|
||||
hint:
|
||||
"Use `paseo speech models` to inspect status and " +
|
||||
"`paseo speech download --model <MODEL_ID>` to fetch missing models.",
|
||||
},
|
||||
"Failed to ensure local speech models"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const getLocalSttEngine = async (
|
||||
modelId: LocalSttModelId
|
||||
): Promise<LocalSttEngine | null> => {
|
||||
const existing = localSttEngines.get(modelId);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
if (!localConfig) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const created = await createLocalSttEngine({
|
||||
modelId,
|
||||
modelsDir: localConfig.modelsDir,
|
||||
logger,
|
||||
});
|
||||
localSttEngines.set(modelId, created);
|
||||
return created;
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
{
|
||||
err,
|
||||
modelsDir: localConfig.modelsDir,
|
||||
modelId,
|
||||
hint: buildModelDownloadHint(modelId),
|
||||
},
|
||||
"Failed to initialize local STT engine (models missing or invalid)"
|
||||
);
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
if (providers.voiceSttProvider === "local") {
|
||||
if (!localConfig) {
|
||||
logger.warn(
|
||||
{ configured: false },
|
||||
"Local STT selected for voice but local provider config is missing; STT will be unavailable"
|
||||
);
|
||||
} else {
|
||||
const voiceEngine = await getLocalSttEngine(localModels.voiceLocalSttModel);
|
||||
if (voiceEngine?.kind === "offline") {
|
||||
sttServices.sttService = new SherpaOnnxParakeetSTT({ engine: voiceEngine.engine }, logger);
|
||||
} else if (voiceEngine?.kind === "online") {
|
||||
sttServices.sttService = new SherpaOnnxSTT({ engine: voiceEngine.engine }, logger);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (providers.dictationSttProvider === "local") {
|
||||
if (!localConfig) {
|
||||
logger.warn(
|
||||
{ configured: false },
|
||||
"Local STT selected for dictation but local provider config is missing; dictation STT will be unavailable"
|
||||
);
|
||||
} else {
|
||||
const dictationEngine = await getLocalSttEngine(localModels.dictationLocalSttModel);
|
||||
if (dictationEngine?.kind === "offline") {
|
||||
sttServices.dictationSttService = {
|
||||
id: "local",
|
||||
createSession: () =>
|
||||
new SherpaParakeetRealtimeTranscriptionSession({ engine: dictationEngine.engine }),
|
||||
};
|
||||
} else if (dictationEngine?.kind === "online") {
|
||||
sttServices.dictationSttService = {
|
||||
id: "local",
|
||||
createSession: () => new SherpaRealtimeTranscriptionSession({ engine: dictationEngine.engine }),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (providers.voiceTtsProvider === "local") {
|
||||
if (!localConfig) {
|
||||
logger.warn(
|
||||
{ configured: false },
|
||||
"Local TTS selected for voice but local provider config is missing; TTS will be unavailable"
|
||||
);
|
||||
} else {
|
||||
try {
|
||||
if (localModels.voiceLocalTtsModel === "pocket-tts-onnx-int8") {
|
||||
const modelDir = getSherpaOnnxModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel);
|
||||
localVoiceTtsProvider = await PocketTtsOnnxTTS.create(
|
||||
{
|
||||
modelDir,
|
||||
precision: "int8",
|
||||
targetChunkMs: 50,
|
||||
},
|
||||
logger
|
||||
);
|
||||
} else {
|
||||
const modelDir = getSherpaOnnxModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel);
|
||||
localVoiceTtsProvider = new SherpaOnnxTTS(
|
||||
{
|
||||
preset: localModels.voiceLocalTtsModel,
|
||||
modelDir,
|
||||
speakerId: speechConfig?.voiceLocalTtsSpeakerId,
|
||||
speed: speechConfig?.voiceLocalTtsSpeed,
|
||||
},
|
||||
logger
|
||||
);
|
||||
}
|
||||
sttServices.ttsService = localVoiceTtsProvider;
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
{
|
||||
err,
|
||||
modelsDir: localConfig.modelsDir,
|
||||
modelId: localModels.voiceLocalTtsModel,
|
||||
hint: buildModelDownloadHint(localModels.voiceLocalTtsModel),
|
||||
},
|
||||
"Failed to initialize local TTS engine (models missing or invalid)"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
...sttServices,
|
||||
localVoiceTtsProvider,
|
||||
localSttEngines,
|
||||
requiredLocalModelIds,
|
||||
};
|
||||
}
|
||||
|
||||
function initializeOpenAiSpeechServices(params: {
|
||||
providers: RequestedSpeechProviders;
|
||||
openaiConfig?: PaseoOpenAIConfig;
|
||||
openAiCredentials: OpenAiCredentialState;
|
||||
existing: {
|
||||
sttService: SpeechToTextProvider | null;
|
||||
ttsService: TextToSpeechProvider | null;
|
||||
dictationSttService: SpeechToTextProvider | null;
|
||||
};
|
||||
logger: Logger;
|
||||
}): {
|
||||
sttService: SpeechToTextProvider | null;
|
||||
ttsService: TextToSpeechProvider | null;
|
||||
dictationSttService: SpeechToTextProvider | null;
|
||||
} {
|
||||
const { providers, openaiConfig, openAiCredentials, existing, logger } = params;
|
||||
|
||||
let sttService = existing.sttService;
|
||||
let ttsService = existing.ttsService;
|
||||
let dictationSttService = existing.dictationSttService;
|
||||
|
||||
const needsOpenAiStt = !sttService && providers.voiceSttProvider === "openai";
|
||||
const needsOpenAiTts = !ttsService && providers.voiceTtsProvider === "openai";
|
||||
const needsOpenAiDictation = !dictationSttService && providers.dictationSttProvider === "openai";
|
||||
|
||||
if (
|
||||
(needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) &&
|
||||
(openAiCredentials.openaiSttApiKey ||
|
||||
openAiCredentials.openaiTtsApiKey ||
|
||||
openAiCredentials.openaiDictationApiKey)
|
||||
) {
|
||||
logger.info("OpenAI speech provider initialized");
|
||||
|
||||
if (needsOpenAiStt && openAiCredentials.openaiSttApiKey) {
|
||||
const { apiKey: _sttApiKey, ...sttConfig } = openaiConfig?.stt ?? {};
|
||||
sttService = new OpenAISTT(
|
||||
{
|
||||
apiKey: openAiCredentials.openaiSttApiKey,
|
||||
...sttConfig,
|
||||
},
|
||||
logger
|
||||
);
|
||||
}
|
||||
|
||||
if (needsOpenAiTts && openAiCredentials.openaiTtsApiKey) {
|
||||
const { apiKey: _ttsApiKey, ...ttsConfig } = openaiConfig?.tts ?? {};
|
||||
ttsService = new OpenAITTS(
|
||||
{
|
||||
apiKey: openAiCredentials.openaiTtsApiKey,
|
||||
voice: "alloy",
|
||||
model: DEFAULT_OPENAI_TTS_MODEL,
|
||||
responseFormat: "pcm",
|
||||
...ttsConfig,
|
||||
},
|
||||
logger
|
||||
);
|
||||
}
|
||||
|
||||
const dictationApiKey = openAiCredentials.openaiDictationApiKey;
|
||||
if (needsOpenAiDictation && dictationApiKey) {
|
||||
dictationSttService = {
|
||||
id: "openai",
|
||||
createSession: ({ logger: sessionLogger, language, prompt }) =>
|
||||
new OpenAIRealtimeTranscriptionSession({
|
||||
apiKey: dictationApiKey,
|
||||
logger: sessionLogger,
|
||||
transcriptionModel:
|
||||
openaiConfig?.realtimeTranscriptionModel
|
||||
?? DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
|
||||
...(language ? { language } : {}),
|
||||
...(prompt ? { prompt } : {}),
|
||||
turnDetection: null,
|
||||
}),
|
||||
};
|
||||
}
|
||||
} else if (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) {
|
||||
logger.warn("OpenAI speech providers are configured but credentials are missing");
|
||||
}
|
||||
|
||||
return {
|
||||
sttService,
|
||||
ttsService,
|
||||
dictationSttService,
|
||||
};
|
||||
}
|
||||
|
||||
export type InitializedSpeechRuntime = {
|
||||
sttService: SpeechToTextProvider | null;
|
||||
ttsService: TextToSpeechProvider | null;
|
||||
@@ -504,7 +31,7 @@ export type InitializedSpeechRuntime = {
|
||||
cleanup: () => void;
|
||||
localModelConfig: {
|
||||
modelsDir: string;
|
||||
defaultModelIds: SherpaOnnxModelId[];
|
||||
defaultModelIds: LocalSpeechModelId[];
|
||||
} | null;
|
||||
};
|
||||
|
||||
@@ -515,16 +42,12 @@ export async function initializeSpeechRuntime(params: {
|
||||
}): Promise<InitializedSpeechRuntime> {
|
||||
const logger = params.logger;
|
||||
const speechConfig = params.speechConfig ?? null;
|
||||
const localConfig = speechConfig?.local ?? null;
|
||||
const openaiConfig = params.openaiConfig;
|
||||
|
||||
const providers = resolveRequestedSpeechProviders(speechConfig);
|
||||
const localModels = resolveConfiguredLocalModels(speechConfig);
|
||||
const openAiCredentials = resolveOpenAiCredentials(openaiConfig);
|
||||
|
||||
validateOpenAiCredentialRequirements({
|
||||
providers,
|
||||
openAiCredentials,
|
||||
openaiConfig,
|
||||
logger,
|
||||
});
|
||||
|
||||
@@ -536,16 +59,8 @@ export async function initializeSpeechRuntime(params: {
|
||||
voiceTts: providers.voiceTtsProvider,
|
||||
},
|
||||
availability: {
|
||||
openai: {
|
||||
stt: Boolean(openAiCredentials.openaiSttApiKey),
|
||||
tts: Boolean(openAiCredentials.openaiTtsApiKey),
|
||||
dictationStt: Boolean(openAiCredentials.openaiDictationApiKey),
|
||||
},
|
||||
local: {
|
||||
configured: Boolean(localConfig),
|
||||
modelsDir: localConfig?.modelsDir ?? null,
|
||||
autoDownload: localConfig?.autoDownload ?? null,
|
||||
},
|
||||
openai: getOpenAiSpeechAvailability(openaiConfig),
|
||||
local: getLocalSpeechAvailability(speechConfig),
|
||||
},
|
||||
},
|
||||
"Speech provider reconciliation started"
|
||||
@@ -553,8 +68,6 @@ export async function initializeSpeechRuntime(params: {
|
||||
|
||||
const localSpeech = await initializeLocalSpeechServices({
|
||||
providers,
|
||||
localConfig,
|
||||
localModels,
|
||||
speechConfig,
|
||||
logger,
|
||||
});
|
||||
@@ -562,7 +75,6 @@ export async function initializeSpeechRuntime(params: {
|
||||
const openAiSpeech = initializeOpenAiSpeechServices({
|
||||
providers,
|
||||
openaiConfig,
|
||||
openAiCredentials,
|
||||
existing: {
|
||||
sttService: localSpeech.sttService,
|
||||
ttsService: localSpeech.ttsService,
|
||||
@@ -610,27 +122,11 @@ export async function initializeSpeechRuntime(params: {
|
||||
"Speech provider reconciliation completed"
|
||||
);
|
||||
|
||||
const cleanup = () => {
|
||||
const maybeFreeable = localSpeech.localVoiceTtsProvider as unknown as { free?: () => void } | null;
|
||||
if (typeof maybeFreeable?.free === "function") {
|
||||
maybeFreeable.free();
|
||||
}
|
||||
for (const engine of localSpeech.localSttEngines.values()) {
|
||||
engine.engine.free();
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
sttService: openAiSpeech.sttService,
|
||||
ttsService: openAiSpeech.ttsService,
|
||||
dictationSttService: openAiSpeech.dictationSttService,
|
||||
cleanup,
|
||||
localModelConfig:
|
||||
localConfig
|
||||
? {
|
||||
modelsDir: localConfig.modelsDir,
|
||||
defaultModelIds: localSpeech.requiredLocalModelIds,
|
||||
}
|
||||
: null,
|
||||
cleanup: localSpeech.cleanup,
|
||||
localModelConfig: localSpeech.localModelConfig,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -2,3 +2,9 @@ import { z } from "zod";
|
||||
|
||||
export const SpeechProviderIdSchema = z.enum(["openai", "local"]);
|
||||
export type SpeechProviderId = z.infer<typeof SpeechProviderIdSchema>;
|
||||
|
||||
export type RequestedSpeechProviders = {
|
||||
dictationSttProvider: SpeechProviderId;
|
||||
voiceSttProvider: SpeechProviderId;
|
||||
voiceTtsProvider: SpeechProviderId;
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user