diff --git a/docs/data-model.md b/docs/data-model.md index c11d7a65f..200413712 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -171,6 +171,8 @@ Single file, validated with `PersistedConfigSchema`. All fields are optional with sensible defaults. +Local speech model ids are intentionally narrow: STT uses `parakeet-tdt-0.6b-v2-int8`, TTS uses `kokoro-en-v0_19`, and turn detection uses the bundled Silero VAD model. + --- ## 3. Schedule diff --git a/package-lock.json b/package-lock.json index e23b3cf97..18e616deb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10647,16 +10647,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@sctg/sentencepiece-js": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@sctg/sentencepiece-js/-/sentencepiece-js-1.3.3.tgz", - "integrity": "sha512-iPnzR2HGjdQQG2SpTyPH3wnnpgQ2aS14B3I2jHjomkoaMprM3Sn+WOQblHwByetnvn3n14y3IwjwogVUHA2cVA==", - "license": "Apache-2.0", - "dependencies": { - "app-root-path": "^3.1.0", - "buffer": "^6.0.3" - } - }, "node_modules/@segment/ajv-human-errors": { "version": "2.16.0", "resolved": "https://registry.npmjs.org/@segment/ajv-human-errors/-/ajv-human-errors-2.16.0.tgz", @@ -13823,15 +13813,6 @@ "node": ">=0.4.0" } }, - "node_modules/adm-zip": { - "version": "0.5.16", - "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.16.tgz", - "integrity": "sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==", - "license": "MIT", - "engines": { - "node": ">=12.0" - } - }, "node_modules/agent-base": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", @@ -14283,15 +14264,6 @@ "node": ">=18" } }, - "node_modules/app-root-path": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/app-root-path/-/app-root-path-3.1.0.tgz", - "integrity": "sha512-biN3PwB2gUtjaYy/isrU3aNWI5w+fAfvHkSvCKeQGxhmYpwKFUxudR3Yya+KqVRHBmEDYh+/lTozYCFbmzX4nA==", - "license": "MIT", - "engines": { - "node": ">= 6.0.0" - } - }, "node_modules/arg": { "version": "5.0.2", "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", @@ -15147,7 +15119,9 @@ "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", "integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==", "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", - "license": "MIT" + "dev": true, + "license": "MIT", + "optional": true }, "node_modules/bplist-creator": { "version": "0.1.0", @@ -17013,7 +16987,9 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", - "license": "MIT" + "dev": true, + "license": "MIT", + "optional": true }, "node_modules/detect-node-es": { "version": "1.1.0", @@ -18115,7 +18091,9 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", - "license": "MIT" + "dev": true, + "license": "MIT", + "optional": true }, "node_modules/esbuild": { "version": "0.27.3", @@ -22620,7 +22598,9 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz", "integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==", + "dev": true, "license": "BSD-3-Clause", + "optional": true, "dependencies": { "boolean": "^3.0.1", "es6-error": "^4.1.1", @@ -22637,7 +22617,9 @@ "version": "7.0.1", "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==", + "dev": true, "license": "MIT", + "optional": true, "dependencies": { "type-fest": "^0.13.1" }, @@ -22652,7 +22634,9 @@ "version": "0.13.1", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==", + "dev": true, "license": "(MIT OR CC0-1.0)", + "optional": true, "engines": { "node": ">=10" }, @@ -22677,6 +22661,7 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz", "integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==", + "dev": true, "license": "MIT", "dependencies": { "define-properties": "^1.2.1", @@ -25331,7 +25316,9 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", - "license": "ISC" + "dev": true, + "license": "ISC", + "optional": true }, "node_modules/json5": { "version": "2.2.3", @@ -26349,7 +26336,9 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==", + "dev": true, "license": "MIT", + "optional": true, "dependencies": { "escape-string-regexp": "^4.0.0" }, @@ -28985,29 +28974,6 @@ "regex-recursion": "^6.0.2" } }, - "node_modules/onnxruntime-common": { - "version": "1.24.3", - "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.24.3.tgz", - "integrity": "sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA==", - "license": "MIT" - }, - "node_modules/onnxruntime-node": { - "version": "1.24.3", - "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.24.3.tgz", - "integrity": "sha512-JH7+czbc8ALA819vlTgcV+Q214/+VjGeBHDjX81+ZCD0PCVCIFGFNtT0V4sXG/1JXypKPgScQcB3ij/hk3YnTg==", - "hasInstallScript": true, - "license": "MIT", - "os": [ - "win32", - "darwin", - "linux" - ], - "dependencies": { - "adm-zip": "^0.5.16", - "global-agent": "^3.0.0", - "onnxruntime-common": "1.24.3" - } - }, "node_modules/open": { "version": "7.4.2", "resolved": "https://registry.npmjs.org/open/-/open-7.4.2.tgz", @@ -31922,7 +31888,9 @@ "version": "2.15.4", "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz", "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==", + "dev": true, "license": "BSD-3-Clause", + "optional": true, "dependencies": { "boolean": "^3.0.1", "detect-node": "^2.0.4", @@ -31939,7 +31907,9 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", - "license": "BSD-3-Clause" + "dev": true, + "license": "BSD-3-Clause", + "optional": true }, "node_modules/rollup": { "version": "4.59.0", @@ -32230,7 +32200,9 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==", - "license": "MIT" + "dev": true, + "license": "MIT", + "optional": true }, "node_modules/send": { "version": "0.19.2", @@ -32515,12 +32487,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/sherpa-onnx": { - "version": "1.12.28", - "resolved": "https://registry.npmjs.org/sherpa-onnx/-/sherpa-onnx-1.12.28.tgz", - "integrity": "sha512-PaOuTCUA/Vh5o3zcq+z3j1Zpv2VlP7iubOgYoVODDyeFDicRxjskiEQqERdWEcpQUi6vh47q3A63bS7Y9FRHNg==", - "license": "Apache-2.0" - }, "node_modules/sherpa-onnx-darwin-arm64": { "version": "1.12.28", "resolved": "https://registry.npmjs.org/sherpa-onnx-darwin-arm64/-/sherpa-onnx-darwin-arm64-1.12.28.tgz", @@ -37603,7 +37569,6 @@ "@isaacs/ttlcache": "^2.1.4", "@modelcontextprotocol/sdk": "^1.20.1", "@opencode-ai/sdk": "1.14.46", - "@sctg/sentencepiece-js": "^1.1.0", "@xterm/headless": "^6.0.0", "ai": "5.0.78", "ajv": "^8.20.0", @@ -37614,7 +37579,6 @@ "lru-cache": "^11.5.1", "mnemonic-id": "^3.2.7", "node-pty": "1.2.0-beta.11", - "onnxruntime-node": "^1.23.0", "openai": "^4.20.0", "p-limit": "^7.3.0", "p-memoize": "^8.0.0", @@ -37622,7 +37586,6 @@ "pino-pretty": "^13.1.3", "qrcode": "^1.5.4", "rotating-file-stream": "^3.2.9", - "sherpa-onnx": "1.12.28", "sherpa-onnx-node": "1.12.28", "strip-ansi": "^7.1.2", "tree-kill": "^1.2.2", diff --git a/packages/desktop/scripts/after-pack.js b/packages/desktop/scripts/after-pack.js index 985245e5b..ecbc69fd9 100644 --- a/packages/desktop/scripts/after-pack.js +++ b/packages/desktop/scripts/after-pack.js @@ -27,29 +27,6 @@ function pruneChildrenExcept(parent, keep) { } } -function pruneOnnxRuntime(nodeModules, platform, arch) { - const onnxBin = path.join(nodeModules, "onnxruntime-node", "bin", "napi-v6"); - if (!fs.existsSync(onnxBin)) return; - - const otherPlatforms = ["darwin", "linux", "win32"].filter((p) => p !== platform); - for (const p of otherPlatforms) { - rmSafe(path.join(onnxBin, p)); - } - - pruneChildrenExcept(path.join(onnxBin, platform), new Set([arch])); - - if (platform === "linux") { - const archDir = path.join(onnxBin, "linux", arch); - if (fs.existsSync(archDir)) { - for (const name of fs.readdirSync(archDir)) { - if (name.includes("cuda") || name.includes("tensorrt")) { - fs.rmSync(path.join(archDir, name), { force: true }); - } - } - } - } -} - function pruneClaudeAgentSdk(nodeModules, platform, arch) { const vendorRoot = path.join(nodeModules, "@anthropic-ai", "claude-agent-sdk", "vendor"); const keepName = RIPGREP_PLATFORM_DIR[platform]?.[arch]; @@ -107,7 +84,6 @@ function pruneNativeModules(appOutDir, platform, arch) { const before = dirSizeSync(nodeModules); - pruneOnnxRuntime(nodeModules, platform, arch); pruneClaudeAgentSdk(nodeModules, platform, arch); pruneNodePty(nodeModules, platform, arch); pruneSharpLibvips(nodeModules, platform, arch); diff --git a/packages/server/package.json b/packages/server/package.json index 8fda4396c..646e83298 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -44,7 +44,6 @@ "generate:config-schema": "tsx scripts/generate-config-schema.ts", "speech:models": "tsx scripts/list-speech-models.ts", "speech:download": "tsx scripts/download-speech-models.ts", - "speech:tts:matrix": "tsx scripts/generate-sherpa-tts-matrix.ts", "speech:transcribe:local": "tsx scripts/transcribe-local-wav.ts", "test": "npm run test:unit && npm run test:integration", "test:unit": "vitest run --exclude \"**/*.e2e.test.ts\"", @@ -70,7 +69,6 @@ "@isaacs/ttlcache": "^2.1.4", "@modelcontextprotocol/sdk": "^1.20.1", "@opencode-ai/sdk": "1.14.46", - "@sctg/sentencepiece-js": "^1.1.0", "@xterm/headless": "^6.0.0", "ai": "5.0.78", "ajv": "^8.20.0", @@ -81,7 +79,6 @@ "lru-cache": "^11.5.1", "mnemonic-id": "^3.2.7", "node-pty": "1.2.0-beta.11", - "onnxruntime-node": "^1.23.0", "openai": "^4.20.0", "p-limit": "^7.3.0", "p-memoize": "^8.0.0", @@ -89,7 +86,6 @@ "pino-pretty": "^13.1.3", "qrcode": "^1.5.4", "rotating-file-stream": "^3.2.9", - "sherpa-onnx": "1.12.28", "sherpa-onnx-node": "1.12.28", "strip-ansi": "^7.1.2", "tree-kill": "^1.2.2", diff --git a/packages/server/scripts/download-speech-models.ts b/packages/server/scripts/download-speech-models.ts index 0a6c610e8..b75086bb7 100644 --- a/packages/server/scripts/download-speech-models.ts +++ b/packages/server/scripts/download-speech-models.ts @@ -31,11 +31,7 @@ function parseArgs(argv: string[]): { modelsDir: string; modelIds: LocalSpeechMo } if (modelIds.length === 0) { - const stt = (process.env.PASEO_LOCAL_STT_MODEL || - DEFAULT_LOCAL_STT_MODEL) as LocalSpeechModelId; - const tts = (process.env.PASEO_LOCAL_TTS_MODEL || - DEFAULT_LOCAL_TTS_MODEL) as LocalSpeechModelId; - modelIds.push(stt, tts); + modelIds.push(DEFAULT_LOCAL_STT_MODEL, DEFAULT_LOCAL_TTS_MODEL); } return { modelsDir, modelIds }; diff --git a/packages/server/scripts/generate-sherpa-tts-matrix.ts b/packages/server/scripts/generate-sherpa-tts-matrix.ts deleted file mode 100644 index 3622eb9f4..000000000 --- a/packages/server/scripts/generate-sherpa-tts-matrix.ts +++ /dev/null @@ -1,436 +0,0 @@ -import { mkdir, writeFile } from "node:fs/promises"; -import path from "node:path"; - -import { createRootLogger } from "../src/server/logger.js"; -import { resolvePaseoHome } from "../src/server/paseo-home.js"; -import { float32ToPcm16le, parsePcmRateFromFormat } from "../src/server/speech/audio.js"; -import { - ensureLocalSpeechModels, - getLocalSpeechModelDir, - type LocalTtsModelId, -} from "../src/server/speech/providers/local/models.js"; -import { PocketTtsOnnxTTS } from "../src/server/speech/providers/local/pocket/pocket-tts-onnx.js"; -import { - LOCAL_TTS_MODEL_IDS, - LocalTtsModelIdSchema, -} from "../src/server/speech/providers/local/sherpa/model-catalog.js"; -import { loadSherpaOnnxNode } from "../src/server/speech/providers/local/sherpa/sherpa-onnx-node-loader.js"; - -const DEFAULT_SAMPLE_TEXT = - "Paseo keeps your coding agents in your pocket, with fast local speech and reliable remote control."; - -interface ScriptOptions { - modelsDir: string; - outputDir: string; - text: string; - speed: number; - modelIds: LocalTtsModelId[]; -} - -interface GeneratedSample { - modelId: LocalTtsModelId; - voiceLabel: string; - speakerId: number | null; - sampleRate: number; - durationSeconds: number; - relativePath: string; -} - -function usage(): string { - return [ - "Generate a local Sherpa/Pocket TTS sample matrix.", - "", - "Options:", - " --models-dir Directory for downloaded local speech models", - " --output-dir Directory where WAV samples are written", - " --model TTS model to include (repeatable)", - " --text Sample sentence used for every generated file", - " --speed TTS speed multiplier for Sherpa models (default: 1.0)", - " --help Show this help", - "", - `Known TTS model IDs: ${LOCAL_TTS_MODEL_IDS.join(", ")}`, - ].join("\n"); -} - -function parsePositiveNumber(raw: string, flag: string): number { - const parsed = Number.parseFloat(raw); - if (!Number.isFinite(parsed) || parsed <= 0) { - throw new Error(`${flag} must be a positive number (received: ${raw})`); - } - return parsed; -} - -function parseArgs(argv: string[]): ScriptOptions { - const paseoHome = resolvePaseoHome(); - const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); - - let modelsDir = process.env.PASEO_LOCAL_MODELS_DIR ?? `${paseoHome}/models/local-speech`; - let outputDir = path.resolve(process.cwd(), ".debug", "sherpa-tts-matrix", timestamp); - let text = DEFAULT_SAMPLE_TEXT; - let speed = parsePositiveNumber(process.env.PASEO_VOICE_LOCAL_TTS_SPEED ?? "1.0", "--speed"); - const requestedModelIds: LocalTtsModelId[] = []; - - for (let i = 0; i < argv.length; i += 1) { - const arg = argv[i]; - - if (arg === "--help") { - // eslint-disable-next-line no-console - console.log(usage()); - process.exit(0); - } - - if (arg === "--models-dir") { - modelsDir = path.resolve(argv[i + 1] ?? modelsDir); - i += 1; - continue; - } - - if (arg === "--output-dir") { - outputDir = path.resolve(argv[i + 1] ?? outputDir); - i += 1; - continue; - } - - if (arg === "--text") { - text = argv[i + 1] ?? text; - i += 1; - continue; - } - - if (arg === "--speed") { - const raw = argv[i + 1]; - if (!raw) { - throw new Error("--speed requires a value"); - } - speed = parsePositiveNumber(raw, "--speed"); - i += 1; - continue; - } - - if (arg === "--model") { - const raw = argv[i + 1]; - if (!raw) { - throw new Error("--model requires a value"); - } - requestedModelIds.push(LocalTtsModelIdSchema.parse(raw)); - i += 1; - continue; - } - - throw new Error(`Unknown argument: ${arg}`); - } - - const trimmed = text.trim(); - if (!trimmed) { - throw new Error("Sample text cannot be empty"); - } - - const modelIds = - requestedModelIds.length > 0 - ? Array.from(new Set(requestedModelIds)) - : [...LOCAL_TTS_MODEL_IDS]; - - return { - modelsDir, - outputDir, - text: trimmed, - speed, - modelIds, - }; -} - -function pcm16MonoToWavBuffer(pcm16: Buffer, sampleRate: number): Buffer { - const channels = 1; - const bitsPerSample = 16; - const headerSize = 44; - const wav = Buffer.alloc(headerSize + pcm16.length); - const byteRate = (sampleRate * channels * bitsPerSample) / 8; - const blockAlign = (channels * bitsPerSample) / 8; - - wav.write("RIFF", 0); - wav.writeUInt32LE(36 + pcm16.length, 4); - wav.write("WAVE", 8); - wav.write("fmt ", 12); - wav.writeUInt32LE(16, 16); - wav.writeUInt16LE(1, 20); - wav.writeUInt16LE(channels, 22); - wav.writeUInt32LE(sampleRate, 24); - wav.writeUInt32LE(byteRate, 28); - wav.writeUInt16LE(blockAlign, 32); - wav.writeUInt16LE(bitsPerSample, 34); - wav.write("data", 36); - wav.writeUInt32LE(pcm16.length, 40); - pcm16.copy(wav, 44); - - return wav; -} - -function sanitizeFilePart(input: string): string { - return input.replace(/[^a-zA-Z0-9._-]/g, "-"); -} - -function toFloatSamples(audio: unknown): Float32Array { - const raw = (audio as { samples?: unknown })?.samples; - if (raw instanceof Float32Array) { - return raw; - } - if (Array.isArray(raw)) { - return Float32Array.from(raw as number[]); - } - throw new Error("Unexpected Sherpa TTS output: missing Float32 samples"); -} - -function toSampleRate(audio: unknown, fallback: number): number { - const sampleRate = (audio as { sampleRate?: unknown })?.sampleRate; - if (typeof sampleRate === "number" && Number.isFinite(sampleRate) && sampleRate > 0) { - return sampleRate; - } - return fallback; -} - -function createSherpaOfflineTts(params: { - modelId: "kokoro-en-v0_19" | "kitten-nano-en-v0_1-fp16"; - modelDir: string; - speed: number; -}): { tts: unknown; speed: number } { - const sherpa = loadSherpaOnnxNode(); - const model = - params.modelId === "kokoro-en-v0_19" - ? { - kokoro: { - model: `${params.modelDir}/model.onnx`, - voices: `${params.modelDir}/voices.bin`, - tokens: `${params.modelDir}/tokens.txt`, - dataDir: `${params.modelDir}/espeak-ng-data`, - lengthScale: 1.0, - }, - } - : { - kitten: { - model: `${params.modelDir}/model.fp16.onnx`, - voices: `${params.modelDir}/voices.bin`, - tokens: `${params.modelDir}/tokens.txt`, - dataDir: `${params.modelDir}/espeak-ng-data`, - lengthScale: 1.0, - }, - }; - - return { - tts: new sherpa.OfflineTts({ - model, - numThreads: 2, - provider: "cpu", - maxNumSentences: 1, - }), - speed: params.speed, - }; -} - -async function readStreamToBuffer(stream: NodeJS.ReadableStream): Promise { - const chunks: Buffer[] = []; - for await (const chunk of stream as AsyncIterable) { - chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); - } - return Buffer.concat(chunks); -} - -function toCsv(rows: GeneratedSample[]): string { - const header = [ - "model_id", - "voice_label", - "speaker_id", - "sample_rate", - "duration_seconds", - "relative_path", - ]; - - const lines = [header.join(",")]; - for (const row of rows) { - const cells = [ - row.modelId, - row.voiceLabel, - row.speakerId === null ? "" : String(row.speakerId), - String(row.sampleRate), - row.durationSeconds.toFixed(3), - row.relativePath, - ]; - - lines.push( - cells - .map((cell) => { - const escaped = cell.replace(/"/g, '""'); - return escaped.includes(",") ? `"${escaped}"` : escaped; - }) - .join(","), - ); - } - return `${lines.join("\n")}\n`; -} - -const logger = createRootLogger({ level: "info", format: "pretty" }); -const options = parseArgs(process.argv.slice(2)); - -logger.info( - { - modelsDir: options.modelsDir, - outputDir: options.outputDir, - modelIds: options.modelIds, - speed: options.speed, - text: options.text, - }, - "Generating TTS matrix across all model/voice combinations", -); - -await ensureLocalSpeechModels({ - modelsDir: options.modelsDir, - modelIds: options.modelIds, - logger, -}); - -await mkdir(options.outputDir, { recursive: true }); - -const generated: GeneratedSample[] = []; - -for (const modelId of options.modelIds) { - const modelDir = getLocalSpeechModelDir(options.modelsDir, modelId); - const modelOutputDir = path.join(options.outputDir, sanitizeFilePart(modelId)); - await mkdir(modelOutputDir, { recursive: true }); - - if (modelId === "pocket-tts-onnx-int8") { - const tts = await PocketTtsOnnxTTS.create( - { - modelDir, - precision: "int8", - targetChunkMs: 50, - }, - logger, - ); - const result = await tts.synthesizeSpeech(options.text); - const pcm16 = await readStreamToBuffer(result.stream); - const sampleRate = parsePcmRateFromFormat(result.format, 24000) ?? 24000; - - const voiceLabel = "default"; - const filename = `${sanitizeFilePart(modelId)}__voice-${voiceLabel}.wav`; - const absPath = path.join(modelOutputDir, filename); - const relPath = path.relative(options.outputDir, absPath); - - await writeFile(absPath, pcm16MonoToWavBuffer(pcm16, sampleRate)); - - generated.push({ - modelId, - voiceLabel, - speakerId: null, - sampleRate, - durationSeconds: pcm16.length / 2 / sampleRate, - relativePath: relPath, - }); - - logger.info({ modelId, voiceLabel, path: relPath }, "Generated sample"); - continue; - } - - if (modelId !== "kokoro-en-v0_19" && modelId !== "kitten-nano-en-v0_1-fp16") { - throw new Error(`Unsupported local Sherpa TTS model for this script: ${modelId}`); - } - - const { tts, speed } = createSherpaOfflineTts({ - modelId, - modelDir, - speed: options.speed, - }); - - try { - const rawNumSpeakers = (tts as { numSpeakers?: unknown }).numSpeakers; - const numSpeakers = - typeof rawNumSpeakers === "number" && Number.isFinite(rawNumSpeakers) && rawNumSpeakers > 0 - ? Math.floor(rawNumSpeakers) - : 1; - const pad = Math.max(2, String(Math.max(0, numSpeakers - 1)).length); - const fallbackSampleRate = - typeof (tts as { sampleRate?: unknown }).sampleRate === "number" - ? Math.max(1, Math.floor((tts as { sampleRate: number }).sampleRate)) - : 24000; - - logger.info({ modelId, numSpeakers }, "Discovered Sherpa speaker count"); - - for (let sid = 0; sid < numSpeakers; sid += 1) { - const audio = ( - tts as { - generate: (config: { text: string; sid: number; speed: number }) => unknown; - } - ).generate({ - text: options.text, - sid, - speed, - }); - const samples = toFloatSamples(audio); - const sampleRate = toSampleRate(audio, fallbackSampleRate); - const pcm16 = float32ToPcm16le(samples); - - const voiceLabel = `sid-${String(sid).padStart(pad, "0")}`; - const filename = `${sanitizeFilePart(modelId)}__voice-${voiceLabel}.wav`; - const absPath = path.join(modelOutputDir, filename); - const relPath = path.relative(options.outputDir, absPath); - - await writeFile(absPath, pcm16MonoToWavBuffer(pcm16, sampleRate)); - - generated.push({ - modelId, - voiceLabel, - speakerId: sid, - sampleRate, - durationSeconds: pcm16.length / 2 / sampleRate, - relativePath: relPath, - }); - - logger.info({ modelId, voiceLabel, path: relPath }, "Generated sample"); - } - } finally { - (tts as { free?: () => void }).free?.(); - } -} - -const manifest = { - createdAt: new Date().toISOString(), - text: options.text, - speed: options.speed, - modelsDir: options.modelsDir, - outputDir: options.outputDir, - namingConvention: "/__voice-.wav", - notes: [ - "For Sherpa kitten/kokoro models, voice-label uses sid-.", - "For Pocket TTS, voice-label is default.", - ], - generated, -}; - -await writeFile( - path.join(options.outputDir, "manifest.json"), - `${JSON.stringify(manifest, null, 2)}\n`, -); -await writeFile(path.join(options.outputDir, "samples.csv"), toCsv(generated)); -await writeFile( - path.join(options.outputDir, "README.txt"), - [ - "Sherpa TTS sample matrix", - "", - `Sample text: ${options.text}`, - `Speed: ${options.speed}`, - "", - "Naming convention:", - " /__voice-.wav", - " voice-label = sid- for Sherpa models", - " voice-label = default for Pocket TTS", - "", - `Total files: ${generated.length}`, - ].join("\n"), -); - -logger.info( - { - outputDir: options.outputDir, - generatedCount: generated.length, - models: Array.from(new Set(generated.map((item) => item.modelId))), - }, - "TTS matrix generation complete", -); diff --git a/packages/server/scripts/transcribe-local-wav.ts b/packages/server/scripts/transcribe-local-wav.ts index b25347fd0..0b323df02 100644 --- a/packages/server/scripts/transcribe-local-wav.ts +++ b/packages/server/scripts/transcribe-local-wav.ts @@ -29,7 +29,7 @@ function usage(): string { " npm run speech:transcribe:local -- ./sample.wav --out ./tmp/sample.transcript.txt", "", "Env fallbacks:", - " PASEO_LOCAL_MODELS_DIR, PASEO_LOCAL_STT_MODEL", + " PASEO_LOCAL_MODELS_DIR", ].join("\n"); } @@ -49,9 +49,7 @@ function parseArgs(argv: string[]): CliOptions { const positional: string[] = []; let outPath: string | undefined; - let model = LocalSttModelIdSchema.parse( - process.env.PASEO_LOCAL_STT_MODEL ?? DEFAULT_LOCAL_STT_MODEL, - ); + let model = LocalSttModelIdSchema.parse(DEFAULT_LOCAL_STT_MODEL); let modelsDir = defaultModelsDir; for (let i = 0; i < argv.length; i++) { diff --git a/packages/server/src/server/bootstrap.smoke.test.ts b/packages/server/src/server/bootstrap.smoke.test.ts index 744cfa027..e70dd9cfc 100644 --- a/packages/server/src/server/bootstrap.smoke.test.ts +++ b/packages/server/src/server/bootstrap.smoke.test.ts @@ -147,8 +147,8 @@ describe("paseo daemon bootstrap", () => { local: { modelsDir: path.join(os.tmpdir(), `paseo-missing-models-${Date.now()}`), models: { - dictationStt: "parakeet-tdt-0.6b-v3-int8", - voiceStt: "parakeet-tdt-0.6b-v3-int8", + dictationStt: "parakeet-tdt-0.6b-v2-int8", + voiceStt: "parakeet-tdt-0.6b-v2-int8", voiceTts: "kokoro-en-v0_19", }, }, diff --git a/packages/server/src/server/daemon-client.e2e.test.ts b/packages/server/src/server/daemon-client.e2e.test.ts index ad9d78aa8..2857e8aa0 100644 --- a/packages/server/src/server/daemon-client.e2e.test.ts +++ b/packages/server/src/server/daemon-client.e2e.test.ts @@ -39,35 +39,25 @@ async function readFixture(fileName: string): Promise { return readFile(fixturePath(fileName)); } -function hasSherpaZipformerModels(modelsDir: string): boolean { +function hasSherpaParakeetModels(modelsDir: string): boolean { return ( existsSync( - path.join( - modelsDir, - "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", - "encoder-epoch-99-avg-1.onnx", - ), + path.join(modelsDir, "sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8", "encoder.int8.onnx"), ) && - existsSync( - path.join( - modelsDir, - "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", - "tokens.txt", - ), - ) + existsSync(path.join(modelsDir, "sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8", "tokens.txt")) ); } -function hasSherpaKittenModels(modelsDir: string): boolean { +function hasSherpaKokoroModels(modelsDir: string): boolean { return ( - existsSync(path.join(modelsDir, "kitten-nano-en-v0_1-fp16", "model.fp16.onnx")) && - existsSync(path.join(modelsDir, "kitten-nano-en-v0_1-fp16", "voices.bin")) && - existsSync(path.join(modelsDir, "kitten-nano-en-v0_1-fp16", "tokens.txt")) + existsSync(path.join(modelsDir, "kokoro-en-v0_19", "model.onnx")) && + existsSync(path.join(modelsDir, "kokoro-en-v0_19", "voices.bin")) && + existsSync(path.join(modelsDir, "kokoro-en-v0_19", "tokens.txt")) ); } const hasLocalSpeech = - hasSherpaZipformerModels(localModelsDir) && hasSherpaKittenModels(localModelsDir); + hasSherpaParakeetModels(localModelsDir) && hasSherpaKokoroModels(localModelsDir); const hasAnySpeech = hasLocalSpeech || Boolean(openaiApiKey); const speechTest = hasAnySpeech ? test : test.skip; @@ -492,11 +482,10 @@ function resolveSpeechConfig() { local: { modelsDir: localModelsDir, models: { - dictationStt: - process.env.PASEO_DICTATION_LOCAL_STT_MODEL ?? "zipformer-bilingual-zh-en-2023-02-20", - voiceStt: - process.env.PASEO_VOICE_LOCAL_STT_MODEL ?? "zipformer-bilingual-zh-en-2023-02-20", - voiceTts: process.env.PASEO_VOICE_LOCAL_TTS_MODEL ?? "kitten-nano-en-v0_1-fp16", + dictationStt: "parakeet-tdt-0.6b-v2-int8", + voiceStt: "parakeet-tdt-0.6b-v2-int8", + voiceTts: "kokoro-en-v0_19", + voiceTtsSpeakerId: 0, }, }, }; diff --git a/packages/server/src/server/speech/providers/local/pocket/onnxruntime-augmentation.d.ts b/packages/server/src/server/speech/providers/local/pocket/onnxruntime-augmentation.d.ts deleted file mode 100644 index 9169cf906..000000000 --- a/packages/server/src/server/speech/providers/local/pocket/onnxruntime-augmentation.d.ts +++ /dev/null @@ -1,15 +0,0 @@ -// Type augmentation for onnxruntime-node to include runtime properties -// not exposed in the official type definitions - -import type * as ort from "onnxruntime-node"; - -declare module "onnxruntime-node" { - interface InferenceSession { - /** Input tensor names (available at runtime but not in types) */ - readonly inputNames: string[]; - /** Output tensor names (available at runtime but not in types) */ - readonly outputNames?: string[]; - /** Input metadata for shape/type info (available at runtime but not in types) */ - readonly inputMetadata?: unknown; - } -} diff --git a/packages/server/src/server/speech/providers/local/pocket/pocket-tts-onnx.ts b/packages/server/src/server/speech/providers/local/pocket/pocket-tts-onnx.ts deleted file mode 100644 index 635765120..000000000 --- a/packages/server/src/server/speech/providers/local/pocket/pocket-tts-onnx.ts +++ /dev/null @@ -1,629 +0,0 @@ -import { existsSync } from "node:fs"; -import { readFile } from "node:fs/promises"; -import { Readable } from "node:stream"; -import type pino from "pino"; - -import type { SpeechStreamResult, TextToSpeechProvider } from "../../../speech-provider.js"; -import { - chunkBuffer, - float32ToPcm16le, - parsePcm16MonoWav, - pcm16leToFloat32, -} from "../../../audio.js"; -import { Pcm16MonoResampler } from "../../../../agent/pcm16-resampler.js"; - -type OrtModule = typeof import("onnxruntime-node"); -type OrtSession = import("onnxruntime-node").InferenceSession; -type OrtTensor = import("onnxruntime-node").Tensor; - -function createSessionFeed(feed: Record): Record { - return feed; -} - -interface SentencePieceProcessor { - encodeIds: (text: string) => number[]; - load?: (modelPath: string) => unknown; - Load?: (modelPath: string) => unknown; -} - -function assertFileExists(filePath: string, label: string): void { - if (!existsSync(filePath)) { - throw new Error(`Missing ${label}: ${filePath}`); - } -} - -function product(dims: number[]): number { - let out = 1; - for (const d of dims) out *= d; - return out; -} - -function normalizeDims(dims: Array): number[] { - // ONNX metadata can contain dynamic dimensions as strings (e.g. "batch") or -1. - // For state tensors we want a valid minimal shape, so coerce unknown/invalid dims to 1. - // Preserve explicit 0 dims (some models use empty initial state buffers with shape [0]). - return dims.map((d) => { - if (typeof d === "number" && Number.isFinite(d)) { - if (d === 0) return 0; - if (d > 0) return d; - return 1; - } - return 1; - }); -} - -function getSessionInputMeta( - session: OrtSession, - inputName: string, -): { type?: string; dims?: Array } | undefined { - const metaAny = session.inputMetadata; - if (!Array.isArray(metaAny)) { - return undefined; - } - const entry = metaAny.find((m) => m.name === inputName); - if (!entry) return undefined; - return { type: String(entry.type), dims: entry.shape }; -} - -function toBigInt64(values: number[]): BigInt64Array { - const out = new BigInt64Array(values.length); - for (let i = 0; i < values.length; i += 1) { - out[i] = BigInt(values[i]); - } - return out; -} - -function randn(): number { - // Box–Muller - let u = 0; - let v = 0; - while (u === 0) u = Math.random(); - while (v === 0) v = Math.random(); - return Math.sqrt(-2.0 * Math.log(u)) * Math.cos(2.0 * Math.PI * v); -} - -function normalizeTextForPocket(text: string): string { - const trimmed = text.trim(); - if (!trimmed) { - throw new Error("Cannot synthesize empty text"); - } - let out = trimmed; - if (out.length > 0 && /[A-Za-z0-9]$/.test(out)) { - out = `${out}.`; - } - if (out.length > 0 && /[a-z]/.test(out[0])) { - out = out[0].toUpperCase() + out.slice(1); - } - return out; -} - -async function loadOrt(): Promise { - return import("onnxruntime-node"); -} - -interface SentencePieceModule { - SentencePieceProcessor?: new () => SentencePieceProcessor; - default?: - | (new () => SentencePieceProcessor) - | { SentencePieceProcessor?: new () => SentencePieceProcessor }; -} - -function isSentencePieceModule(mod: unknown): mod is SentencePieceModule { - return mod !== null && typeof mod === "object"; -} - -function getSentencePieceProcessor( - mod: SentencePieceModule, -): (new () => SentencePieceProcessor) | undefined { - if (mod.SentencePieceProcessor) { - return mod.SentencePieceProcessor; - } - const defaultValue = mod.default; - if ( - defaultValue && - typeof defaultValue === "object" && - "SentencePieceProcessor" in defaultValue - ) { - return defaultValue.SentencePieceProcessor; - } - if (typeof defaultValue === "function") { - return defaultValue as new () => SentencePieceProcessor; - } - return undefined; -} - -async function loadSentencePiece(tokenizerModelPath: string): Promise { - const mod = await import("@sctg/sentencepiece-js"); - - if (!isSentencePieceModule(mod)) { - throw new Error("@sctg/sentencepiece-js module has unexpected shape"); - } - - const Processor = getSentencePieceProcessor(mod); - - if (!Processor) { - throw new Error("Failed to load SentencePiece processor from @sctg/sentencepiece-js"); - } - - const sp = new Processor(); - - if (typeof sp.load === "function") { - await sp.load(tokenizerModelPath); - } else if (typeof sp.Load === "function") { - sp.Load(tokenizerModelPath); - } else { - throw new Error("SentencePiece processor does not expose load()/Load()"); - } - - return sp; -} - -function getOrtProviders(ort: OrtModule, device: "auto" | "cpu" | "cuda"): string[] { - // NOTE: onnxruntime-node uses backend names like "cpu"/"coreml"/"webgpu" (not "CPUExecutionProvider"). - if (device === "cpu") return ["cpu"]; - if (device === "cuda") return ["cuda", "cpu"]; - // auto - // CoreML EP does not support some dynamic/zero-length shapes used by Pocket TTS (e.g. [1, 0, 32]). - // Default to CPU to keep behavior predictable across platforms. - void ort; - return ["cpu"]; -} - -function createZeroTensorForInput( - ort: OrtModule, - session: OrtSession, - inputName: string, -): OrtTensor { - const meta = getSessionInputMeta(session, inputName); - const dims = normalizeDims(meta?.dims ?? []); - if (dims.length === 0) { - throw new Error(`Missing input metadata shape for ${inputName}`); - } - - const type = (meta?.type ?? "float32").toLowerCase(); - const size = product(dims); - - if (type.includes("int64")) { - return new ort.Tensor("int64", new BigInt64Array(size), dims); - } - if (type.includes("bool")) { - return new ort.Tensor("bool", new Uint8Array(size), dims); - } - return new ort.Tensor("float32", new Float32Array(size), dims); -} - -function initState(session: OrtSession, ort: OrtModule): Record { - const out: Record = {}; - for (const name of session.inputNames) { - if (name.startsWith("state_")) { - out[name] = createZeroTensorForInput(ort, session, name); - } - } - return out; -} - -function updateStateFromOutputs( - state: Record, - outputs: Record, -): void { - for (const [name, tensor] of Object.entries(outputs)) { - if (!name.startsWith("out_state_")) continue; - const idx = Number.parseInt(name.replace("out_state_", ""), 10); - if (Number.isFinite(idx)) { - state[`state_${idx}`] = tensor; - } - } -} - -interface OrtTensorWithData { - data: unknown; -} - -function tensorDataFloat32(t: OrtTensor): Float32Array { - const tensorWithData = t as OrtTensorWithData; - const data = tensorWithData.data; - if (data instanceof Float32Array) return data; - if (Array.isArray(data)) return Float32Array.from(data); - throw new Error("Unexpected tensor data type (expected Float32Array)"); -} - -export interface PocketTtsOnnxConfig { - modelDir: string; - precision?: "int8" | "fp32"; - device?: "auto" | "cpu" | "cuda"; - temperature?: number; - lsdSteps?: number; - maxFrames?: number; - framesAfterEos?: number; - firstChunkFrames?: number; - maxChunkFrames?: number; - targetChunkMs?: number; - referenceAudioFile?: string; -} - -class PocketTtsOnnxEngine { - static readonly SAMPLE_RATE = 24000; - static readonly SAMPLES_PER_FRAME = 1920; - - private readonly ort: OrtModule; - - private readonly temperature: number; - private readonly lsdSteps: number; - private readonly maxFrames: number; - private readonly framesAfterEos: number; - - private readonly firstChunkFrames: number; - private readonly maxChunkFrames: number; - - private readonly tokenizer: SentencePieceProcessor; - private readonly textConditioner: OrtSession; - private readonly flowLmMain: OrtSession; - private readonly flowLmFlow: OrtSession; - private readonly mimiDecoder: OrtSession; - - private readonly stBuffers: Array<{ s: OrtTensor; t: OrtTensor }>; - private readonly voiceEmbeddings: OrtTensor; - - private constructor(args: { - ort: OrtModule; - temperature: number; - lsdSteps: number; - maxFrames: number; - framesAfterEos: number; - firstChunkFrames: number; - maxChunkFrames: number; - tokenizer: SentencePieceProcessor; - textConditioner: OrtSession; - flowLmMain: OrtSession; - flowLmFlow: OrtSession; - mimiDecoder: OrtSession; - stBuffers: Array<{ s: OrtTensor; t: OrtTensor }>; - voiceEmbeddings: OrtTensor; - }) { - this.ort = args.ort; - this.temperature = args.temperature; - this.lsdSteps = args.lsdSteps; - this.maxFrames = args.maxFrames; - this.framesAfterEos = args.framesAfterEos; - this.firstChunkFrames = args.firstChunkFrames; - this.maxChunkFrames = args.maxChunkFrames; - this.tokenizer = args.tokenizer; - this.textConditioner = args.textConditioner; - this.flowLmMain = args.flowLmMain; - this.flowLmFlow = args.flowLmFlow; - this.mimiDecoder = args.mimiDecoder; - this.stBuffers = args.stBuffers; - this.voiceEmbeddings = args.voiceEmbeddings; - } - - static async create( - config: PocketTtsOnnxConfig, - logger: pino.Logger, - ): Promise { - const log = logger.child({ - module: "speech", - provider: "pocket-tts", - component: "onnx-engine", - }); - - const modelDir = config.modelDir; - const onnxDir = `${modelDir}/onnx`; - const precision = config.precision ?? "int8"; - const device = config.device ?? "auto"; - const temperature = config.temperature ?? 0.7; - const lsdSteps = config.lsdSteps ?? 10; - const maxFrames = config.maxFrames ?? 500; - const framesAfterEos = config.framesAfterEos ?? 3; - const firstChunkFrames = config.firstChunkFrames ?? 2; - const maxChunkFrames = config.maxChunkFrames ?? 15; - - const tokenizerPath = `${modelDir}/tokenizer.model`; - const referenceAudioFile = config.referenceAudioFile ?? `${modelDir}/reference_sample.wav`; - - const flowMainFile = precision === "int8" ? "flow_lm_main_int8.onnx" : "flow_lm_main.onnx"; - const flowFlowFile = precision === "int8" ? "flow_lm_flow_int8.onnx" : "flow_lm_flow.onnx"; - const decoderFile = precision === "int8" ? "mimi_decoder_int8.onnx" : "mimi_decoder.onnx"; - - assertFileExists(`${onnxDir}/mimi_encoder.onnx`, "PocketTTS mimi_encoder"); - assertFileExists(`${onnxDir}/text_conditioner.onnx`, "PocketTTS text_conditioner"); - assertFileExists(`${onnxDir}/${flowMainFile}`, "PocketTTS flow_lm_main"); - assertFileExists(`${onnxDir}/${flowFlowFile}`, "PocketTTS flow_lm_flow"); - assertFileExists(`${onnxDir}/${decoderFile}`, "PocketTTS mimi_decoder"); - assertFileExists(tokenizerPath, "PocketTTS tokenizer.model"); - assertFileExists(referenceAudioFile, "PocketTTS reference_sample.wav"); - - const ort = await loadOrt(); - const providers = getOrtProviders(ort, device); - - const [tokenizer, mimiEncoder, textConditioner, flowLmMain, flowLmFlow, mimiDecoder] = - await Promise.all([ - loadSentencePiece(tokenizerPath), - ort.InferenceSession.create(`${onnxDir}/mimi_encoder.onnx`, { - executionProviders: providers, - }), - ort.InferenceSession.create(`${onnxDir}/text_conditioner.onnx`, { - executionProviders: providers, - }), - ort.InferenceSession.create(`${onnxDir}/${flowMainFile}`, { - executionProviders: providers, - }), - ort.InferenceSession.create(`${onnxDir}/${flowFlowFile}`, { - executionProviders: providers, - }), - ort.InferenceSession.create(`${onnxDir}/${decoderFile}`, { executionProviders: providers }), - ]); - - // Precompute flow matching time-step buffers. - const stBuffers: Array<{ s: OrtTensor; t: OrtTensor }> = []; - for (let j = 0; j < lsdSteps; j += 1) { - const s = j / lsdSteps; - const t = s + 1.0 / lsdSteps; - stBuffers.push({ - s: new ort.Tensor("float32", new Float32Array([s]), [1, 1]), - t: new ort.Tensor("float32", new Float32Array([t]), [1, 1]), - }); - } - - // Precompute reference voice embeddings once. - const refWav = await readFile(referenceAudioFile); - const parsed = parsePcm16MonoWav(refWav); - let pcm16 = parsed.pcm16; - if (parsed.sampleRate !== PocketTtsOnnxEngine.SAMPLE_RATE) { - const resampler = new Pcm16MonoResampler({ - inputRate: parsed.sampleRate, - outputRate: PocketTtsOnnxEngine.SAMPLE_RATE, - }); - pcm16 = resampler.processChunk(pcm16); - } - const floatAudio = pcm16leToFloat32(pcm16); - const audioTensor = new ort.Tensor("float32", floatAudio, [1, 1, floatAudio.length]); - - const encoded = await mimiEncoder.run({ audio: audioTensor }); - const firstOutName = mimiEncoder.outputNames?.[0]; - const encodedRecord = encoded as Record; - const voiceEmb = firstOutName ? encodedRecord[firstOutName] : Object.values(encodedRecord)[0]; - if (!voiceEmb) { - throw new Error("PocketTTS mimi_encoder: missing output"); - } - - log.info({ precision, device, providers, lsdSteps, temperature }, "PocketTTS ONNX initialized"); - - return new PocketTtsOnnxEngine({ - ort, - temperature, - lsdSteps, - maxFrames, - framesAfterEos, - firstChunkFrames, - maxChunkFrames, - tokenizer, - textConditioner, - flowLmMain, - flowLmFlow, - mimiDecoder, - stBuffers, - voiceEmbeddings: voiceEmb, - }); - } - - private tokenize(text: string): OrtTensor { - const normalized = normalizeTextForPocket(text); - const ids = this.tokenizer.encodeIds(normalized); - const data = toBigInt64(ids ?? []); - return new this.ort.Tensor("int64", data, [1, data.length]); - } - - private async runTextConditioner(tokenIds: OrtTensor): Promise { - const feed: Record = { token_ids: tokenIds }; - const out = await this.textConditioner.run(feed); - const firstOutName = this.textConditioner.outputNames?.[0]; - const outRecord = out as Record; - const t = firstOutName ? outRecord[firstOutName] : Object.values(outRecord)[0]; - if (!t) throw new Error("PocketTTS text_conditioner: missing output"); - return t; - } - - private async *runFlowLm(textEmbeddings: OrtTensor): AsyncGenerator { - const ort = this.ort; - const state = initState(this.flowLmMain, ort); - - const emptySeq = new ort.Tensor("float32", new Float32Array(0), [1, 0, 32]); - const emptyText = new ort.Tensor("float32", new Float32Array(0), [1, 0, 1024]); - - // Voice conditioning pass - const resVoice = await this.flowLmMain.run( - createSessionFeed({ - sequence: emptySeq, - text_embeddings: this.voiceEmbeddings, - ...state, - }), - ); - updateStateFromOutputs(state, resVoice as Record); - - // Text conditioning pass - const resText = await this.flowLmMain.run( - createSessionFeed({ - sequence: emptySeq, - text_embeddings: textEmbeddings, - ...state, - }), - ); - updateStateFromOutputs(state, resText as Record); - - // Autoregressive generation - const curr = new Float32Array(32); - curr.fill(Number.NaN); - let currTensor = new ort.Tensor("float32", curr, [1, 1, 32]); - - const dt = 1.0 / this.lsdSteps; - let eosStep: number | null = null; - - for (let step = 0; step < this.maxFrames; step += 1) { - const resStep = await this.flowLmMain.run( - createSessionFeed({ - sequence: currTensor, - text_embeddings: emptyText, - ...state, - }), - ); - - const outputNames = this.flowLmMain.outputNames; - const resStepRecord = resStep as Record; - const conditioningName = outputNames?.[0] ?? Object.keys(resStepRecord)[0]; - const eosName = outputNames?.[1] ?? Object.keys(resStepRecord)[1]; - - const conditioning = resStepRecord[conditioningName]; - const eos = resStepRecord[eosName]; - if (!conditioning || !eos) { - throw new Error("PocketTTS flow_lm_main: missing conditioning/EOS outputs"); - } - updateStateFromOutputs(state, resStepRecord); - - const eosData = tensorDataFloat32(eos); - if (eosData[0] > -4.0 && eosStep === null) { - eosStep = step; - } - if (eosStep !== null && step >= eosStep + this.framesAfterEos) { - break; - } - - // Flow matching with external Euler loop. - const std = this.temperature > 0 ? Math.sqrt(this.temperature) : 0; - const x = new Float32Array(32); - if (std > 0) { - for (let i = 0; i < x.length; i += 1) { - x[i] = randn() * std; - } - } - - for (const st of this.stBuffers) { - const xTensor = new ort.Tensor("float32", x, [1, 32]); - const flowOut = await this.flowLmFlow.run( - createSessionFeed({ - c: conditioning, - s: st.s, - t: st.t, - x: xTensor, - }), - ); - const first = this.flowLmFlow.outputNames?.[0]; - const flowOutRecord = flowOut as Record; - const flowTensor = first ? flowOutRecord[first] : Object.values(flowOutRecord)[0]; - if (!flowTensor) throw new Error("PocketTTS flow_lm_flow: missing output"); - const delta = tensorDataFloat32(flowTensor); - for (let i = 0; i < x.length; i += 1) { - x[i] = x[i] + delta[i] * dt; - } - } - - yield x; - currTensor = new ort.Tensor("float32", x, [1, 1, 32]); - } - } - - private async decodeLatentsChunk( - frames: Float32Array[], - state: Record, - ): Promise { - const ort = this.ort; - const frameCount = frames.length; - const flattened = new Float32Array(frameCount * 32); - for (let i = 0; i < frameCount; i += 1) { - flattened.set(frames[i], i * 32); - } - const latent = new ort.Tensor("float32", flattened, [1, frameCount, 32]); - - const out = await this.mimiDecoder.run(createSessionFeed({ latent, ...state })); - const outRecord = out as Record; - updateStateFromOutputs(state, outRecord); - - const firstOutName = this.mimiDecoder.outputNames?.[0]; - const audioTensor = firstOutName ? outRecord[firstOutName] : Object.values(outRecord)[0]; - if (!audioTensor) { - throw new Error("PocketTTS mimi_decoder: missing audio output"); - } - return tensorDataFloat32(audioTensor); - } - - async *streamAudio(text: string): AsyncGenerator { - const tokenIds = this.tokenize(text); - const textEmb = await this.runTextConditioner(tokenIds); - - const decoderState = initState(this.mimiDecoder, this.ort); - - const generated: Float32Array[] = []; - let decodedFrames = 0; - - for await (const latent of this.runFlowLm(textEmb)) { - generated.push(latent); - const pending = generated.length - decodedFrames; - - let chunkSize = 0; - if (decodedFrames === 0) { - if (pending >= this.firstChunkFrames) { - chunkSize = this.firstChunkFrames; - } - } else if (pending >= this.maxChunkFrames) { - chunkSize = this.maxChunkFrames; - } - - if (chunkSize > 0) { - const audio = await this.decodeLatentsChunk( - generated.slice(decodedFrames, decodedFrames + chunkSize), - decoderState, - ); - decodedFrames += chunkSize; - yield audio; - } - } - - if (decodedFrames < generated.length) { - const audio = await this.decodeLatentsChunk(generated.slice(decodedFrames), decoderState); - yield audio; - } - } -} - -export class PocketTtsOnnxTTS implements TextToSpeechProvider { - private readonly engine: PocketTtsOnnxEngine; - private readonly chunkMs: number; - private readonly logger: pino.Logger; - - private constructor(engine: PocketTtsOnnxEngine, logger: pino.Logger, chunkMs: number) { - this.engine = engine; - this.chunkMs = chunkMs; - this.logger = logger.child({ module: "speech", provider: "pocket-tts", component: "tts" }); - } - - static async create(config: PocketTtsOnnxConfig, logger: pino.Logger): Promise { - const engine = await PocketTtsOnnxEngine.create(config, logger); - const chunkMs = config.targetChunkMs ?? 50; - return new PocketTtsOnnxTTS(engine, logger, chunkMs); - } - - async synthesizeSpeech(text: string): Promise { - const start = Date.now(); - const sampleRate = PocketTtsOnnxEngine.SAMPLE_RATE; - - const pcmChunkBytes = Math.max(2, Math.round((sampleRate * this.chunkMs) / 1000) * 2); - - const iterable = (async function* (engine: PocketTtsOnnxEngine) { - for await (const floatChunk of engine.streamAudio(text)) { - const pcm = float32ToPcm16le(floatChunk); - for (const chunk of chunkBuffer(pcm, pcmChunkBytes)) { - yield chunk; - } - } - })(this.engine); - - this.logger.debug( - { ms: Date.now() - start, textLength: text.length }, - "PocketTTS stream ready", - ); - - return { - stream: Readable.from(iterable), - format: `pcm;rate=${sampleRate}`, - }; - } -} diff --git a/packages/server/src/server/speech/providers/local/runtime.ts b/packages/server/src/server/speech/providers/local/runtime.ts index 7056baade..23c6d3bc1 100644 --- a/packages/server/src/server/speech/providers/local/runtime.ts +++ b/packages/server/src/server/speech/providers/local/runtime.ts @@ -4,7 +4,6 @@ import type { PaseoSpeechConfig } from "../../../bootstrap.js"; import type { SpeechToTextProvider, TextToSpeechProvider } from "../../speech-provider.js"; import type { RequestedSpeechProviders } from "../../speech-types.js"; import type { TurnDetectionProvider } from "../../turn-detection-provider.js"; -import { PocketTtsOnnxTTS } from "./pocket/pocket-tts-onnx.js"; import { getLocalSpeechModelDir, DEFAULT_LOCAL_STT_MODEL, @@ -16,20 +15,15 @@ import { type LocalTtsModelId, } from "./models.js"; import { SherpaOfflineRecognizerEngine } from "./sherpa/sherpa-offline-recognizer.js"; -import { SherpaOnlineRecognizerEngine } from "./sherpa/sherpa-online-recognizer.js"; import { SherpaOnnxParakeetSTT } from "./sherpa/sherpa-parakeet-stt.js"; import { SherpaParakeetRealtimeTranscriptionSession } from "./sherpa/sherpa-parakeet-realtime-session.js"; -import { SherpaRealtimeTranscriptionSession } from "./sherpa/sherpa-realtime-session.js"; -import { SherpaOnnxSTT } from "./sherpa/sherpa-stt.js"; import { SherpaOnnxTTS } from "./sherpa/sherpa-tts.js"; import { ensureSileroVadModel, SherpaSileroTurnDetectionProvider, } from "./sherpa/silero-vad-provider.js"; -type LocalSttEngine = - | { kind: "offline"; engine: SherpaOfflineRecognizerEngine } - | { kind: "online"; engine: SherpaOnlineRecognizerEngine }; +type LocalSttEngine = SherpaOfflineRecognizerEngine; interface ResolvedLocalModels { dictationLocalSttModel: LocalSttModelId; @@ -117,70 +111,21 @@ async function createLocalSttEngine(params: { }): Promise { const { modelId, modelsDir, logger } = params; - if (modelId === "parakeet-tdt-0.6b-v3-int8" || modelId === "parakeet-tdt-0.6b-v2-int8") { - const modelDir = getLocalSpeechModelDir(modelsDir, modelId); - return { - kind: "offline", - engine: new SherpaOfflineRecognizerEngine( - { - model: { - kind: "nemo_transducer", - encoder: `${modelDir}/encoder.int8.onnx`, - decoder: `${modelDir}/decoder.int8.onnx`, - joiner: `${modelDir}/joiner.int8.onnx`, - tokens: `${modelDir}/tokens.txt`, - }, - numThreads: 2, - debug: 0, - }, - logger, - ), - }; - } - - if (modelId === "paraformer-bilingual-zh-en") { - const modelDir = getLocalSpeechModelDir(modelsDir, modelId); - return { - kind: "online", - engine: new SherpaOnlineRecognizerEngine( - { - model: { - kind: "paraformer", - encoder: `${modelDir}/encoder.int8.onnx`, - decoder: `${modelDir}/decoder.int8.onnx`, - tokens: `${modelDir}/tokens.txt`, - }, - numThreads: 1, - debug: 0, - }, - logger, - ), - }; - } - - if (modelId === "zipformer-bilingual-zh-en-2023-02-20") { - const modelDir = getLocalSpeechModelDir(modelsDir, modelId); - return { - kind: "online", - engine: new SherpaOnlineRecognizerEngine( - { - model: { - kind: "transducer", - encoder: `${modelDir}/encoder-epoch-99-avg-1.onnx`, - decoder: `${modelDir}/decoder-epoch-99-avg-1.onnx`, - joiner: `${modelDir}/joiner-epoch-99-avg-1.onnx`, - tokens: `${modelDir}/tokens.txt`, - modelType: "zipformer", - }, - numThreads: 1, - debug: 0, - }, - logger, - ), - }; - } - - throw new Error(`Unsupported local STT model '${modelId}'`); + const modelDir = getLocalSpeechModelDir(modelsDir, modelId); + return new SherpaOfflineRecognizerEngine( + { + model: { + kind: "nemo_transducer", + encoder: `${modelDir}/encoder.int8.onnx`, + decoder: `${modelDir}/decoder.int8.onnx`, + joiner: `${modelDir}/joiner.int8.onnx`, + tokens: `${modelDir}/tokens.txt`, + }, + numThreads: 2, + debug: 0, + }, + logger, + ); } type LocalConfig = NonNullable; @@ -219,13 +164,7 @@ async function initializeLocalVoiceStt(params: { return null; } const voiceEngine = await getLocalSttEngine(modelId); - if (voiceEngine?.kind === "offline") { - return new SherpaOnnxParakeetSTT({ engine: voiceEngine.engine }, logger); - } - if (voiceEngine?.kind === "online") { - return new SherpaOnnxSTT({ engine: voiceEngine.engine }, logger); - } - return null; + return voiceEngine ? new SherpaOnnxParakeetSTT({ engine: voiceEngine }, logger) : null; } async function initializeLocalDictationStt(params: { @@ -243,18 +182,11 @@ async function initializeLocalDictationStt(params: { return null; } const dictationEngine = await getLocalSttEngine(modelId); - if (dictationEngine?.kind === "offline") { + if (dictationEngine) { return { id: "local", createSession: () => - new SherpaParakeetRealtimeTranscriptionSession({ engine: dictationEngine.engine }), - }; - } - if (dictationEngine?.kind === "online") { - return { - id: "local", - createSession: () => - new SherpaRealtimeTranscriptionSession({ engine: dictationEngine.engine }), + new SherpaParakeetRealtimeTranscriptionSession({ engine: dictationEngine }), }; } return null; @@ -276,16 +208,6 @@ async function initializeLocalVoiceTts(params: { } try { const modelDir = getLocalSpeechModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel); - if (localModels.voiceLocalTtsModel === "pocket-tts-onnx-int8") { - return await PocketTtsOnnxTTS.create( - { - modelDir, - precision: "int8", - targetChunkMs: 50, - }, - logger, - ); - } return new SherpaOnnxTTS( { preset: localModels.voiceLocalTtsModel, @@ -401,7 +323,7 @@ export async function initializeLocalSpeechServices(params: { maybeFreeable.free(); } for (const engine of localSttEngines.values()) { - engine.engine.free(); + engine.free(); } }; diff --git a/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts b/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts index 14eb4e47e..c1bbf3b62 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts @@ -1,44 +1,19 @@ import { z } from "zod"; -export type SherpaOnnxModelKind = "stt-online" | "stt-offline" | "tts"; +export type SherpaOnnxModelKind = "stt-offline" | "tts"; type DefaultModelRole = "stt" | "tts"; interface SherpaOnnxCatalogEntry { kind: SherpaOnnxModelKind; - archiveUrl?: string; - downloadFiles?: Array<{ url: string; relPath: string }>; + archiveUrl: string; extractedDir: string; requiredFiles: string[]; description: string; - aliases?: readonly string[]; defaultFor?: DefaultModelRole; } export const SHERPA_ONNX_MODEL_CATALOG = { - "zipformer-bilingual-zh-en-2023-02-20": { - kind: "stt-online", - archiveUrl: - "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2", - extractedDir: "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", - requiredFiles: [ - "encoder-epoch-99-avg-1.onnx", - "decoder-epoch-99-avg-1.onnx", - "joiner-epoch-99-avg-1.onnx", - "tokens.txt", - ], - description: "Streaming Zipformer transducer (fast, good accuracy).", - aliases: ["zipformer", "zipformer-bilingual"], - }, - "paraformer-bilingual-zh-en": { - kind: "stt-online", - archiveUrl: - "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2", - extractedDir: "sherpa-onnx-streaming-paraformer-bilingual-zh-en", - requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "tokens.txt"], - description: "Streaming Paraformer (often strong accuracy; heavier).", - aliases: ["paraformer"], - }, "parakeet-tdt-0.6b-v2-int8": { kind: "stt-offline", archiveUrl: @@ -46,27 +21,8 @@ export const SHERPA_ONNX_MODEL_CATALOG = { extractedDir: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8", requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", "tokens.txt"], description: "NVIDIA Parakeet TDT v2 (offline NeMo transducer, English).", - aliases: ["parakeet-v2", "parakeet-tdt-v2"], defaultFor: "stt", }, - "parakeet-tdt-0.6b-v3-int8": { - kind: "stt-offline", - archiveUrl: - "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2", - extractedDir: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8", - requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", "tokens.txt"], - description: "NVIDIA Parakeet TDT v3 (offline NeMo transducer, multilingual).", - aliases: ["parakeet", "parakeet-v3", "parakeet-tdt"], - }, - "kitten-nano-en-v0_1-fp16": { - kind: "tts", - archiveUrl: - "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2", - extractedDir: "kitten-nano-en-v0_1-fp16", - requiredFiles: ["model.fp16.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"], - description: "KittenTTS (small, fast English TTS).", - aliases: ["kitten"], - }, "kokoro-en-v0_19": { kind: "tts", archiveUrl: @@ -74,54 +30,8 @@ export const SHERPA_ONNX_MODEL_CATALOG = { extractedDir: "kokoro-en-v0_19", requiredFiles: ["model.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"], description: "Kokoro TTS (higher quality; larger).", - aliases: ["kokoro"], defaultFor: "tts", }, - "pocket-tts-onnx-int8": { - kind: "tts", - extractedDir: "pocket-tts-onnx-int8", - downloadFiles: [ - { - url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/mimi_encoder.onnx", - relPath: "onnx/mimi_encoder.onnx", - }, - { - url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/text_conditioner.onnx", - relPath: "onnx/text_conditioner.onnx", - }, - { - url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/flow_lm_main_int8.onnx", - relPath: "onnx/flow_lm_main_int8.onnx", - }, - { - url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/flow_lm_flow_int8.onnx", - relPath: "onnx/flow_lm_flow_int8.onnx", - }, - { - url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/mimi_decoder_int8.onnx", - relPath: "onnx/mimi_decoder_int8.onnx", - }, - { - url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/tokenizer.model", - relPath: "tokenizer.model", - }, - { - url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/reference_sample.wav", - relPath: "reference_sample.wav", - }, - ], - requiredFiles: [ - "onnx/mimi_encoder.onnx", - "onnx/text_conditioner.onnx", - "onnx/flow_lm_main_int8.onnx", - "onnx/flow_lm_flow_int8.onnx", - "onnx/mimi_decoder_int8.onnx", - "tokenizer.model", - "reference_sample.wav", - ], - description: "Pocket TTS ONNX (INT8) with streaming decode support (via onnxruntime).", - aliases: ["pocket", "pocket-tts"], - }, } as const satisfies Record; export type SherpaOnnxModelId = keyof typeof SHERPA_ONNX_MODEL_CATALOG; @@ -131,7 +41,7 @@ type ModelIdByKind = { [Id in SherpaOnnxModelId]: (typeof SHERPA_ONNX_MODEL_CATALOG)[Id]["kind"] extends K ? Id : never; }[SherpaOnnxModelId]; -export type LocalSttModelId = ModelIdByKind<"stt-online"> | ModelIdByKind<"stt-offline">; +export type LocalSttModelId = ModelIdByKind<"stt-offline">; export type LocalTtsModelId = ModelIdByKind<"tts">; const ALL_MODEL_IDS: SherpaOnnxModelId[] = Object.keys(SHERPA_ONNX_MODEL_CATALOG).filter( @@ -166,48 +76,22 @@ function resolveDefaultModelId(role: DefaultModelRole): SherpaOnnxModelId { export const DEFAULT_LOCAL_STT_MODEL = resolveDefaultModelId("stt"); export const DEFAULT_LOCAL_TTS_MODEL = resolveDefaultModelId("tts"); -function buildAliasMap(modelIds: readonly T[]): Record { - const aliasMap: Record = {}; - for (const modelId of modelIds) { - const aliases = SHERPA_ONNX_MODEL_CATALOG[modelId].aliases ?? []; - for (const alias of aliases) { - aliasMap[alias.trim().toLowerCase()] = modelId; - } - } - return aliasMap; -} - -function createAliasedModelIdSchema(params: { - modelIds: readonly T[]; - aliases: Record; -}): z.ZodType { - const validIds = new Set(params.modelIds); +function createModelIdSchema( + modelIds: readonly T[], +): z.ZodType { + const validIds = new Set(modelIds); return z .string() .trim() .toLowerCase() - .refine( - (value): value is T => - validIds.has(value) || Object.prototype.hasOwnProperty.call(params.aliases, value), - { - message: "Invalid model id", - }, - ) - .transform((value) => params.aliases[value] ?? value); + .refine((value) => validIds.has(value), { + message: "Invalid model id", + }) + .transform((value) => value as T); } -const STT_MODEL_ALIASES = buildAliasMap(LOCAL_STT_MODEL_IDS); -const TTS_MODEL_ALIASES = buildAliasMap(LOCAL_TTS_MODEL_IDS); - -export const LocalSttModelIdSchema = createAliasedModelIdSchema({ - modelIds: LOCAL_STT_MODEL_IDS, - aliases: STT_MODEL_ALIASES, -}); - -export const LocalTtsModelIdSchema = createAliasedModelIdSchema({ - modelIds: LOCAL_TTS_MODEL_IDS, - aliases: TTS_MODEL_ALIASES, -}); +export const LocalSttModelIdSchema = createModelIdSchema(LOCAL_STT_MODEL_IDS); +export const LocalTtsModelIdSchema = createModelIdSchema(LOCAL_TTS_MODEL_IDS); export type SherpaOnnxModelSpec = SherpaOnnxCatalogEntry & { id: SherpaOnnxModelId; diff --git a/packages/server/src/server/speech/providers/local/sherpa/model-downloader.test.ts b/packages/server/src/server/speech/providers/local/sherpa/model-downloader.test.ts index ec4a9e83f..993c959b4 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/model-downloader.test.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/model-downloader.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test, vi } from "vitest"; +import { describe, expect, test } from "vitest"; import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -15,69 +15,27 @@ const logger = pino({ level: "silent" }); describe("sherpa model downloader", () => { test("getSherpaOnnxModelDir maps modelId to extractedDir", () => { const modelsDir = "/tmp/models"; - expect(getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v3-int8")).toContain( - "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8", - ); - expect(getSherpaOnnxModelDir(modelsDir, "pocket-tts-onnx-int8")).toContain( - "pocket-tts-onnx-int8", + expect(getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v2-int8")).toContain( + "sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8", ); + expect(getSherpaOnnxModelDir(modelsDir, "kokoro-en-v0_19")).toContain("kokoro-en-v0_19"); }); test("ensureSherpaOnnxModel succeeds without downloading when files exist", async () => { const modelsDir = makeTmpDir(); - const modelDir = getSherpaOnnxModelDir(modelsDir, "kitten-nano-en-v0_1-fp16"); + const modelDir = getSherpaOnnxModelDir(modelsDir, "kokoro-en-v0_19"); mkdirSync(path.join(modelDir, "espeak-ng-data"), { recursive: true }); - writeFileSync(path.join(modelDir, "model.fp16.onnx"), "x"); + writeFileSync(path.join(modelDir, "model.onnx"), "x"); writeFileSync(path.join(modelDir, "voices.bin"), "x"); writeFileSync(path.join(modelDir, "tokens.txt"), "x"); const out = await ensureSherpaOnnxModel({ modelsDir, - modelId: "kitten-nano-en-v0_1-fp16", + modelId: "kokoro-en-v0_19", logger, }); expect(out).toBe(modelDir); }); - - test("ensureSherpaOnnxModel logs lifecycle events without progress spam", async () => { - const modelsDir = makeTmpDir(); - const infoMessages: string[] = []; - - const loggerWithSpy = { - child: () => loggerWithSpy, - info: (_obj?: unknown, msg?: string) => { - if (typeof msg === "string") { - infoMessages.push(msg); - } - }, - error: () => undefined, - } as unknown as pino.Logger; - - const originalFetch = globalThis.fetch; - const payload = Buffer.alloc(128 * 1024, 7); - const fetchMock = vi.fn(async () => { - return new Response(payload, { - status: 200, - headers: { "content-length": String(payload.length) }, - }); - }); - globalThis.fetch = fetchMock as typeof fetch; - - try { - await ensureSherpaOnnxModel({ - modelsDir, - modelId: "pocket-tts-onnx-int8", - logger: loggerWithSpy, - }); - } finally { - globalThis.fetch = originalFetch; - } - - expect(fetchMock).toHaveBeenCalled(); - expect(infoMessages).toContain("Starting model download"); - expect(infoMessages).toContain("Model download completed"); - expect(infoMessages).not.toContain("Downloading model artifact"); - }); }); diff --git a/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts b/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts index 232c55be5..393237d37 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts @@ -111,92 +111,55 @@ export async function ensureSherpaOnnxModel( logger.info({ modelsDir: options.modelsDir }, "Starting model download"); try { - if (spec.archiveUrl) { - const downloadsDir = path.join(options.modelsDir, ".downloads"); - const archiveFilename = path.basename(new URL(spec.archiveUrl).pathname); - const archivePath = path.join(downloadsDir, archiveFilename); + const downloadsDir = path.join(options.modelsDir, ".downloads"); + const archiveFilename = path.basename(new URL(spec.archiveUrl).pathname); + const archivePath = path.join(downloadsDir, archiveFilename); - if (!(await isNonEmptyFile(archivePath))) { - await downloadToFile({ - url: spec.archiveUrl, - outputPath: archivePath, - }); - } - - logger.info( - { - modelId: options.modelId, - archivePath, - modelDir, - }, - "Extracting model archive", - ); - await extractTarArchive(archivePath, options.modelsDir); - - logger.info( - { - modelId: options.modelId, - modelDir, - }, - "Verifying downloaded model files", - ); - if (!(await hasRequiredFiles(modelDir, spec.requiredFiles))) { - throw new Error( - `Downloaded and extracted ${archiveFilename}, but required files are still missing in ${modelDir}.`, - ); - } - - logger.info( - { - modelId: options.modelId, - archivePath, - }, - "Finalizing model artifacts", - ); - try { - await rm(archivePath, { force: true }); - } catch { - // ignore - } - - logger.info({ modelDir }, "Model download completed"); - return modelDir; + if (!(await isNonEmptyFile(archivePath))) { + await downloadToFile({ + url: spec.archiveUrl, + outputPath: archivePath, + }); } - if (spec.downloadFiles && spec.downloadFiles.length > 0) { - await mkdir(modelDir, { recursive: true }); + logger.info( + { + modelId: options.modelId, + archivePath, + modelDir, + }, + "Extracting model archive", + ); + await extractTarArchive(archivePath, options.modelsDir); - await Promise.all( - spec.downloadFiles.map(async (file) => { - const dst = path.join(modelDir, file.relPath); - if (await isNonEmptyFile(dst)) { - return; - } - await downloadToFile({ - url: file.url, - outputPath: dst, - }); - }), + logger.info( + { + modelId: options.modelId, + modelDir, + }, + "Verifying downloaded model files", + ); + if (!(await hasRequiredFiles(modelDir, spec.requiredFiles))) { + throw new Error( + `Downloaded and extracted ${archiveFilename}, but required files are still missing in ${modelDir}.`, ); - - logger.info( - { - modelId: options.modelId, - modelDir, - }, - "Verifying downloaded model files", - ); - if (!(await hasRequiredFiles(modelDir, spec.requiredFiles))) { - throw new Error( - `Downloaded files for ${options.modelId}, but required files are still missing in ${modelDir}.`, - ); - } - - logger.info({ modelDir }, "Model download completed"); - return modelDir; } - throw new Error(`Model spec for ${options.modelId} has no archiveUrl or downloadFiles`); + logger.info( + { + modelId: options.modelId, + archivePath, + }, + "Finalizing model artifacts", + ); + try { + await rm(archivePath, { force: true }); + } catch { + // ignore + } + + logger.info({ modelDir }, "Model download completed"); + return modelDir; } catch (error) { logger.error({ err: error }, "Model download failed"); throw error; diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-online-recognizer.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-online-recognizer.ts deleted file mode 100644 index 6c728ec3c..000000000 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-online-recognizer.ts +++ /dev/null @@ -1,157 +0,0 @@ -import { existsSync } from "node:fs"; -import type pino from "pino"; - -import { loadSherpaOnnx } from "./sherpa-onnx-loader.js"; - -export type SherpaOnlineRecognizerModel = - | { - kind: "transducer"; - encoder: string; - decoder: string; - joiner: string; - tokens: string; - modelType?: string; - } - | { - kind: "paraformer"; - encoder: string; - decoder: string; - tokens: string; - }; - -export interface SherpaOnlineRecognizerConfig { - model: SherpaOnlineRecognizerModel; - numThreads?: number; - provider?: "cpu"; - debug?: 0 | 1; - sampleRate?: number; - featureDim?: number; - decodingMethod?: "greedy_search"; - maxActivePaths?: number; - enableEndpoint?: 0 | 1; - rule1MinTrailingSilence?: number; - rule2MinTrailingSilence?: number; - rule3MinUtteranceLength?: number; -} - -function assertFileExists(filePath: string, label: string): void { - if (!existsSync(filePath)) { - throw new Error(`Missing ${label}: ${filePath}`); - } -} - -function validateModelFiles(model: SherpaOnlineRecognizerModel): void { - if (model.kind === "transducer") { - assertFileExists(model.encoder, "transducer encoder"); - assertFileExists(model.decoder, "transducer decoder"); - assertFileExists(model.joiner, "transducer joiner"); - assertFileExists(model.tokens, "tokens"); - } else { - assertFileExists(model.encoder, "paraformer encoder"); - assertFileExists(model.decoder, "paraformer decoder"); - assertFileExists(model.tokens, "tokens"); - } -} - -function buildModelConfig(model: SherpaOnlineRecognizerModel): object { - if (model.kind === "transducer") { - return { - transducer: { - encoder: model.encoder, - decoder: model.decoder, - joiner: model.joiner, - }, - tokens: model.tokens, - modelType: model.modelType ?? "zipformer", - }; - } - return { - paraformer: { - encoder: model.encoder, - decoder: model.decoder, - }, - tokens: model.tokens, - }; -} - -export interface SherpaOnlineStreamNative { - acceptWaveform: (sampleRate: number, samples: Float32Array) => void; - free?: () => void; -} - -export interface SherpaOnlineRecognizerNative { - config?: { featConfig?: { sampleRate?: number } }; - createStream: () => SherpaOnlineStreamNative; - isReady: (stream: SherpaOnlineStreamNative) => boolean; - decode: (stream: SherpaOnlineStreamNative) => void; - getResult: (stream: SherpaOnlineStreamNative) => { text?: string } | string | undefined; - reset?: (stream: SherpaOnlineStreamNative) => void; - free?: () => void; -} - -export class SherpaOnlineRecognizerEngine { - public readonly recognizer: SherpaOnlineRecognizerNative; - public readonly sampleRate: number; - private readonly logger: pino.Logger; - - constructor(config: SherpaOnlineRecognizerConfig, logger: pino.Logger) { - this.logger = logger.child({ - module: "speech", - provider: "local", - component: "online-recognizer", - }); - - const { model } = config; - validateModelFiles(model); - - const sherpa = loadSherpaOnnx(); - const modelConfig = buildModelConfig(model); - - const featConfig = { - sampleRate: config.sampleRate ?? 16000, - featureDim: config.featureDim ?? 80, - }; - - const recognizerConfig = { - featConfig, - modelConfig: { - ...modelConfig, - // NOTE: In the WASM-backed `sherpa-onnx` npm package, online recognizers - // error when `numThreads > 1`. Keep the default conservative. - numThreads: config.numThreads ?? 1, - provider: config.provider ?? "cpu", - debug: config.debug ?? 0, - }, - decodingMethod: config.decodingMethod ?? "greedy_search", - maxActivePaths: config.maxActivePaths ?? 4, - enableEndpoint: config.enableEndpoint ?? 0, - rule1MinTrailingSilence: config.rule1MinTrailingSilence ?? 2.4, - rule2MinTrailingSilence: config.rule2MinTrailingSilence ?? 1.2, - rule3MinUtteranceLength: config.rule3MinUtteranceLength ?? 20, - }; - - this.recognizer = sherpa.createOnlineRecognizer( - recognizerConfig, - ) as SherpaOnlineRecognizerNative; - const sr = this.recognizer?.config?.featConfig?.sampleRate; - this.sampleRate = - typeof sr === "number" && Number.isFinite(sr) && sr > 0 ? sr : featConfig.sampleRate; - - this.logger.info( - { sampleRate: this.sampleRate, modelKind: model.kind, numThreads: config.numThreads ?? 2 }, - "Sherpa online recognizer initialized", - ); - } - - createStream(): SherpaOnlineStreamNative { - return this.recognizer.createStream(); - } - - free(): void { - try { - this.recognizer?.free?.(); - } catch (err) { - this.logger.warn({ err }, "Failed to free sherpa recognizer"); - } - } -} diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-loader.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-loader.ts deleted file mode 100644 index a8c25ba5f..000000000 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-loader.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { createRequire } from "node:module"; - -export interface SherpaOnnxModule { - createOnlineRecognizer: (config: unknown) => unknown; - createOfflineRecognizer: (config: unknown) => unknown; - createOfflineTts: (config: unknown) => unknown; -} - -let cached: SherpaOnnxModule | null = null; - -export function loadSherpaOnnx(): SherpaOnnxModule { - if (cached) { - return cached; - } - - const require = createRequire(import.meta.url); - cached = require("sherpa-onnx") as SherpaOnnxModule; - return cached; -} diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-realtime-session.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-realtime-session.ts deleted file mode 100644 index 4106d96bd..000000000 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-realtime-session.ts +++ /dev/null @@ -1,146 +0,0 @@ -import { EventEmitter } from "node:events"; -import { v4 as uuidv4 } from "uuid"; - -import type { StreamingTranscriptionSession } from "../../../speech-provider.js"; -import { pcm16lePeakAbs, pcm16leToFloat32 } from "../../../audio.js"; -import { - SherpaOnlineRecognizerEngine, - type SherpaOnlineStreamNative, -} from "./sherpa-online-recognizer.js"; - -export class SherpaRealtimeTranscriptionSession - extends EventEmitter - implements StreamingTranscriptionSession -{ - private readonly engine: SherpaOnlineRecognizerEngine; - private stream: SherpaOnlineStreamNative | null = null; - private connected = false; - - public readonly requiredSampleRate: number; - private currentSegmentId: string | null = null; - private previousSegmentId: string | null = null; - private lastPartialText = ""; - private readonly tailPaddingMs: number; - - constructor(params: { engine: SherpaOnlineRecognizerEngine; tailPaddingMs?: number }) { - super(); - this.engine = params.engine; - this.requiredSampleRate = this.engine.sampleRate; - this.tailPaddingMs = params.tailPaddingMs ?? 500; - } - - async connect(): Promise { - if (this.connected) { - return; - } - this.stream = this.engine.createStream(); - this.currentSegmentId = uuidv4(); - this.connected = true; - } - - appendPcm16(pcm16le: Buffer): void { - if (!this.connected || !this.stream || !this.currentSegmentId) { - this.emit("error", new Error("Sherpa realtime session not connected")); - return; - } - - try { - const peak = pcm16lePeakAbs(pcm16le); - const peakFloat = peak / 32768.0; - const targetPeak = 0.6; - const maxGain = 50; - const gain = - peakFloat > 0 && peakFloat < targetPeak ? Math.min(maxGain, targetPeak / peakFloat) : 1; - const floatSamples = pcm16leToFloat32(pcm16le, gain); - this.stream.acceptWaveform(this.engine.sampleRate, floatSamples); - - while (this.engine.recognizer.isReady(this.stream)) { - this.engine.recognizer.decode(this.stream); - } - - const rawResult = this.engine.recognizer.getResult(this.stream); - const text = ( - (typeof rawResult === "object" && rawResult && "text" in rawResult - ? rawResult.text - : undefined) ?? "" - ).trim(); - if (text !== this.lastPartialText) { - this.lastPartialText = text; - this.emit("transcript", { - segmentId: this.currentSegmentId, - transcript: text, - isFinal: false, - }); - } - } catch (err) { - this.emit("error", err instanceof Error ? err : new Error(String(err))); - } - } - - commit(): void { - if (!this.connected || !this.stream || !this.currentSegmentId) { - this.emit("error", new Error("Sherpa realtime session not connected")); - return; - } - - try { - const padSamples = Math.max( - 0, - Math.round((this.engine.sampleRate * this.tailPaddingMs) / 1000), - ); - if (padSamples > 0) { - this.stream.acceptWaveform(this.engine.sampleRate, new Float32Array(padSamples)); - } - - while (this.engine.recognizer.isReady(this.stream)) { - this.engine.recognizer.decode(this.stream); - } - - const rawFinal = this.engine.recognizer.getResult(this.stream); - const finalText = ( - (typeof rawFinal === "object" && rawFinal && "text" in rawFinal - ? rawFinal.text - : undefined) ?? "" - ).trim(); - const segmentId = this.currentSegmentId; - const previousSegmentId = this.previousSegmentId; - - this.emit("committed", { segmentId, previousSegmentId }); - this.emit("transcript", { segmentId, transcript: finalText, isFinal: true }); - - this.previousSegmentId = segmentId; - this.currentSegmentId = uuidv4(); - this.lastPartialText = ""; - this.engine.recognizer.reset?.(this.stream); - } catch (err) { - this.emit("error", err instanceof Error ? err : new Error(String(err))); - } - } - - clear(): void { - if (!this.connected || !this.stream) { - return; - } - try { - this.engine.recognizer.reset?.(this.stream); - this.currentSegmentId = uuidv4(); - this.lastPartialText = ""; - } catch (err) { - this.emit("error", err instanceof Error ? err : new Error(String(err))); - } - } - - close(): void { - if (!this.stream) { - return; - } - try { - this.stream.free?.(); - } catch { - // ignore - } finally { - this.stream = null; - this.connected = false; - } - } -} diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-stt.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-stt.ts deleted file mode 100644 index 12c1e73c0..000000000 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-stt.ts +++ /dev/null @@ -1,184 +0,0 @@ -import { EventEmitter } from "node:events"; -import { v4 as uuidv4 } from "uuid"; -import type pino from "pino"; - -import type { - SpeechToTextProvider, - StreamingTranscriptionSession, - TranscriptionResult, -} from "../../../speech-provider.js"; -import { Pcm16MonoResampler } from "../../../../agent/pcm16-resampler.js"; -import { - parsePcm16MonoWav, - parsePcmRateFromFormat, - pcm16lePeakAbs, - pcm16leToFloat32, -} from "../../../audio.js"; -import { SherpaOnlineRecognizerEngine } from "./sherpa-online-recognizer.js"; - -export interface SherpaSttConfig { - engine: SherpaOnlineRecognizerEngine; - silencePeakThreshold?: number; - tailPaddingMs?: number; -} - -export class SherpaOnnxSTT implements SpeechToTextProvider { - private readonly engine: SherpaOnlineRecognizerEngine; - private readonly silencePeakThreshold: number; - private readonly tailPaddingMs: number; - private readonly logger: pino.Logger; - public readonly id = "local" as const; - - constructor(config: SherpaSttConfig, logger: pino.Logger) { - this.engine = config.engine; - this.silencePeakThreshold = config.silencePeakThreshold ?? 300; - this.tailPaddingMs = config.tailPaddingMs ?? 500; - this.logger = logger.child({ module: "speech", provider: "local", component: "stt" }); - } - - public createSession(params: { - logger: pino.Logger; - language?: string; - prompt?: string; - }): StreamingTranscriptionSession { - const emitter = new EventEmitter(); - void params; - const requiredSampleRate = this.engine.sampleRate; - let connected = false; - let segmentId = uuidv4(); - let previousSegmentId: string | null = null; - let pcm16: Buffer = Buffer.alloc(0); - - return { - requiredSampleRate, - async connect() { - connected = true; - }, - appendPcm16(chunk: Buffer) { - if (!connected) { - emitter.emit("error", new Error("STT session not connected")); - return; - } - pcm16 = pcm16.length === 0 ? chunk : Buffer.concat([pcm16, chunk]); - }, - commit: () => { - if (!connected) { - emitter.emit("error", new Error("STT session not connected")); - return; - } - - const committedId = segmentId; - const prev = previousSegmentId; - emitter.emit("committed", { segmentId: committedId, previousSegmentId: prev }); - - void (async () => { - try { - const rt = await this.transcribeAudio(pcm16, `audio/pcm;rate=${requiredSampleRate}`); - emitter.emit("transcript", { - segmentId: committedId, - transcript: rt.text, - isFinal: true, - language: rt.language, - logprobs: rt.logprobs, - avgLogprob: rt.avgLogprob, - isLowConfidence: rt.isLowConfidence, - }); - } catch (err) { - emitter.emit("error", err); - } finally { - previousSegmentId = committedId; - segmentId = uuidv4(); - pcm16 = Buffer.alloc(0); - } - })(); - }, - clear() { - pcm16 = Buffer.alloc(0); - segmentId = uuidv4(); - }, - close() { - connected = false; - pcm16 = Buffer.alloc(0); - }, - on(event: "committed" | "transcript" | "error", handler: (payload: never) => void) { - emitter.on(event, handler as (...args: unknown[]) => void); - return undefined; - }, - }; - } - - public async transcribeAudio(audioBuffer: Buffer, format: string): Promise { - const start = Date.now(); - - let inputRate: number; - let pcm16: Buffer; - - if (format.toLowerCase().includes("audio/wav")) { - const parsed = parsePcm16MonoWav(audioBuffer); - inputRate = parsed.sampleRate; - pcm16 = parsed.pcm16; - } else if (format.toLowerCase().includes("audio/pcm")) { - inputRate = parsePcmRateFromFormat(format, this.engine.sampleRate) ?? this.engine.sampleRate; - pcm16 = audioBuffer; - } else { - throw new Error(`Unsupported audio format for sherpa STT: ${format}`); - } - - const peak = pcm16lePeakAbs(pcm16); - if (peak < this.silencePeakThreshold) { - return { text: "", duration: Date.now() - start, isLowConfidence: true }; - } - - let pcmForModel = pcm16; - if (inputRate !== this.engine.sampleRate) { - const resampler = new Pcm16MonoResampler({ inputRate, outputRate: this.engine.sampleRate }); - pcmForModel = resampler.processChunk(pcm16); - inputRate = this.engine.sampleRate; - } - - const peakForModel = pcm16lePeakAbs(pcmForModel); - const peakFloat = peakForModel / 32768.0; - const targetPeak = 0.6; - const maxGain = 50; - const gain = - peakFloat > 0 && peakFloat < targetPeak ? Math.min(maxGain, targetPeak / peakFloat) : 1; - - const stream = this.engine.createStream(); - try { - const floatSamples = pcm16leToFloat32(pcmForModel, gain); - stream.acceptWaveform(inputRate, floatSamples); - - while (this.engine.recognizer.isReady(stream)) { - this.engine.recognizer.decode(stream); - } - - const padSamples = Math.max( - 0, - Math.round((this.engine.sampleRate * this.tailPaddingMs) / 1000), - ); - if (padSamples > 0) { - stream.acceptWaveform(this.engine.sampleRate, new Float32Array(padSamples)); - } - - while (this.engine.recognizer.isReady(stream)) { - this.engine.recognizer.decode(stream); - } - - const rawResult = this.engine.recognizer.getResult(stream); - const text = ( - (typeof rawResult === "object" && rawResult && "text" in rawResult - ? rawResult.text - : undefined) ?? "" - ).trim(); - const duration = Date.now() - start; - this.logger.debug({ duration, textLength: text.length }, "Sherpa transcription complete"); - return { text, duration, ...(text.length === 0 ? { isLowConfidence: true } : {}) }; - } finally { - try { - stream.free?.(); - } catch { - // ignore - } - } - } -} diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts index 12a25f485..e212ba6af 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts @@ -6,7 +6,7 @@ import type { SpeechStreamResult, TextToSpeechProvider } from "../../../speech-p import { chunkBuffer, float32ToPcm16le } from "../../../audio.js"; import { loadSherpaOnnxNode } from "./sherpa-onnx-node-loader.js"; -export type SherpaTtsPreset = "kokoro-en-v0_19" | "kitten-nano-en-v0_1-fp16"; +export type SherpaTtsPreset = "kokoro-en-v0_19"; export interface SherpaTtsConfig { preset: SherpaTtsPreset; @@ -41,9 +41,6 @@ export class SherpaOnnxTTS implements TextToSpeechProvider { private readonly logger: pino.Logger; constructor(config: SherpaTtsConfig, logger: pino.Logger) { - if (config.preset !== "kokoro-en-v0_19" && config.preset !== "kitten-nano-en-v0_1-fp16") { - throw new Error(`Unsupported Sherpa TTS preset: ${config.preset}`); - } this.logger = logger.child({ module: "speech", provider: "local", component: "tts" }); this.speakerId = config.speakerId ?? 0; this.speed = config.speed ?? 1.0; @@ -53,8 +50,7 @@ export class SherpaOnnxTTS implements TextToSpeechProvider { throw new Error("sherpa-onnx-node OfflineTts is unavailable"); } - const modelFile = config.preset === "kokoro-en-v0_19" ? "model.onnx" : "model.fp16.onnx"; - const modelPath = `${config.modelDir}/${modelFile}`; + const modelPath = `${config.modelDir}/model.onnx`; const voicesPath = `${config.modelDir}/voices.bin`; const tokensPath = `${config.modelDir}/tokens.txt`; const dataDir = `${config.modelDir}/espeak-ng-data`; @@ -64,26 +60,15 @@ export class SherpaOnnxTTS implements TextToSpeechProvider { assertFileExists(tokensPath, "TTS tokens"); assertFileExists(dataDir, "TTS espeak-ng dataDir"); - const modelConfig = - config.preset === "kokoro-en-v0_19" - ? { - kokoro: { - model: modelPath, - voices: voicesPath, - tokens: tokensPath, - dataDir, - lengthScale: config.lengthScale ?? 1.0, - }, - } - : { - kitten: { - model: modelPath, - voices: voicesPath, - tokens: tokensPath, - dataDir, - lengthScale: config.lengthScale ?? 1.0, - }, - }; + const modelConfig = { + kokoro: { + model: modelPath, + voices: voicesPath, + tokens: tokensPath, + dataDir, + lengthScale: config.lengthScale ?? 1.0, + }, + }; const offlineTtsConfig = { model: modelConfig, diff --git a/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts b/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts index 97d33923e..ab2eefef5 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts @@ -5,28 +5,15 @@ import path from "node:path"; import pino from "pino"; import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./model-downloader.js"; -import type { SherpaOnnxModelId } from "./model-catalog.js"; import { createDaemonTestContext } from "../../../../test-utils/index.js"; import { parsePcm16MonoWav, wordSimilarity } from "../../../../test-utils/dictation-e2e.js"; import { SherpaOnnxTTS } from "./sherpa-tts.js"; -import { PocketTtsOnnxTTS } from "../pocket/pocket-tts-onnx.js"; -import { SherpaOnlineRecognizerEngine } from "./sherpa-online-recognizer.js"; -import { SherpaOnnxSTT } from "./sherpa-stt.js"; import { SherpaOfflineRecognizerEngine } from "./sherpa-offline-recognizer.js"; import { SherpaOnnxParakeetSTT } from "./sherpa-parakeet-stt.js"; const RUN = process.env.PASEO_SPEECH_E2E_DOWNLOAD === "1"; const downloadTest = RUN ? test : test.skip; -type ModelSet = "zipformer-kitten" | "parakeet-pocket"; - -function getModelSet(): ModelSet { - const raw = (process.env.PASEO_SPEECH_E2E_MODEL_SET ?? "parakeet-pocket").trim().toLowerCase(); - if (raw === "zipformer-kitten" || raw === "zipformer") return "zipformer-kitten"; - if (raw === "parakeet-pocket" || raw === "parakeet") return "parakeet-pocket"; - throw new Error(`Unknown PASEO_SPEECH_E2E_MODEL_SET: ${raw}`); -} - async function readFixtureWav(): Promise { const fixturePath = path.resolve(process.cwd(), "..", "app", "e2e", "fixtures", "recording.wav"); return import("node:fs/promises").then((fs) => fs.readFile(fixturePath)); @@ -92,27 +79,20 @@ function toAudioPcmFormat(format: string): string { const rest = trimmed.replace(/^pcm;?/i, ""); return rest ? `audio/pcm;${rest}` : "audio/pcm"; } - // Fall back to treating it as a suffix (e.g. "rate=24000") return `audio/pcm;${trimmed}`; } downloadTest( - "downloads models and runs streaming STT + streaming TTS", + "downloads default local speech models and runs STT + TTS", async () => { const logger = pino({ level: "silent" }); - const set = getModelSet(); const paseoHomeRoot = mkdtempSync(path.join(tmpdir(), "paseo-speech-download-")); const modelsDir = path.join(paseoHomeRoot, ".paseo", "models", "local-speech"); - const modelIds: SherpaOnnxModelId[] = - set === "parakeet-pocket" - ? ["parakeet-tdt-0.6b-v3-int8", "pocket-tts-onnx-int8"] - : ["zipformer-bilingual-zh-en-2023-02-20", "kitten-nano-en-v0_1-fp16"]; - await ensureSherpaOnnxModels({ modelsDir, - modelIds, + modelIds: ["parakeet-tdt-0.6b-v2-int8", "kokoro-en-v0_19"], logger, }); @@ -128,16 +108,10 @@ downloadTest( local: { modelsDir, models: { - dictationStt: - set === "parakeet-pocket" - ? "parakeet-tdt-0.6b-v3-int8" - : "zipformer-bilingual-zh-en-2023-02-20", - voiceStt: - set === "parakeet-pocket" - ? "parakeet-tdt-0.6b-v3-int8" - : "zipformer-bilingual-zh-en-2023-02-20", - voiceTts: - set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16", + dictationStt: "parakeet-tdt-0.6b-v2-int8", + voiceStt: "parakeet-tdt-0.6b-v2-int8", + voiceTts: "kokoro-en-v0_19", + voiceTtsSpeakerId: 0, }, }, }, @@ -149,7 +123,6 @@ downloadTest( expect(sampleRate).toBe(16000); const format = "audio/pcm;rate=16000;bits=16"; - // Streaming STT: dictation path (verifies websocket streaming + partials + final) const dictationId = `dict-download-${Date.now()}`; let partialCount = 0; const unsubscribe = ctx.client.on("dictation_stream_partial", (message) => { @@ -159,15 +132,14 @@ downloadTest( }); await ctx.client.startDictationStream(dictationId, format); - const chunkBytes = 3200; // 100ms @ 16kHz mono PCM16 + const chunkBytes = 3200; let seq = 0; for (let offset = 0; offset < pcm16.length; offset += chunkBytes) { const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes)); ctx.client.sendDictationStreamChunk(dictationId, seq, chunk.toString("base64"), format); seq += 1; } - const finalSeq = seq - 1; - const dictationFinal = await ctx.client.finishDictationStream(dictationId, finalSeq); + const dictationFinal = await ctx.client.finishDictationStream(dictationId, seq - 1); unsubscribe(); expect(dictationFinal.text.toLowerCase()).toContain("voice note"); @@ -175,7 +147,6 @@ downloadTest( expect(wordSimilarity(dictationFinal.text, baseline)).toBeGreaterThan(0.45); expect(partialCount).toBeGreaterThan(0); - // Voice-mode STT: chunked upload until isLast=true const transcriptionPromise = waitForSignal(30000, (resolve, reject) => { const offResult = ctx.client.on("transcription_result", (message) => { if (message.type !== "transcription_result") return; @@ -218,80 +189,39 @@ downloadTest( await ctx.client.setVoiceMode(false); rmSync(voiceCwd, { recursive: true, force: true }); - // Streaming TTS: generate locally from downloaded model and validate chunking. const ttsText = "This is a voice note."; - if (set === "parakeet-pocket") { - const modelDir = getSherpaOnnxModelDir(modelsDir, "pocket-tts-onnx-int8"); - const tts = await PocketTtsOnnxTTS.create( - { modelDir, precision: "int8", targetChunkMs: 50 }, - logger, - ); - const { stream, format: ttsFormat } = await tts.synthesizeSpeech(ttsText); - const { chunks, combined } = await readAllChunks(stream); + const ttsModelDir = getSherpaOnnxModelDir(modelsDir, "kokoro-en-v0_19"); + const tts = new SherpaOnnxTTS( + { preset: "kokoro-en-v0_19", modelDir: ttsModelDir, speakerId: 0 }, + logger, + ); + const { stream, format: ttsFormat } = await tts.synthesizeSpeech(ttsText); + const { chunks, combined } = await readAllChunks(stream); + tts.free(); - expect(ttsFormat).toMatch(/rate=\d+/); - expect(chunks.length).toBeGreaterThan(3); - expect(combined.byteLength).toBeGreaterThan(2000); + expect(ttsFormat).toMatch(/rate=\d+/); + expect(chunks.length).toBeGreaterThan(3); + expect(combined.byteLength).toBeGreaterThan(2000); - // Round trip: TTS -> STT (offline parakeet) - const sttModelDir = getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v3-int8"); - const engine = new SherpaOfflineRecognizerEngine( - { - model: { - kind: "nemo_transducer", - encoder: `${sttModelDir}/encoder.int8.onnx`, - decoder: `${sttModelDir}/decoder.int8.onnx`, - joiner: `${sttModelDir}/joiner.int8.onnx`, - tokens: `${sttModelDir}/tokens.txt`, - }, - numThreads: 2, - debug: 0, + const sttModelDir = getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v2-int8"); + const engine = new SherpaOfflineRecognizerEngine( + { + model: { + kind: "nemo_transducer", + encoder: `${sttModelDir}/encoder.int8.onnx`, + decoder: `${sttModelDir}/decoder.int8.onnx`, + joiner: `${sttModelDir}/joiner.int8.onnx`, + tokens: `${sttModelDir}/tokens.txt`, }, - logger, - ); - const stt = new SherpaOnnxParakeetSTT({ engine }, logger); - const rt = await stt.transcribeAudio(combined, toAudioPcmFormat(ttsFormat)); - engine.free(); - expect(wordSimilarity(rt.text, ttsText)).toBeGreaterThan(0.25); - } else { - const ttsModelDir = path.join(modelsDir, "kitten-nano-en-v0_1-fp16"); - const tts = new SherpaOnnxTTS( - { preset: "kitten-nano-en-v0_1-fp16", modelDir: ttsModelDir }, - logger, - ); - const { stream, format: ttsFormat } = await tts.synthesizeSpeech(ttsText); - const { chunks, combined } = await readAllChunks(stream); - tts.free(); - - expect(ttsFormat).toMatch(/rate=\d+/); - expect(chunks.length).toBeGreaterThan(3); - expect(combined.byteLength).toBeGreaterThan(2000); - - // Round trip: TTS -> STT (online zipformer, offline segment) - const sttModelDir = path.join( - modelsDir, - "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", - ); - const engine = new SherpaOnlineRecognizerEngine( - { - model: { - kind: "transducer", - encoder: `${sttModelDir}/encoder-epoch-99-avg-1.onnx`, - decoder: `${sttModelDir}/decoder-epoch-99-avg-1.onnx`, - joiner: `${sttModelDir}/joiner-epoch-99-avg-1.onnx`, - tokens: `${sttModelDir}/tokens.txt`, - modelType: "zipformer", - }, - numThreads: 1, - debug: 0, - }, - logger, - ); - const stt = new SherpaOnnxSTT({ engine }, logger); - const rt = await stt.transcribeAudio(combined, toAudioPcmFormat(ttsFormat)); - engine.free(); - expect(wordSimilarity(rt.text, ttsText)).toBeGreaterThan(0.25); - } + numThreads: 2, + debug: 0, + }, + logger, + ); + const stt = new SherpaOnnxParakeetSTT({ engine }, logger); + const rt = await stt.transcribeAudio(combined, toAudioPcmFormat(ttsFormat)); + engine.free(); + expect(wordSimilarity(rt.text, ttsText)).toBeGreaterThan(0.25); } finally { await ctx.cleanup(); } diff --git a/packages/server/src/server/speech/speech-config-resolver.test.ts b/packages/server/src/server/speech/speech-config-resolver.test.ts index 86caed8cc..3d93b3673 100644 --- a/packages/server/src/server/speech/speech-config-resolver.test.ts +++ b/packages/server/src/server/speech/speech-config-resolver.test.ts @@ -57,7 +57,7 @@ describe("resolveSpeechConfig", () => { }); }); - test("resolves feature-scoped local model env vars", () => { + test("resolves feature-scoped local speech settings", () => { const persisted = PersistedConfigSchema.parse({ features: { voiceMode: { @@ -70,9 +70,9 @@ describe("resolveSpeechConfig", () => { }, }); const env = { - PASEO_DICTATION_LOCAL_STT_MODEL: "zipformer", - PASEO_VOICE_LOCAL_STT_MODEL: "parakeet", - PASEO_VOICE_LOCAL_TTS_MODEL: "kitten", + PASEO_DICTATION_LOCAL_STT_MODEL: "parakeet-tdt-0.6b-v2-int8", + PASEO_VOICE_LOCAL_STT_MODEL: "parakeet-tdt-0.6b-v2-int8", + PASEO_VOICE_LOCAL_TTS_MODEL: "kokoro-en-v0_19", PASEO_VOICE_LOCAL_TTS_SPEAKER_ID: "5", PASEO_VOICE_LOCAL_TTS_SPEED: "1.35", PASEO_DICTATION_LANGUAGE: "es", @@ -93,9 +93,9 @@ describe("resolveSpeechConfig", () => { expect(result.speech.local).toEqual({ modelsDir: "/tmp/models", models: { - dictationStt: "zipformer-bilingual-zh-en-2023-02-20", - voiceStt: "parakeet-tdt-0.6b-v3-int8", - voiceTts: "kitten-nano-en-v0_1-fp16", + dictationStt: "parakeet-tdt-0.6b-v2-int8", + voiceStt: "parakeet-tdt-0.6b-v2-int8", + voiceTts: "kokoro-en-v0_19", voiceTtsSpeakerId: 5, voiceTtsSpeed: 1.35, }, @@ -120,9 +120,9 @@ describe("resolveSpeechConfig", () => { explicit: true, enabled: true, }); - expect(result.speech.local?.models.dictationStt).toBe("zipformer-bilingual-zh-en-2023-02-20"); - expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8"); - expect(result.speech.local?.models.voiceTts).toBe("kitten-nano-en-v0_1-fp16"); + expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v2-int8"); + expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v2-int8"); + expect(result.speech.local?.models.voiceTts).toBe("kokoro-en-v0_19"); expect(result.speech.local?.models.voiceTtsSpeakerId).toBe(5); expect(result.speech.local?.models.voiceTtsSpeed).toBe(1.35); expect(result.speech.sttLanguages).toEqual({ @@ -164,25 +164,6 @@ describe("resolveSpeechConfig", () => { }); }); - test("ignores deprecated shared local model env vars", () => { - const persisted = PersistedConfigSchema.parse({}); - const env = { - PASEO_LOCAL_STT_MODEL: "zipformer-bilingual-zh-en-2023-02-20", - PASEO_LOCAL_TTS_MODEL: "kitten-nano-en-v0_1-fp16", - } as NodeJS.ProcessEnv; - - const result = resolveSpeechConfig({ - paseoHome: "/tmp/paseo-home", - env, - persisted, - }); - - expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v2-int8"); - expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v2-int8"); - expect(result.speech.local?.models.voiceTts).toBe("kokoro-en-v0_19"); - expect(result.speech.local?.models.voiceTtsSpeakerId).toBe(0); - }); - test("respects disabled dictation and voice mode feature flags", () => { const persisted = PersistedConfigSchema.parse({ features: { diff --git a/packages/server/src/server/websocket-server.relay-reconnect.test.ts b/packages/server/src/server/websocket-server.relay-reconnect.test.ts index 1929f5441..8bdda0869 100644 --- a/packages/server/src/server/websocket-server.relay-reconnect.test.ts +++ b/packages/server/src/server/websocket-server.relay-reconnect.test.ts @@ -293,8 +293,8 @@ function createReadySpeechReadinessSnapshot(): SpeechReadinessSnapshot { function createDownloadInProgressSpeechReadinessSnapshot(): SpeechReadinessSnapshot { return { generatedAt: "2026-02-14T00:00:00.000Z", - requiredLocalModelIds: ["sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20"], - missingLocalModelIds: ["sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20"], + requiredLocalModelIds: ["parakeet-tdt-0.6b-v2-int8"], + missingLocalModelIds: ["parakeet-tdt-0.6b-v2-int8"], download: { inProgress: true, error: null, @@ -320,9 +320,9 @@ function createDownloadInProgressSpeechReadinessSnapshot(): SpeechReadinessSnaps available: false, reasonCode: "model_download_in_progress", message: - "Voice features are unavailable while models download in the background (sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20).", + "Voice features are unavailable while models download in the background (parakeet-tdt-0.6b-v2-int8).", retryable: true, - missingModelIds: ["sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20"], + missingModelIds: ["parakeet-tdt-0.6b-v2-int8"], }, }; }