mirror of
https://github.com/getpaseo/paseo.git
synced 2026-07-29 12:01:31 +00:00
Simplify local speech models
This commit is contained in:
Mohamed Boudra
@@ -171,6 +171,8 @@ Single file, validated with `PersistedConfigSchema`.
|
||||
|
||||
All fields are optional with sensible defaults.
|
||||
|
||||
Local speech model ids are intentionally narrow: STT uses `parakeet-tdt-0.6b-v2-int8`, TTS uses `kokoro-en-v0_19`, and turn detection uses the bundled Silero VAD model.
|
||||
|
||||
---
|
||||
|
||||
## 3. Schedule
|
||||
|
||||
95
package-lock.json
generated
95
package-lock.json
generated
@@ -10647,16 +10647,6 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@sctg/sentencepiece-js": {
|
||||
"version": "1.3.3",
|
||||
"resolved": "https://registry.npmjs.org/@sctg/sentencepiece-js/-/sentencepiece-js-1.3.3.tgz",
|
||||
"integrity": "sha512-iPnzR2HGjdQQG2SpTyPH3wnnpgQ2aS14B3I2jHjomkoaMprM3Sn+WOQblHwByetnvn3n14y3IwjwogVUHA2cVA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"app-root-path": "^3.1.0",
|
||||
"buffer": "^6.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@segment/ajv-human-errors": {
|
||||
"version": "2.16.0",
|
||||
"resolved": "https://registry.npmjs.org/@segment/ajv-human-errors/-/ajv-human-errors-2.16.0.tgz",
|
||||
@@ -13823,15 +13813,6 @@
|
||||
"node": ">=0.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/adm-zip": {
|
||||
"version": "0.5.16",
|
||||
"resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.16.tgz",
|
||||
"integrity": "sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/agent-base": {
|
||||
"version": "6.0.2",
|
||||
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
|
||||
@@ -14283,15 +14264,6 @@
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/app-root-path": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/app-root-path/-/app-root-path-3.1.0.tgz",
|
||||
"integrity": "sha512-biN3PwB2gUtjaYy/isrU3aNWI5w+fAfvHkSvCKeQGxhmYpwKFUxudR3Yya+KqVRHBmEDYh+/lTozYCFbmzX4nA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 6.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/arg": {
|
||||
"version": "5.0.2",
|
||||
"resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz",
|
||||
@@ -15147,7 +15119,9 @@
|
||||
"resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz",
|
||||
"integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==",
|
||||
"deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.",
|
||||
"license": "MIT"
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true
|
||||
},
|
||||
"node_modules/bplist-creator": {
|
||||
"version": "0.1.0",
|
||||
@@ -17013,7 +16987,9 @@
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz",
|
||||
"integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==",
|
||||
"license": "MIT"
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true
|
||||
},
|
||||
"node_modules/detect-node-es": {
|
||||
"version": "1.1.0",
|
||||
@@ -18115,7 +18091,9 @@
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz",
|
||||
"integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==",
|
||||
"license": "MIT"
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true
|
||||
},
|
||||
"node_modules/esbuild": {
|
||||
"version": "0.27.3",
|
||||
@@ -22620,7 +22598,9 @@
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz",
|
||||
"integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==",
|
||||
"dev": true,
|
||||
"license": "BSD-3-Clause",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"boolean": "^3.0.1",
|
||||
"es6-error": "^4.1.1",
|
||||
@@ -22637,7 +22617,9 @@
|
||||
"version": "7.0.1",
|
||||
"resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz",
|
||||
"integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"type-fest": "^0.13.1"
|
||||
},
|
||||
@@ -22652,7 +22634,9 @@
|
||||
"version": "0.13.1",
|
||||
"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz",
|
||||
"integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==",
|
||||
"dev": true,
|
||||
"license": "(MIT OR CC0-1.0)",
|
||||
"optional": true,
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
},
|
||||
@@ -22677,6 +22661,7 @@
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz",
|
||||
"integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"define-properties": "^1.2.1",
|
||||
@@ -25331,7 +25316,9 @@
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
|
||||
"integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==",
|
||||
"license": "ISC"
|
||||
"dev": true,
|
||||
"license": "ISC",
|
||||
"optional": true
|
||||
},
|
||||
"node_modules/json5": {
|
||||
"version": "2.2.3",
|
||||
@@ -26349,7 +26336,9 @@
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz",
|
||||
"integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"escape-string-regexp": "^4.0.0"
|
||||
},
|
||||
@@ -28985,29 +28974,6 @@
|
||||
"regex-recursion": "^6.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/onnxruntime-common": {
|
||||
"version": "1.24.3",
|
||||
"resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.24.3.tgz",
|
||||
"integrity": "sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/onnxruntime-node": {
|
||||
"version": "1.24.3",
|
||||
"resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.24.3.tgz",
|
||||
"integrity": "sha512-JH7+czbc8ALA819vlTgcV+Q214/+VjGeBHDjX81+ZCD0PCVCIFGFNtT0V4sXG/1JXypKPgScQcB3ij/hk3YnTg==",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"os": [
|
||||
"win32",
|
||||
"darwin",
|
||||
"linux"
|
||||
],
|
||||
"dependencies": {
|
||||
"adm-zip": "^0.5.16",
|
||||
"global-agent": "^3.0.0",
|
||||
"onnxruntime-common": "1.24.3"
|
||||
}
|
||||
},
|
||||
"node_modules/open": {
|
||||
"version": "7.4.2",
|
||||
"resolved": "https://registry.npmjs.org/open/-/open-7.4.2.tgz",
|
||||
@@ -31922,7 +31888,9 @@
|
||||
"version": "2.15.4",
|
||||
"resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz",
|
||||
"integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==",
|
||||
"dev": true,
|
||||
"license": "BSD-3-Clause",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"boolean": "^3.0.1",
|
||||
"detect-node": "^2.0.4",
|
||||
@@ -31939,7 +31907,9 @@
|
||||
"version": "1.1.3",
|
||||
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz",
|
||||
"integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==",
|
||||
"license": "BSD-3-Clause"
|
||||
"dev": true,
|
||||
"license": "BSD-3-Clause",
|
||||
"optional": true
|
||||
},
|
||||
"node_modules/rollup": {
|
||||
"version": "4.59.0",
|
||||
@@ -32230,7 +32200,9 @@
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz",
|
||||
"integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==",
|
||||
"license": "MIT"
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true
|
||||
},
|
||||
"node_modules/send": {
|
||||
"version": "0.19.2",
|
||||
@@ -32515,12 +32487,6 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/sherpa-onnx": {
|
||||
"version": "1.12.28",
|
||||
"resolved": "https://registry.npmjs.org/sherpa-onnx/-/sherpa-onnx-1.12.28.tgz",
|
||||
"integrity": "sha512-PaOuTCUA/Vh5o3zcq+z3j1Zpv2VlP7iubOgYoVODDyeFDicRxjskiEQqERdWEcpQUi6vh47q3A63bS7Y9FRHNg==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/sherpa-onnx-darwin-arm64": {
|
||||
"version": "1.12.28",
|
||||
"resolved": "https://registry.npmjs.org/sherpa-onnx-darwin-arm64/-/sherpa-onnx-darwin-arm64-1.12.28.tgz",
|
||||
@@ -37603,7 +37569,6 @@
|
||||
"@isaacs/ttlcache": "^2.1.4",
|
||||
"@modelcontextprotocol/sdk": "^1.20.1",
|
||||
"@opencode-ai/sdk": "1.14.46",
|
||||
"@sctg/sentencepiece-js": "^1.1.0",
|
||||
"@xterm/headless": "^6.0.0",
|
||||
"ai": "5.0.78",
|
||||
"ajv": "^8.20.0",
|
||||
@@ -37614,7 +37579,6 @@
|
||||
"lru-cache": "^11.5.1",
|
||||
"mnemonic-id": "^3.2.7",
|
||||
"node-pty": "1.2.0-beta.11",
|
||||
"onnxruntime-node": "^1.23.0",
|
||||
"openai": "^4.20.0",
|
||||
"p-limit": "^7.3.0",
|
||||
"p-memoize": "^8.0.0",
|
||||
@@ -37622,7 +37586,6 @@
|
||||
"pino-pretty": "^13.1.3",
|
||||
"qrcode": "^1.5.4",
|
||||
"rotating-file-stream": "^3.2.9",
|
||||
"sherpa-onnx": "1.12.28",
|
||||
"sherpa-onnx-node": "1.12.28",
|
||||
"strip-ansi": "^7.1.2",
|
||||
"tree-kill": "^1.2.2",
|
||||
|
||||
@@ -27,29 +27,6 @@ function pruneChildrenExcept(parent, keep) {
|
||||
}
|
||||
}
|
||||
|
||||
function pruneOnnxRuntime(nodeModules, platform, arch) {
|
||||
const onnxBin = path.join(nodeModules, "onnxruntime-node", "bin", "napi-v6");
|
||||
if (!fs.existsSync(onnxBin)) return;
|
||||
|
||||
const otherPlatforms = ["darwin", "linux", "win32"].filter((p) => p !== platform);
|
||||
for (const p of otherPlatforms) {
|
||||
rmSafe(path.join(onnxBin, p));
|
||||
}
|
||||
|
||||
pruneChildrenExcept(path.join(onnxBin, platform), new Set([arch]));
|
||||
|
||||
if (platform === "linux") {
|
||||
const archDir = path.join(onnxBin, "linux", arch);
|
||||
if (fs.existsSync(archDir)) {
|
||||
for (const name of fs.readdirSync(archDir)) {
|
||||
if (name.includes("cuda") || name.includes("tensorrt")) {
|
||||
fs.rmSync(path.join(archDir, name), { force: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function pruneClaudeAgentSdk(nodeModules, platform, arch) {
|
||||
const vendorRoot = path.join(nodeModules, "@anthropic-ai", "claude-agent-sdk", "vendor");
|
||||
const keepName = RIPGREP_PLATFORM_DIR[platform]?.[arch];
|
||||
@@ -107,7 +84,6 @@ function pruneNativeModules(appOutDir, platform, arch) {
|
||||
|
||||
const before = dirSizeSync(nodeModules);
|
||||
|
||||
pruneOnnxRuntime(nodeModules, platform, arch);
|
||||
pruneClaudeAgentSdk(nodeModules, platform, arch);
|
||||
pruneNodePty(nodeModules, platform, arch);
|
||||
pruneSharpLibvips(nodeModules, platform, arch);
|
||||
|
||||
@@ -44,7 +44,6 @@
|
||||
"generate:config-schema": "tsx scripts/generate-config-schema.ts",
|
||||
"speech:models": "tsx scripts/list-speech-models.ts",
|
||||
"speech:download": "tsx scripts/download-speech-models.ts",
|
||||
"speech:tts:matrix": "tsx scripts/generate-sherpa-tts-matrix.ts",
|
||||
"speech:transcribe:local": "tsx scripts/transcribe-local-wav.ts",
|
||||
"test": "npm run test:unit && npm run test:integration",
|
||||
"test:unit": "vitest run --exclude \"**/*.e2e.test.ts\"",
|
||||
@@ -70,7 +69,6 @@
|
||||
"@isaacs/ttlcache": "^2.1.4",
|
||||
"@modelcontextprotocol/sdk": "^1.20.1",
|
||||
"@opencode-ai/sdk": "1.14.46",
|
||||
"@sctg/sentencepiece-js": "^1.1.0",
|
||||
"@xterm/headless": "^6.0.0",
|
||||
"ai": "5.0.78",
|
||||
"ajv": "^8.20.0",
|
||||
@@ -81,7 +79,6 @@
|
||||
"lru-cache": "^11.5.1",
|
||||
"mnemonic-id": "^3.2.7",
|
||||
"node-pty": "1.2.0-beta.11",
|
||||
"onnxruntime-node": "^1.23.0",
|
||||
"openai": "^4.20.0",
|
||||
"p-limit": "^7.3.0",
|
||||
"p-memoize": "^8.0.0",
|
||||
@@ -89,7 +86,6 @@
|
||||
"pino-pretty": "^13.1.3",
|
||||
"qrcode": "^1.5.4",
|
||||
"rotating-file-stream": "^3.2.9",
|
||||
"sherpa-onnx": "1.12.28",
|
||||
"sherpa-onnx-node": "1.12.28",
|
||||
"strip-ansi": "^7.1.2",
|
||||
"tree-kill": "^1.2.2",
|
||||
|
||||
@@ -31,11 +31,7 @@ function parseArgs(argv: string[]): { modelsDir: string; modelIds: LocalSpeechMo
|
||||
}
|
||||
|
||||
if (modelIds.length === 0) {
|
||||
const stt = (process.env.PASEO_LOCAL_STT_MODEL ||
|
||||
DEFAULT_LOCAL_STT_MODEL) as LocalSpeechModelId;
|
||||
const tts = (process.env.PASEO_LOCAL_TTS_MODEL ||
|
||||
DEFAULT_LOCAL_TTS_MODEL) as LocalSpeechModelId;
|
||||
modelIds.push(stt, tts);
|
||||
modelIds.push(DEFAULT_LOCAL_STT_MODEL, DEFAULT_LOCAL_TTS_MODEL);
|
||||
}
|
||||
|
||||
return { modelsDir, modelIds };
|
||||
|
||||
@@ -1,436 +0,0 @@
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
import { createRootLogger } from "../src/server/logger.js";
|
||||
import { resolvePaseoHome } from "../src/server/paseo-home.js";
|
||||
import { float32ToPcm16le, parsePcmRateFromFormat } from "../src/server/speech/audio.js";
|
||||
import {
|
||||
ensureLocalSpeechModels,
|
||||
getLocalSpeechModelDir,
|
||||
type LocalTtsModelId,
|
||||
} from "../src/server/speech/providers/local/models.js";
|
||||
import { PocketTtsOnnxTTS } from "../src/server/speech/providers/local/pocket/pocket-tts-onnx.js";
|
||||
import {
|
||||
LOCAL_TTS_MODEL_IDS,
|
||||
LocalTtsModelIdSchema,
|
||||
} from "../src/server/speech/providers/local/sherpa/model-catalog.js";
|
||||
import { loadSherpaOnnxNode } from "../src/server/speech/providers/local/sherpa/sherpa-onnx-node-loader.js";
|
||||
|
||||
const DEFAULT_SAMPLE_TEXT =
|
||||
"Paseo keeps your coding agents in your pocket, with fast local speech and reliable remote control.";
|
||||
|
||||
interface ScriptOptions {
|
||||
modelsDir: string;
|
||||
outputDir: string;
|
||||
text: string;
|
||||
speed: number;
|
||||
modelIds: LocalTtsModelId[];
|
||||
}
|
||||
|
||||
interface GeneratedSample {
|
||||
modelId: LocalTtsModelId;
|
||||
voiceLabel: string;
|
||||
speakerId: number | null;
|
||||
sampleRate: number;
|
||||
durationSeconds: number;
|
||||
relativePath: string;
|
||||
}
|
||||
|
||||
function usage(): string {
|
||||
return [
|
||||
"Generate a local Sherpa/Pocket TTS sample matrix.",
|
||||
"",
|
||||
"Options:",
|
||||
" --models-dir <dir> Directory for downloaded local speech models",
|
||||
" --output-dir <dir> Directory where WAV samples are written",
|
||||
" --model <id> TTS model to include (repeatable)",
|
||||
" --text <sentence> Sample sentence used for every generated file",
|
||||
" --speed <number> TTS speed multiplier for Sherpa models (default: 1.0)",
|
||||
" --help Show this help",
|
||||
"",
|
||||
`Known TTS model IDs: ${LOCAL_TTS_MODEL_IDS.join(", ")}`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function parsePositiveNumber(raw: string, flag: string): number {
|
||||
const parsed = Number.parseFloat(raw);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) {
|
||||
throw new Error(`${flag} must be a positive number (received: ${raw})`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): ScriptOptions {
|
||||
const paseoHome = resolvePaseoHome();
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
|
||||
|
||||
let modelsDir = process.env.PASEO_LOCAL_MODELS_DIR ?? `${paseoHome}/models/local-speech`;
|
||||
let outputDir = path.resolve(process.cwd(), ".debug", "sherpa-tts-matrix", timestamp);
|
||||
let text = DEFAULT_SAMPLE_TEXT;
|
||||
let speed = parsePositiveNumber(process.env.PASEO_VOICE_LOCAL_TTS_SPEED ?? "1.0", "--speed");
|
||||
const requestedModelIds: LocalTtsModelId[] = [];
|
||||
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const arg = argv[i];
|
||||
|
||||
if (arg === "--help") {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(usage());
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (arg === "--models-dir") {
|
||||
modelsDir = path.resolve(argv[i + 1] ?? modelsDir);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--output-dir") {
|
||||
outputDir = path.resolve(argv[i + 1] ?? outputDir);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--text") {
|
||||
text = argv[i + 1] ?? text;
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--speed") {
|
||||
const raw = argv[i + 1];
|
||||
if (!raw) {
|
||||
throw new Error("--speed requires a value");
|
||||
}
|
||||
speed = parsePositiveNumber(raw, "--speed");
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--model") {
|
||||
const raw = argv[i + 1];
|
||||
if (!raw) {
|
||||
throw new Error("--model requires a value");
|
||||
}
|
||||
requestedModelIds.push(LocalTtsModelIdSchema.parse(raw));
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed) {
|
||||
throw new Error("Sample text cannot be empty");
|
||||
}
|
||||
|
||||
const modelIds =
|
||||
requestedModelIds.length > 0
|
||||
? Array.from(new Set(requestedModelIds))
|
||||
: [...LOCAL_TTS_MODEL_IDS];
|
||||
|
||||
return {
|
||||
modelsDir,
|
||||
outputDir,
|
||||
text: trimmed,
|
||||
speed,
|
||||
modelIds,
|
||||
};
|
||||
}
|
||||
|
||||
function pcm16MonoToWavBuffer(pcm16: Buffer, sampleRate: number): Buffer {
|
||||
const channels = 1;
|
||||
const bitsPerSample = 16;
|
||||
const headerSize = 44;
|
||||
const wav = Buffer.alloc(headerSize + pcm16.length);
|
||||
const byteRate = (sampleRate * channels * bitsPerSample) / 8;
|
||||
const blockAlign = (channels * bitsPerSample) / 8;
|
||||
|
||||
wav.write("RIFF", 0);
|
||||
wav.writeUInt32LE(36 + pcm16.length, 4);
|
||||
wav.write("WAVE", 8);
|
||||
wav.write("fmt ", 12);
|
||||
wav.writeUInt32LE(16, 16);
|
||||
wav.writeUInt16LE(1, 20);
|
||||
wav.writeUInt16LE(channels, 22);
|
||||
wav.writeUInt32LE(sampleRate, 24);
|
||||
wav.writeUInt32LE(byteRate, 28);
|
||||
wav.writeUInt16LE(blockAlign, 32);
|
||||
wav.writeUInt16LE(bitsPerSample, 34);
|
||||
wav.write("data", 36);
|
||||
wav.writeUInt32LE(pcm16.length, 40);
|
||||
pcm16.copy(wav, 44);
|
||||
|
||||
return wav;
|
||||
}
|
||||
|
||||
function sanitizeFilePart(input: string): string {
|
||||
return input.replace(/[^a-zA-Z0-9._-]/g, "-");
|
||||
}
|
||||
|
||||
function toFloatSamples(audio: unknown): Float32Array {
|
||||
const raw = (audio as { samples?: unknown })?.samples;
|
||||
if (raw instanceof Float32Array) {
|
||||
return raw;
|
||||
}
|
||||
if (Array.isArray(raw)) {
|
||||
return Float32Array.from(raw as number[]);
|
||||
}
|
||||
throw new Error("Unexpected Sherpa TTS output: missing Float32 samples");
|
||||
}
|
||||
|
||||
function toSampleRate(audio: unknown, fallback: number): number {
|
||||
const sampleRate = (audio as { sampleRate?: unknown })?.sampleRate;
|
||||
if (typeof sampleRate === "number" && Number.isFinite(sampleRate) && sampleRate > 0) {
|
||||
return sampleRate;
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function createSherpaOfflineTts(params: {
|
||||
modelId: "kokoro-en-v0_19" | "kitten-nano-en-v0_1-fp16";
|
||||
modelDir: string;
|
||||
speed: number;
|
||||
}): { tts: unknown; speed: number } {
|
||||
const sherpa = loadSherpaOnnxNode();
|
||||
const model =
|
||||
params.modelId === "kokoro-en-v0_19"
|
||||
? {
|
||||
kokoro: {
|
||||
model: `${params.modelDir}/model.onnx`,
|
||||
voices: `${params.modelDir}/voices.bin`,
|
||||
tokens: `${params.modelDir}/tokens.txt`,
|
||||
dataDir: `${params.modelDir}/espeak-ng-data`,
|
||||
lengthScale: 1.0,
|
||||
},
|
||||
}
|
||||
: {
|
||||
kitten: {
|
||||
model: `${params.modelDir}/model.fp16.onnx`,
|
||||
voices: `${params.modelDir}/voices.bin`,
|
||||
tokens: `${params.modelDir}/tokens.txt`,
|
||||
dataDir: `${params.modelDir}/espeak-ng-data`,
|
||||
lengthScale: 1.0,
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
tts: new sherpa.OfflineTts({
|
||||
model,
|
||||
numThreads: 2,
|
||||
provider: "cpu",
|
||||
maxNumSentences: 1,
|
||||
}),
|
||||
speed: params.speed,
|
||||
};
|
||||
}
|
||||
|
||||
async function readStreamToBuffer(stream: NodeJS.ReadableStream): Promise<Buffer> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of stream as AsyncIterable<Buffer | Uint8Array>) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
}
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
function toCsv(rows: GeneratedSample[]): string {
|
||||
const header = [
|
||||
"model_id",
|
||||
"voice_label",
|
||||
"speaker_id",
|
||||
"sample_rate",
|
||||
"duration_seconds",
|
||||
"relative_path",
|
||||
];
|
||||
|
||||
const lines = [header.join(",")];
|
||||
for (const row of rows) {
|
||||
const cells = [
|
||||
row.modelId,
|
||||
row.voiceLabel,
|
||||
row.speakerId === null ? "" : String(row.speakerId),
|
||||
String(row.sampleRate),
|
||||
row.durationSeconds.toFixed(3),
|
||||
row.relativePath,
|
||||
];
|
||||
|
||||
lines.push(
|
||||
cells
|
||||
.map((cell) => {
|
||||
const escaped = cell.replace(/"/g, '""');
|
||||
return escaped.includes(",") ? `"${escaped}"` : escaped;
|
||||
})
|
||||
.join(","),
|
||||
);
|
||||
}
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
const logger = createRootLogger({ level: "info", format: "pretty" });
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
|
||||
logger.info(
|
||||
{
|
||||
modelsDir: options.modelsDir,
|
||||
outputDir: options.outputDir,
|
||||
modelIds: options.modelIds,
|
||||
speed: options.speed,
|
||||
text: options.text,
|
||||
},
|
||||
"Generating TTS matrix across all model/voice combinations",
|
||||
);
|
||||
|
||||
await ensureLocalSpeechModels({
|
||||
modelsDir: options.modelsDir,
|
||||
modelIds: options.modelIds,
|
||||
logger,
|
||||
});
|
||||
|
||||
await mkdir(options.outputDir, { recursive: true });
|
||||
|
||||
const generated: GeneratedSample[] = [];
|
||||
|
||||
for (const modelId of options.modelIds) {
|
||||
const modelDir = getLocalSpeechModelDir(options.modelsDir, modelId);
|
||||
const modelOutputDir = path.join(options.outputDir, sanitizeFilePart(modelId));
|
||||
await mkdir(modelOutputDir, { recursive: true });
|
||||
|
||||
if (modelId === "pocket-tts-onnx-int8") {
|
||||
const tts = await PocketTtsOnnxTTS.create(
|
||||
{
|
||||
modelDir,
|
||||
precision: "int8",
|
||||
targetChunkMs: 50,
|
||||
},
|
||||
logger,
|
||||
);
|
||||
const result = await tts.synthesizeSpeech(options.text);
|
||||
const pcm16 = await readStreamToBuffer(result.stream);
|
||||
const sampleRate = parsePcmRateFromFormat(result.format, 24000) ?? 24000;
|
||||
|
||||
const voiceLabel = "default";
|
||||
const filename = `${sanitizeFilePart(modelId)}__voice-${voiceLabel}.wav`;
|
||||
const absPath = path.join(modelOutputDir, filename);
|
||||
const relPath = path.relative(options.outputDir, absPath);
|
||||
|
||||
await writeFile(absPath, pcm16MonoToWavBuffer(pcm16, sampleRate));
|
||||
|
||||
generated.push({
|
||||
modelId,
|
||||
voiceLabel,
|
||||
speakerId: null,
|
||||
sampleRate,
|
||||
durationSeconds: pcm16.length / 2 / sampleRate,
|
||||
relativePath: relPath,
|
||||
});
|
||||
|
||||
logger.info({ modelId, voiceLabel, path: relPath }, "Generated sample");
|
||||
continue;
|
||||
}
|
||||
|
||||
if (modelId !== "kokoro-en-v0_19" && modelId !== "kitten-nano-en-v0_1-fp16") {
|
||||
throw new Error(`Unsupported local Sherpa TTS model for this script: ${modelId}`);
|
||||
}
|
||||
|
||||
const { tts, speed } = createSherpaOfflineTts({
|
||||
modelId,
|
||||
modelDir,
|
||||
speed: options.speed,
|
||||
});
|
||||
|
||||
try {
|
||||
const rawNumSpeakers = (tts as { numSpeakers?: unknown }).numSpeakers;
|
||||
const numSpeakers =
|
||||
typeof rawNumSpeakers === "number" && Number.isFinite(rawNumSpeakers) && rawNumSpeakers > 0
|
||||
? Math.floor(rawNumSpeakers)
|
||||
: 1;
|
||||
const pad = Math.max(2, String(Math.max(0, numSpeakers - 1)).length);
|
||||
const fallbackSampleRate =
|
||||
typeof (tts as { sampleRate?: unknown }).sampleRate === "number"
|
||||
? Math.max(1, Math.floor((tts as { sampleRate: number }).sampleRate))
|
||||
: 24000;
|
||||
|
||||
logger.info({ modelId, numSpeakers }, "Discovered Sherpa speaker count");
|
||||
|
||||
for (let sid = 0; sid < numSpeakers; sid += 1) {
|
||||
const audio = (
|
||||
tts as {
|
||||
generate: (config: { text: string; sid: number; speed: number }) => unknown;
|
||||
}
|
||||
).generate({
|
||||
text: options.text,
|
||||
sid,
|
||||
speed,
|
||||
});
|
||||
const samples = toFloatSamples(audio);
|
||||
const sampleRate = toSampleRate(audio, fallbackSampleRate);
|
||||
const pcm16 = float32ToPcm16le(samples);
|
||||
|
||||
const voiceLabel = `sid-${String(sid).padStart(pad, "0")}`;
|
||||
const filename = `${sanitizeFilePart(modelId)}__voice-${voiceLabel}.wav`;
|
||||
const absPath = path.join(modelOutputDir, filename);
|
||||
const relPath = path.relative(options.outputDir, absPath);
|
||||
|
||||
await writeFile(absPath, pcm16MonoToWavBuffer(pcm16, sampleRate));
|
||||
|
||||
generated.push({
|
||||
modelId,
|
||||
voiceLabel,
|
||||
speakerId: sid,
|
||||
sampleRate,
|
||||
durationSeconds: pcm16.length / 2 / sampleRate,
|
||||
relativePath: relPath,
|
||||
});
|
||||
|
||||
logger.info({ modelId, voiceLabel, path: relPath }, "Generated sample");
|
||||
}
|
||||
} finally {
|
||||
(tts as { free?: () => void }).free?.();
|
||||
}
|
||||
}
|
||||
|
||||
const manifest = {
|
||||
createdAt: new Date().toISOString(),
|
||||
text: options.text,
|
||||
speed: options.speed,
|
||||
modelsDir: options.modelsDir,
|
||||
outputDir: options.outputDir,
|
||||
namingConvention: "<model-id>/<model-id>__voice-<voice-label>.wav",
|
||||
notes: [
|
||||
"For Sherpa kitten/kokoro models, voice-label uses sid-<speaker-id>.",
|
||||
"For Pocket TTS, voice-label is default.",
|
||||
],
|
||||
generated,
|
||||
};
|
||||
|
||||
await writeFile(
|
||||
path.join(options.outputDir, "manifest.json"),
|
||||
`${JSON.stringify(manifest, null, 2)}\n`,
|
||||
);
|
||||
await writeFile(path.join(options.outputDir, "samples.csv"), toCsv(generated));
|
||||
await writeFile(
|
||||
path.join(options.outputDir, "README.txt"),
|
||||
[
|
||||
"Sherpa TTS sample matrix",
|
||||
"",
|
||||
`Sample text: ${options.text}`,
|
||||
`Speed: ${options.speed}`,
|
||||
"",
|
||||
"Naming convention:",
|
||||
" <model-id>/<model-id>__voice-<voice-label>.wav",
|
||||
" voice-label = sid-<speaker-id> for Sherpa models",
|
||||
" voice-label = default for Pocket TTS",
|
||||
"",
|
||||
`Total files: ${generated.length}`,
|
||||
].join("\n"),
|
||||
);
|
||||
|
||||
logger.info(
|
||||
{
|
||||
outputDir: options.outputDir,
|
||||
generatedCount: generated.length,
|
||||
models: Array.from(new Set(generated.map((item) => item.modelId))),
|
||||
},
|
||||
"TTS matrix generation complete",
|
||||
);
|
||||
@@ -29,7 +29,7 @@ function usage(): string {
|
||||
" npm run speech:transcribe:local -- ./sample.wav --out ./tmp/sample.transcript.txt",
|
||||
"",
|
||||
"Env fallbacks:",
|
||||
" PASEO_LOCAL_MODELS_DIR, PASEO_LOCAL_STT_MODEL",
|
||||
" PASEO_LOCAL_MODELS_DIR",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
@@ -49,9 +49,7 @@ function parseArgs(argv: string[]): CliOptions {
|
||||
|
||||
const positional: string[] = [];
|
||||
let outPath: string | undefined;
|
||||
let model = LocalSttModelIdSchema.parse(
|
||||
process.env.PASEO_LOCAL_STT_MODEL ?? DEFAULT_LOCAL_STT_MODEL,
|
||||
);
|
||||
let model = LocalSttModelIdSchema.parse(DEFAULT_LOCAL_STT_MODEL);
|
||||
let modelsDir = defaultModelsDir;
|
||||
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
|
||||
@@ -147,8 +147,8 @@ describe("paseo daemon bootstrap", () => {
|
||||
local: {
|
||||
modelsDir: path.join(os.tmpdir(), `paseo-missing-models-${Date.now()}`),
|
||||
models: {
|
||||
dictationStt: "parakeet-tdt-0.6b-v3-int8",
|
||||
voiceStt: "parakeet-tdt-0.6b-v3-int8",
|
||||
dictationStt: "parakeet-tdt-0.6b-v2-int8",
|
||||
voiceStt: "parakeet-tdt-0.6b-v2-int8",
|
||||
voiceTts: "kokoro-en-v0_19",
|
||||
},
|
||||
},
|
||||
|
||||
@@ -39,35 +39,25 @@ async function readFixture(fileName: string): Promise<Buffer> {
|
||||
return readFile(fixturePath(fileName));
|
||||
}
|
||||
|
||||
function hasSherpaZipformerModels(modelsDir: string): boolean {
|
||||
function hasSherpaParakeetModels(modelsDir: string): boolean {
|
||||
return (
|
||||
existsSync(
|
||||
path.join(
|
||||
modelsDir,
|
||||
"sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20",
|
||||
"encoder-epoch-99-avg-1.onnx",
|
||||
),
|
||||
path.join(modelsDir, "sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8", "encoder.int8.onnx"),
|
||||
) &&
|
||||
existsSync(
|
||||
path.join(
|
||||
modelsDir,
|
||||
"sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20",
|
||||
"tokens.txt",
|
||||
),
|
||||
)
|
||||
existsSync(path.join(modelsDir, "sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8", "tokens.txt"))
|
||||
);
|
||||
}
|
||||
|
||||
function hasSherpaKittenModels(modelsDir: string): boolean {
|
||||
function hasSherpaKokoroModels(modelsDir: string): boolean {
|
||||
return (
|
||||
existsSync(path.join(modelsDir, "kitten-nano-en-v0_1-fp16", "model.fp16.onnx")) &&
|
||||
existsSync(path.join(modelsDir, "kitten-nano-en-v0_1-fp16", "voices.bin")) &&
|
||||
existsSync(path.join(modelsDir, "kitten-nano-en-v0_1-fp16", "tokens.txt"))
|
||||
existsSync(path.join(modelsDir, "kokoro-en-v0_19", "model.onnx")) &&
|
||||
existsSync(path.join(modelsDir, "kokoro-en-v0_19", "voices.bin")) &&
|
||||
existsSync(path.join(modelsDir, "kokoro-en-v0_19", "tokens.txt"))
|
||||
);
|
||||
}
|
||||
|
||||
const hasLocalSpeech =
|
||||
hasSherpaZipformerModels(localModelsDir) && hasSherpaKittenModels(localModelsDir);
|
||||
hasSherpaParakeetModels(localModelsDir) && hasSherpaKokoroModels(localModelsDir);
|
||||
const hasAnySpeech = hasLocalSpeech || Boolean(openaiApiKey);
|
||||
const speechTest = hasAnySpeech ? test : test.skip;
|
||||
|
||||
@@ -492,11 +482,10 @@ function resolveSpeechConfig() {
|
||||
local: {
|
||||
modelsDir: localModelsDir,
|
||||
models: {
|
||||
dictationStt:
|
||||
process.env.PASEO_DICTATION_LOCAL_STT_MODEL ?? "zipformer-bilingual-zh-en-2023-02-20",
|
||||
voiceStt:
|
||||
process.env.PASEO_VOICE_LOCAL_STT_MODEL ?? "zipformer-bilingual-zh-en-2023-02-20",
|
||||
voiceTts: process.env.PASEO_VOICE_LOCAL_TTS_MODEL ?? "kitten-nano-en-v0_1-fp16",
|
||||
dictationStt: "parakeet-tdt-0.6b-v2-int8",
|
||||
voiceStt: "parakeet-tdt-0.6b-v2-int8",
|
||||
voiceTts: "kokoro-en-v0_19",
|
||||
voiceTtsSpeakerId: 0,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
// Type augmentation for onnxruntime-node to include runtime properties
|
||||
// not exposed in the official type definitions
|
||||
|
||||
import type * as ort from "onnxruntime-node";
|
||||
|
||||
declare module "onnxruntime-node" {
|
||||
interface InferenceSession {
|
||||
/** Input tensor names (available at runtime but not in types) */
|
||||
readonly inputNames: string[];
|
||||
/** Output tensor names (available at runtime but not in types) */
|
||||
readonly outputNames?: string[];
|
||||
/** Input metadata for shape/type info (available at runtime but not in types) */
|
||||
readonly inputMetadata?: unknown;
|
||||
}
|
||||
}
|
||||
@@ -1,629 +0,0 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { Readable } from "node:stream";
|
||||
import type pino from "pino";
|
||||
|
||||
import type { SpeechStreamResult, TextToSpeechProvider } from "../../../speech-provider.js";
|
||||
import {
|
||||
chunkBuffer,
|
||||
float32ToPcm16le,
|
||||
parsePcm16MonoWav,
|
||||
pcm16leToFloat32,
|
||||
} from "../../../audio.js";
|
||||
import { Pcm16MonoResampler } from "../../../../agent/pcm16-resampler.js";
|
||||
|
||||
type OrtModule = typeof import("onnxruntime-node");
|
||||
type OrtSession = import("onnxruntime-node").InferenceSession;
|
||||
type OrtTensor = import("onnxruntime-node").Tensor;
|
||||
|
||||
function createSessionFeed(feed: Record<string, OrtTensor>): Record<string, OrtTensor> {
|
||||
return feed;
|
||||
}
|
||||
|
||||
interface SentencePieceProcessor {
|
||||
encodeIds: (text: string) => number[];
|
||||
load?: (modelPath: string) => unknown;
|
||||
Load?: (modelPath: string) => unknown;
|
||||
}
|
||||
|
||||
function assertFileExists(filePath: string, label: string): void {
|
||||
if (!existsSync(filePath)) {
|
||||
throw new Error(`Missing ${label}: ${filePath}`);
|
||||
}
|
||||
}
|
||||
|
||||
function product(dims: number[]): number {
|
||||
let out = 1;
|
||||
for (const d of dims) out *= d;
|
||||
return out;
|
||||
}
|
||||
|
||||
function normalizeDims(dims: Array<number | string | null | undefined>): number[] {
|
||||
// ONNX metadata can contain dynamic dimensions as strings (e.g. "batch") or -1.
|
||||
// For state tensors we want a valid minimal shape, so coerce unknown/invalid dims to 1.
|
||||
// Preserve explicit 0 dims (some models use empty initial state buffers with shape [0]).
|
||||
return dims.map((d) => {
|
||||
if (typeof d === "number" && Number.isFinite(d)) {
|
||||
if (d === 0) return 0;
|
||||
if (d > 0) return d;
|
||||
return 1;
|
||||
}
|
||||
return 1;
|
||||
});
|
||||
}
|
||||
|
||||
function getSessionInputMeta(
|
||||
session: OrtSession,
|
||||
inputName: string,
|
||||
): { type?: string; dims?: Array<number | string | null> } | undefined {
|
||||
const metaAny = session.inputMetadata;
|
||||
if (!Array.isArray(metaAny)) {
|
||||
return undefined;
|
||||
}
|
||||
const entry = metaAny.find((m) => m.name === inputName);
|
||||
if (!entry) return undefined;
|
||||
return { type: String(entry.type), dims: entry.shape };
|
||||
}
|
||||
|
||||
function toBigInt64(values: number[]): BigInt64Array {
|
||||
const out = new BigInt64Array(values.length);
|
||||
for (let i = 0; i < values.length; i += 1) {
|
||||
out[i] = BigInt(values[i]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function randn(): number {
|
||||
// Box–Muller
|
||||
let u = 0;
|
||||
let v = 0;
|
||||
while (u === 0) u = Math.random();
|
||||
while (v === 0) v = Math.random();
|
||||
return Math.sqrt(-2.0 * Math.log(u)) * Math.cos(2.0 * Math.PI * v);
|
||||
}
|
||||
|
||||
function normalizeTextForPocket(text: string): string {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed) {
|
||||
throw new Error("Cannot synthesize empty text");
|
||||
}
|
||||
let out = trimmed;
|
||||
if (out.length > 0 && /[A-Za-z0-9]$/.test(out)) {
|
||||
out = `${out}.`;
|
||||
}
|
||||
if (out.length > 0 && /[a-z]/.test(out[0])) {
|
||||
out = out[0].toUpperCase() + out.slice(1);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function loadOrt(): Promise<OrtModule> {
|
||||
return import("onnxruntime-node");
|
||||
}
|
||||
|
||||
interface SentencePieceModule {
|
||||
SentencePieceProcessor?: new () => SentencePieceProcessor;
|
||||
default?:
|
||||
| (new () => SentencePieceProcessor)
|
||||
| { SentencePieceProcessor?: new () => SentencePieceProcessor };
|
||||
}
|
||||
|
||||
function isSentencePieceModule(mod: unknown): mod is SentencePieceModule {
|
||||
return mod !== null && typeof mod === "object";
|
||||
}
|
||||
|
||||
function getSentencePieceProcessor(
|
||||
mod: SentencePieceModule,
|
||||
): (new () => SentencePieceProcessor) | undefined {
|
||||
if (mod.SentencePieceProcessor) {
|
||||
return mod.SentencePieceProcessor;
|
||||
}
|
||||
const defaultValue = mod.default;
|
||||
if (
|
||||
defaultValue &&
|
||||
typeof defaultValue === "object" &&
|
||||
"SentencePieceProcessor" in defaultValue
|
||||
) {
|
||||
return defaultValue.SentencePieceProcessor;
|
||||
}
|
||||
if (typeof defaultValue === "function") {
|
||||
return defaultValue as new () => SentencePieceProcessor;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async function loadSentencePiece(tokenizerModelPath: string): Promise<SentencePieceProcessor> {
|
||||
const mod = await import("@sctg/sentencepiece-js");
|
||||
|
||||
if (!isSentencePieceModule(mod)) {
|
||||
throw new Error("@sctg/sentencepiece-js module has unexpected shape");
|
||||
}
|
||||
|
||||
const Processor = getSentencePieceProcessor(mod);
|
||||
|
||||
if (!Processor) {
|
||||
throw new Error("Failed to load SentencePiece processor from @sctg/sentencepiece-js");
|
||||
}
|
||||
|
||||
const sp = new Processor();
|
||||
|
||||
if (typeof sp.load === "function") {
|
||||
await sp.load(tokenizerModelPath);
|
||||
} else if (typeof sp.Load === "function") {
|
||||
sp.Load(tokenizerModelPath);
|
||||
} else {
|
||||
throw new Error("SentencePiece processor does not expose load()/Load()");
|
||||
}
|
||||
|
||||
return sp;
|
||||
}
|
||||
|
||||
function getOrtProviders(ort: OrtModule, device: "auto" | "cpu" | "cuda"): string[] {
|
||||
// NOTE: onnxruntime-node uses backend names like "cpu"/"coreml"/"webgpu" (not "CPUExecutionProvider").
|
||||
if (device === "cpu") return ["cpu"];
|
||||
if (device === "cuda") return ["cuda", "cpu"];
|
||||
// auto
|
||||
// CoreML EP does not support some dynamic/zero-length shapes used by Pocket TTS (e.g. [1, 0, 32]).
|
||||
// Default to CPU to keep behavior predictable across platforms.
|
||||
void ort;
|
||||
return ["cpu"];
|
||||
}
|
||||
|
||||
function createZeroTensorForInput(
|
||||
ort: OrtModule,
|
||||
session: OrtSession,
|
||||
inputName: string,
|
||||
): OrtTensor {
|
||||
const meta = getSessionInputMeta(session, inputName);
|
||||
const dims = normalizeDims(meta?.dims ?? []);
|
||||
if (dims.length === 0) {
|
||||
throw new Error(`Missing input metadata shape for ${inputName}`);
|
||||
}
|
||||
|
||||
const type = (meta?.type ?? "float32").toLowerCase();
|
||||
const size = product(dims);
|
||||
|
||||
if (type.includes("int64")) {
|
||||
return new ort.Tensor("int64", new BigInt64Array(size), dims);
|
||||
}
|
||||
if (type.includes("bool")) {
|
||||
return new ort.Tensor("bool", new Uint8Array(size), dims);
|
||||
}
|
||||
return new ort.Tensor("float32", new Float32Array(size), dims);
|
||||
}
|
||||
|
||||
function initState(session: OrtSession, ort: OrtModule): Record<string, OrtTensor> {
|
||||
const out: Record<string, OrtTensor> = {};
|
||||
for (const name of session.inputNames) {
|
||||
if (name.startsWith("state_")) {
|
||||
out[name] = createZeroTensorForInput(ort, session, name);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function updateStateFromOutputs(
|
||||
state: Record<string, OrtTensor>,
|
||||
outputs: Record<string, OrtTensor>,
|
||||
): void {
|
||||
for (const [name, tensor] of Object.entries(outputs)) {
|
||||
if (!name.startsWith("out_state_")) continue;
|
||||
const idx = Number.parseInt(name.replace("out_state_", ""), 10);
|
||||
if (Number.isFinite(idx)) {
|
||||
state[`state_${idx}`] = tensor;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
interface OrtTensorWithData {
|
||||
data: unknown;
|
||||
}
|
||||
|
||||
function tensorDataFloat32(t: OrtTensor): Float32Array {
|
||||
const tensorWithData = t as OrtTensorWithData;
|
||||
const data = tensorWithData.data;
|
||||
if (data instanceof Float32Array) return data;
|
||||
if (Array.isArray(data)) return Float32Array.from(data);
|
||||
throw new Error("Unexpected tensor data type (expected Float32Array)");
|
||||
}
|
||||
|
||||
export interface PocketTtsOnnxConfig {
|
||||
modelDir: string;
|
||||
precision?: "int8" | "fp32";
|
||||
device?: "auto" | "cpu" | "cuda";
|
||||
temperature?: number;
|
||||
lsdSteps?: number;
|
||||
maxFrames?: number;
|
||||
framesAfterEos?: number;
|
||||
firstChunkFrames?: number;
|
||||
maxChunkFrames?: number;
|
||||
targetChunkMs?: number;
|
||||
referenceAudioFile?: string;
|
||||
}
|
||||
|
||||
class PocketTtsOnnxEngine {
|
||||
static readonly SAMPLE_RATE = 24000;
|
||||
static readonly SAMPLES_PER_FRAME = 1920;
|
||||
|
||||
private readonly ort: OrtModule;
|
||||
|
||||
private readonly temperature: number;
|
||||
private readonly lsdSteps: number;
|
||||
private readonly maxFrames: number;
|
||||
private readonly framesAfterEos: number;
|
||||
|
||||
private readonly firstChunkFrames: number;
|
||||
private readonly maxChunkFrames: number;
|
||||
|
||||
private readonly tokenizer: SentencePieceProcessor;
|
||||
private readonly textConditioner: OrtSession;
|
||||
private readonly flowLmMain: OrtSession;
|
||||
private readonly flowLmFlow: OrtSession;
|
||||
private readonly mimiDecoder: OrtSession;
|
||||
|
||||
private readonly stBuffers: Array<{ s: OrtTensor; t: OrtTensor }>;
|
||||
private readonly voiceEmbeddings: OrtTensor;
|
||||
|
||||
private constructor(args: {
|
||||
ort: OrtModule;
|
||||
temperature: number;
|
||||
lsdSteps: number;
|
||||
maxFrames: number;
|
||||
framesAfterEos: number;
|
||||
firstChunkFrames: number;
|
||||
maxChunkFrames: number;
|
||||
tokenizer: SentencePieceProcessor;
|
||||
textConditioner: OrtSession;
|
||||
flowLmMain: OrtSession;
|
||||
flowLmFlow: OrtSession;
|
||||
mimiDecoder: OrtSession;
|
||||
stBuffers: Array<{ s: OrtTensor; t: OrtTensor }>;
|
||||
voiceEmbeddings: OrtTensor;
|
||||
}) {
|
||||
this.ort = args.ort;
|
||||
this.temperature = args.temperature;
|
||||
this.lsdSteps = args.lsdSteps;
|
||||
this.maxFrames = args.maxFrames;
|
||||
this.framesAfterEos = args.framesAfterEos;
|
||||
this.firstChunkFrames = args.firstChunkFrames;
|
||||
this.maxChunkFrames = args.maxChunkFrames;
|
||||
this.tokenizer = args.tokenizer;
|
||||
this.textConditioner = args.textConditioner;
|
||||
this.flowLmMain = args.flowLmMain;
|
||||
this.flowLmFlow = args.flowLmFlow;
|
||||
this.mimiDecoder = args.mimiDecoder;
|
||||
this.stBuffers = args.stBuffers;
|
||||
this.voiceEmbeddings = args.voiceEmbeddings;
|
||||
}
|
||||
|
||||
static async create(
|
||||
config: PocketTtsOnnxConfig,
|
||||
logger: pino.Logger,
|
||||
): Promise<PocketTtsOnnxEngine> {
|
||||
const log = logger.child({
|
||||
module: "speech",
|
||||
provider: "pocket-tts",
|
||||
component: "onnx-engine",
|
||||
});
|
||||
|
||||
const modelDir = config.modelDir;
|
||||
const onnxDir = `${modelDir}/onnx`;
|
||||
const precision = config.precision ?? "int8";
|
||||
const device = config.device ?? "auto";
|
||||
const temperature = config.temperature ?? 0.7;
|
||||
const lsdSteps = config.lsdSteps ?? 10;
|
||||
const maxFrames = config.maxFrames ?? 500;
|
||||
const framesAfterEos = config.framesAfterEos ?? 3;
|
||||
const firstChunkFrames = config.firstChunkFrames ?? 2;
|
||||
const maxChunkFrames = config.maxChunkFrames ?? 15;
|
||||
|
||||
const tokenizerPath = `${modelDir}/tokenizer.model`;
|
||||
const referenceAudioFile = config.referenceAudioFile ?? `${modelDir}/reference_sample.wav`;
|
||||
|
||||
const flowMainFile = precision === "int8" ? "flow_lm_main_int8.onnx" : "flow_lm_main.onnx";
|
||||
const flowFlowFile = precision === "int8" ? "flow_lm_flow_int8.onnx" : "flow_lm_flow.onnx";
|
||||
const decoderFile = precision === "int8" ? "mimi_decoder_int8.onnx" : "mimi_decoder.onnx";
|
||||
|
||||
assertFileExists(`${onnxDir}/mimi_encoder.onnx`, "PocketTTS mimi_encoder");
|
||||
assertFileExists(`${onnxDir}/text_conditioner.onnx`, "PocketTTS text_conditioner");
|
||||
assertFileExists(`${onnxDir}/${flowMainFile}`, "PocketTTS flow_lm_main");
|
||||
assertFileExists(`${onnxDir}/${flowFlowFile}`, "PocketTTS flow_lm_flow");
|
||||
assertFileExists(`${onnxDir}/${decoderFile}`, "PocketTTS mimi_decoder");
|
||||
assertFileExists(tokenizerPath, "PocketTTS tokenizer.model");
|
||||
assertFileExists(referenceAudioFile, "PocketTTS reference_sample.wav");
|
||||
|
||||
const ort = await loadOrt();
|
||||
const providers = getOrtProviders(ort, device);
|
||||
|
||||
const [tokenizer, mimiEncoder, textConditioner, flowLmMain, flowLmFlow, mimiDecoder] =
|
||||
await Promise.all([
|
||||
loadSentencePiece(tokenizerPath),
|
||||
ort.InferenceSession.create(`${onnxDir}/mimi_encoder.onnx`, {
|
||||
executionProviders: providers,
|
||||
}),
|
||||
ort.InferenceSession.create(`${onnxDir}/text_conditioner.onnx`, {
|
||||
executionProviders: providers,
|
||||
}),
|
||||
ort.InferenceSession.create(`${onnxDir}/${flowMainFile}`, {
|
||||
executionProviders: providers,
|
||||
}),
|
||||
ort.InferenceSession.create(`${onnxDir}/${flowFlowFile}`, {
|
||||
executionProviders: providers,
|
||||
}),
|
||||
ort.InferenceSession.create(`${onnxDir}/${decoderFile}`, { executionProviders: providers }),
|
||||
]);
|
||||
|
||||
// Precompute flow matching time-step buffers.
|
||||
const stBuffers: Array<{ s: OrtTensor; t: OrtTensor }> = [];
|
||||
for (let j = 0; j < lsdSteps; j += 1) {
|
||||
const s = j / lsdSteps;
|
||||
const t = s + 1.0 / lsdSteps;
|
||||
stBuffers.push({
|
||||
s: new ort.Tensor("float32", new Float32Array([s]), [1, 1]),
|
||||
t: new ort.Tensor("float32", new Float32Array([t]), [1, 1]),
|
||||
});
|
||||
}
|
||||
|
||||
// Precompute reference voice embeddings once.
|
||||
const refWav = await readFile(referenceAudioFile);
|
||||
const parsed = parsePcm16MonoWav(refWav);
|
||||
let pcm16 = parsed.pcm16;
|
||||
if (parsed.sampleRate !== PocketTtsOnnxEngine.SAMPLE_RATE) {
|
||||
const resampler = new Pcm16MonoResampler({
|
||||
inputRate: parsed.sampleRate,
|
||||
outputRate: PocketTtsOnnxEngine.SAMPLE_RATE,
|
||||
});
|
||||
pcm16 = resampler.processChunk(pcm16);
|
||||
}
|
||||
const floatAudio = pcm16leToFloat32(pcm16);
|
||||
const audioTensor = new ort.Tensor("float32", floatAudio, [1, 1, floatAudio.length]);
|
||||
|
||||
const encoded = await mimiEncoder.run({ audio: audioTensor });
|
||||
const firstOutName = mimiEncoder.outputNames?.[0];
|
||||
const encodedRecord = encoded as Record<string, OrtTensor>;
|
||||
const voiceEmb = firstOutName ? encodedRecord[firstOutName] : Object.values(encodedRecord)[0];
|
||||
if (!voiceEmb) {
|
||||
throw new Error("PocketTTS mimi_encoder: missing output");
|
||||
}
|
||||
|
||||
log.info({ precision, device, providers, lsdSteps, temperature }, "PocketTTS ONNX initialized");
|
||||
|
||||
return new PocketTtsOnnxEngine({
|
||||
ort,
|
||||
temperature,
|
||||
lsdSteps,
|
||||
maxFrames,
|
||||
framesAfterEos,
|
||||
firstChunkFrames,
|
||||
maxChunkFrames,
|
||||
tokenizer,
|
||||
textConditioner,
|
||||
flowLmMain,
|
||||
flowLmFlow,
|
||||
mimiDecoder,
|
||||
stBuffers,
|
||||
voiceEmbeddings: voiceEmb,
|
||||
});
|
||||
}
|
||||
|
||||
private tokenize(text: string): OrtTensor {
|
||||
const normalized = normalizeTextForPocket(text);
|
||||
const ids = this.tokenizer.encodeIds(normalized);
|
||||
const data = toBigInt64(ids ?? []);
|
||||
return new this.ort.Tensor("int64", data, [1, data.length]);
|
||||
}
|
||||
|
||||
private async runTextConditioner(tokenIds: OrtTensor): Promise<OrtTensor> {
|
||||
const feed: Record<string, OrtTensor> = { token_ids: tokenIds };
|
||||
const out = await this.textConditioner.run(feed);
|
||||
const firstOutName = this.textConditioner.outputNames?.[0];
|
||||
const outRecord = out as Record<string, OrtTensor>;
|
||||
const t = firstOutName ? outRecord[firstOutName] : Object.values(outRecord)[0];
|
||||
if (!t) throw new Error("PocketTTS text_conditioner: missing output");
|
||||
return t;
|
||||
}
|
||||
|
||||
private async *runFlowLm(textEmbeddings: OrtTensor): AsyncGenerator<Float32Array> {
|
||||
const ort = this.ort;
|
||||
const state = initState(this.flowLmMain, ort);
|
||||
|
||||
const emptySeq = new ort.Tensor("float32", new Float32Array(0), [1, 0, 32]);
|
||||
const emptyText = new ort.Tensor("float32", new Float32Array(0), [1, 0, 1024]);
|
||||
|
||||
// Voice conditioning pass
|
||||
const resVoice = await this.flowLmMain.run(
|
||||
createSessionFeed({
|
||||
sequence: emptySeq,
|
||||
text_embeddings: this.voiceEmbeddings,
|
||||
...state,
|
||||
}),
|
||||
);
|
||||
updateStateFromOutputs(state, resVoice as Record<string, OrtTensor>);
|
||||
|
||||
// Text conditioning pass
|
||||
const resText = await this.flowLmMain.run(
|
||||
createSessionFeed({
|
||||
sequence: emptySeq,
|
||||
text_embeddings: textEmbeddings,
|
||||
...state,
|
||||
}),
|
||||
);
|
||||
updateStateFromOutputs(state, resText as Record<string, OrtTensor>);
|
||||
|
||||
// Autoregressive generation
|
||||
const curr = new Float32Array(32);
|
||||
curr.fill(Number.NaN);
|
||||
let currTensor = new ort.Tensor("float32", curr, [1, 1, 32]);
|
||||
|
||||
const dt = 1.0 / this.lsdSteps;
|
||||
let eosStep: number | null = null;
|
||||
|
||||
for (let step = 0; step < this.maxFrames; step += 1) {
|
||||
const resStep = await this.flowLmMain.run(
|
||||
createSessionFeed({
|
||||
sequence: currTensor,
|
||||
text_embeddings: emptyText,
|
||||
...state,
|
||||
}),
|
||||
);
|
||||
|
||||
const outputNames = this.flowLmMain.outputNames;
|
||||
const resStepRecord = resStep as Record<string, OrtTensor>;
|
||||
const conditioningName = outputNames?.[0] ?? Object.keys(resStepRecord)[0];
|
||||
const eosName = outputNames?.[1] ?? Object.keys(resStepRecord)[1];
|
||||
|
||||
const conditioning = resStepRecord[conditioningName];
|
||||
const eos = resStepRecord[eosName];
|
||||
if (!conditioning || !eos) {
|
||||
throw new Error("PocketTTS flow_lm_main: missing conditioning/EOS outputs");
|
||||
}
|
||||
updateStateFromOutputs(state, resStepRecord);
|
||||
|
||||
const eosData = tensorDataFloat32(eos);
|
||||
if (eosData[0] > -4.0 && eosStep === null) {
|
||||
eosStep = step;
|
||||
}
|
||||
if (eosStep !== null && step >= eosStep + this.framesAfterEos) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Flow matching with external Euler loop.
|
||||
const std = this.temperature > 0 ? Math.sqrt(this.temperature) : 0;
|
||||
const x = new Float32Array(32);
|
||||
if (std > 0) {
|
||||
for (let i = 0; i < x.length; i += 1) {
|
||||
x[i] = randn() * std;
|
||||
}
|
||||
}
|
||||
|
||||
for (const st of this.stBuffers) {
|
||||
const xTensor = new ort.Tensor("float32", x, [1, 32]);
|
||||
const flowOut = await this.flowLmFlow.run(
|
||||
createSessionFeed({
|
||||
c: conditioning,
|
||||
s: st.s,
|
||||
t: st.t,
|
||||
x: xTensor,
|
||||
}),
|
||||
);
|
||||
const first = this.flowLmFlow.outputNames?.[0];
|
||||
const flowOutRecord = flowOut as Record<string, OrtTensor>;
|
||||
const flowTensor = first ? flowOutRecord[first] : Object.values(flowOutRecord)[0];
|
||||
if (!flowTensor) throw new Error("PocketTTS flow_lm_flow: missing output");
|
||||
const delta = tensorDataFloat32(flowTensor);
|
||||
for (let i = 0; i < x.length; i += 1) {
|
||||
x[i] = x[i] + delta[i] * dt;
|
||||
}
|
||||
}
|
||||
|
||||
yield x;
|
||||
currTensor = new ort.Tensor("float32", x, [1, 1, 32]);
|
||||
}
|
||||
}
|
||||
|
||||
private async decodeLatentsChunk(
|
||||
frames: Float32Array[],
|
||||
state: Record<string, OrtTensor>,
|
||||
): Promise<Float32Array> {
|
||||
const ort = this.ort;
|
||||
const frameCount = frames.length;
|
||||
const flattened = new Float32Array(frameCount * 32);
|
||||
for (let i = 0; i < frameCount; i += 1) {
|
||||
flattened.set(frames[i], i * 32);
|
||||
}
|
||||
const latent = new ort.Tensor("float32", flattened, [1, frameCount, 32]);
|
||||
|
||||
const out = await this.mimiDecoder.run(createSessionFeed({ latent, ...state }));
|
||||
const outRecord = out as Record<string, OrtTensor>;
|
||||
updateStateFromOutputs(state, outRecord);
|
||||
|
||||
const firstOutName = this.mimiDecoder.outputNames?.[0];
|
||||
const audioTensor = firstOutName ? outRecord[firstOutName] : Object.values(outRecord)[0];
|
||||
if (!audioTensor) {
|
||||
throw new Error("PocketTTS mimi_decoder: missing audio output");
|
||||
}
|
||||
return tensorDataFloat32(audioTensor);
|
||||
}
|
||||
|
||||
async *streamAudio(text: string): AsyncGenerator<Float32Array> {
|
||||
const tokenIds = this.tokenize(text);
|
||||
const textEmb = await this.runTextConditioner(tokenIds);
|
||||
|
||||
const decoderState = initState(this.mimiDecoder, this.ort);
|
||||
|
||||
const generated: Float32Array[] = [];
|
||||
let decodedFrames = 0;
|
||||
|
||||
for await (const latent of this.runFlowLm(textEmb)) {
|
||||
generated.push(latent);
|
||||
const pending = generated.length - decodedFrames;
|
||||
|
||||
let chunkSize = 0;
|
||||
if (decodedFrames === 0) {
|
||||
if (pending >= this.firstChunkFrames) {
|
||||
chunkSize = this.firstChunkFrames;
|
||||
}
|
||||
} else if (pending >= this.maxChunkFrames) {
|
||||
chunkSize = this.maxChunkFrames;
|
||||
}
|
||||
|
||||
if (chunkSize > 0) {
|
||||
const audio = await this.decodeLatentsChunk(
|
||||
generated.slice(decodedFrames, decodedFrames + chunkSize),
|
||||
decoderState,
|
||||
);
|
||||
decodedFrames += chunkSize;
|
||||
yield audio;
|
||||
}
|
||||
}
|
||||
|
||||
if (decodedFrames < generated.length) {
|
||||
const audio = await this.decodeLatentsChunk(generated.slice(decodedFrames), decoderState);
|
||||
yield audio;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class PocketTtsOnnxTTS implements TextToSpeechProvider {
|
||||
private readonly engine: PocketTtsOnnxEngine;
|
||||
private readonly chunkMs: number;
|
||||
private readonly logger: pino.Logger;
|
||||
|
||||
private constructor(engine: PocketTtsOnnxEngine, logger: pino.Logger, chunkMs: number) {
|
||||
this.engine = engine;
|
||||
this.chunkMs = chunkMs;
|
||||
this.logger = logger.child({ module: "speech", provider: "pocket-tts", component: "tts" });
|
||||
}
|
||||
|
||||
static async create(config: PocketTtsOnnxConfig, logger: pino.Logger): Promise<PocketTtsOnnxTTS> {
|
||||
const engine = await PocketTtsOnnxEngine.create(config, logger);
|
||||
const chunkMs = config.targetChunkMs ?? 50;
|
||||
return new PocketTtsOnnxTTS(engine, logger, chunkMs);
|
||||
}
|
||||
|
||||
async synthesizeSpeech(text: string): Promise<SpeechStreamResult> {
|
||||
const start = Date.now();
|
||||
const sampleRate = PocketTtsOnnxEngine.SAMPLE_RATE;
|
||||
|
||||
const pcmChunkBytes = Math.max(2, Math.round((sampleRate * this.chunkMs) / 1000) * 2);
|
||||
|
||||
const iterable = (async function* (engine: PocketTtsOnnxEngine) {
|
||||
for await (const floatChunk of engine.streamAudio(text)) {
|
||||
const pcm = float32ToPcm16le(floatChunk);
|
||||
for (const chunk of chunkBuffer(pcm, pcmChunkBytes)) {
|
||||
yield chunk;
|
||||
}
|
||||
}
|
||||
})(this.engine);
|
||||
|
||||
this.logger.debug(
|
||||
{ ms: Date.now() - start, textLength: text.length },
|
||||
"PocketTTS stream ready",
|
||||
);
|
||||
|
||||
return {
|
||||
stream: Readable.from(iterable),
|
||||
format: `pcm;rate=${sampleRate}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,6 @@ import type { PaseoSpeechConfig } from "../../../bootstrap.js";
|
||||
import type { SpeechToTextProvider, TextToSpeechProvider } from "../../speech-provider.js";
|
||||
import type { RequestedSpeechProviders } from "../../speech-types.js";
|
||||
import type { TurnDetectionProvider } from "../../turn-detection-provider.js";
|
||||
import { PocketTtsOnnxTTS } from "./pocket/pocket-tts-onnx.js";
|
||||
import {
|
||||
getLocalSpeechModelDir,
|
||||
DEFAULT_LOCAL_STT_MODEL,
|
||||
@@ -16,20 +15,15 @@ import {
|
||||
type LocalTtsModelId,
|
||||
} from "./models.js";
|
||||
import { SherpaOfflineRecognizerEngine } from "./sherpa/sherpa-offline-recognizer.js";
|
||||
import { SherpaOnlineRecognizerEngine } from "./sherpa/sherpa-online-recognizer.js";
|
||||
import { SherpaOnnxParakeetSTT } from "./sherpa/sherpa-parakeet-stt.js";
|
||||
import { SherpaParakeetRealtimeTranscriptionSession } from "./sherpa/sherpa-parakeet-realtime-session.js";
|
||||
import { SherpaRealtimeTranscriptionSession } from "./sherpa/sherpa-realtime-session.js";
|
||||
import { SherpaOnnxSTT } from "./sherpa/sherpa-stt.js";
|
||||
import { SherpaOnnxTTS } from "./sherpa/sherpa-tts.js";
|
||||
import {
|
||||
ensureSileroVadModel,
|
||||
SherpaSileroTurnDetectionProvider,
|
||||
} from "./sherpa/silero-vad-provider.js";
|
||||
|
||||
type LocalSttEngine =
|
||||
| { kind: "offline"; engine: SherpaOfflineRecognizerEngine }
|
||||
| { kind: "online"; engine: SherpaOnlineRecognizerEngine };
|
||||
type LocalSttEngine = SherpaOfflineRecognizerEngine;
|
||||
|
||||
interface ResolvedLocalModels {
|
||||
dictationLocalSttModel: LocalSttModelId;
|
||||
@@ -117,70 +111,21 @@ async function createLocalSttEngine(params: {
|
||||
}): Promise<LocalSttEngine> {
|
||||
const { modelId, modelsDir, logger } = params;
|
||||
|
||||
if (modelId === "parakeet-tdt-0.6b-v3-int8" || modelId === "parakeet-tdt-0.6b-v2-int8") {
|
||||
const modelDir = getLocalSpeechModelDir(modelsDir, modelId);
|
||||
return {
|
||||
kind: "offline",
|
||||
engine: new SherpaOfflineRecognizerEngine(
|
||||
{
|
||||
model: {
|
||||
kind: "nemo_transducer",
|
||||
encoder: `${modelDir}/encoder.int8.onnx`,
|
||||
decoder: `${modelDir}/decoder.int8.onnx`,
|
||||
joiner: `${modelDir}/joiner.int8.onnx`,
|
||||
tokens: `${modelDir}/tokens.txt`,
|
||||
},
|
||||
numThreads: 2,
|
||||
debug: 0,
|
||||
},
|
||||
logger,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
if (modelId === "paraformer-bilingual-zh-en") {
|
||||
const modelDir = getLocalSpeechModelDir(modelsDir, modelId);
|
||||
return {
|
||||
kind: "online",
|
||||
engine: new SherpaOnlineRecognizerEngine(
|
||||
{
|
||||
model: {
|
||||
kind: "paraformer",
|
||||
encoder: `${modelDir}/encoder.int8.onnx`,
|
||||
decoder: `${modelDir}/decoder.int8.onnx`,
|
||||
tokens: `${modelDir}/tokens.txt`,
|
||||
},
|
||||
numThreads: 1,
|
||||
debug: 0,
|
||||
},
|
||||
logger,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
if (modelId === "zipformer-bilingual-zh-en-2023-02-20") {
|
||||
const modelDir = getLocalSpeechModelDir(modelsDir, modelId);
|
||||
return {
|
||||
kind: "online",
|
||||
engine: new SherpaOnlineRecognizerEngine(
|
||||
{
|
||||
model: {
|
||||
kind: "transducer",
|
||||
encoder: `${modelDir}/encoder-epoch-99-avg-1.onnx`,
|
||||
decoder: `${modelDir}/decoder-epoch-99-avg-1.onnx`,
|
||||
joiner: `${modelDir}/joiner-epoch-99-avg-1.onnx`,
|
||||
tokens: `${modelDir}/tokens.txt`,
|
||||
modelType: "zipformer",
|
||||
},
|
||||
numThreads: 1,
|
||||
debug: 0,
|
||||
},
|
||||
logger,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error(`Unsupported local STT model '${modelId}'`);
|
||||
const modelDir = getLocalSpeechModelDir(modelsDir, modelId);
|
||||
return new SherpaOfflineRecognizerEngine(
|
||||
{
|
||||
model: {
|
||||
kind: "nemo_transducer",
|
||||
encoder: `${modelDir}/encoder.int8.onnx`,
|
||||
decoder: `${modelDir}/decoder.int8.onnx`,
|
||||
joiner: `${modelDir}/joiner.int8.onnx`,
|
||||
tokens: `${modelDir}/tokens.txt`,
|
||||
},
|
||||
numThreads: 2,
|
||||
debug: 0,
|
||||
},
|
||||
logger,
|
||||
);
|
||||
}
|
||||
|
||||
type LocalConfig = NonNullable<PaseoSpeechConfig["local"]>;
|
||||
@@ -219,13 +164,7 @@ async function initializeLocalVoiceStt(params: {
|
||||
return null;
|
||||
}
|
||||
const voiceEngine = await getLocalSttEngine(modelId);
|
||||
if (voiceEngine?.kind === "offline") {
|
||||
return new SherpaOnnxParakeetSTT({ engine: voiceEngine.engine }, logger);
|
||||
}
|
||||
if (voiceEngine?.kind === "online") {
|
||||
return new SherpaOnnxSTT({ engine: voiceEngine.engine }, logger);
|
||||
}
|
||||
return null;
|
||||
return voiceEngine ? new SherpaOnnxParakeetSTT({ engine: voiceEngine }, logger) : null;
|
||||
}
|
||||
|
||||
async function initializeLocalDictationStt(params: {
|
||||
@@ -243,18 +182,11 @@ async function initializeLocalDictationStt(params: {
|
||||
return null;
|
||||
}
|
||||
const dictationEngine = await getLocalSttEngine(modelId);
|
||||
if (dictationEngine?.kind === "offline") {
|
||||
if (dictationEngine) {
|
||||
return {
|
||||
id: "local",
|
||||
createSession: () =>
|
||||
new SherpaParakeetRealtimeTranscriptionSession({ engine: dictationEngine.engine }),
|
||||
};
|
||||
}
|
||||
if (dictationEngine?.kind === "online") {
|
||||
return {
|
||||
id: "local",
|
||||
createSession: () =>
|
||||
new SherpaRealtimeTranscriptionSession({ engine: dictationEngine.engine }),
|
||||
new SherpaParakeetRealtimeTranscriptionSession({ engine: dictationEngine }),
|
||||
};
|
||||
}
|
||||
return null;
|
||||
@@ -276,16 +208,6 @@ async function initializeLocalVoiceTts(params: {
|
||||
}
|
||||
try {
|
||||
const modelDir = getLocalSpeechModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel);
|
||||
if (localModels.voiceLocalTtsModel === "pocket-tts-onnx-int8") {
|
||||
return await PocketTtsOnnxTTS.create(
|
||||
{
|
||||
modelDir,
|
||||
precision: "int8",
|
||||
targetChunkMs: 50,
|
||||
},
|
||||
logger,
|
||||
);
|
||||
}
|
||||
return new SherpaOnnxTTS(
|
||||
{
|
||||
preset: localModels.voiceLocalTtsModel,
|
||||
@@ -401,7 +323,7 @@ export async function initializeLocalSpeechServices(params: {
|
||||
maybeFreeable.free();
|
||||
}
|
||||
for (const engine of localSttEngines.values()) {
|
||||
engine.engine.free();
|
||||
engine.free();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -1,44 +1,19 @@
|
||||
import { z } from "zod";
|
||||
|
||||
export type SherpaOnnxModelKind = "stt-online" | "stt-offline" | "tts";
|
||||
export type SherpaOnnxModelKind = "stt-offline" | "tts";
|
||||
|
||||
type DefaultModelRole = "stt" | "tts";
|
||||
|
||||
interface SherpaOnnxCatalogEntry {
|
||||
kind: SherpaOnnxModelKind;
|
||||
archiveUrl?: string;
|
||||
downloadFiles?: Array<{ url: string; relPath: string }>;
|
||||
archiveUrl: string;
|
||||
extractedDir: string;
|
||||
requiredFiles: string[];
|
||||
description: string;
|
||||
aliases?: readonly string[];
|
||||
defaultFor?: DefaultModelRole;
|
||||
}
|
||||
|
||||
export const SHERPA_ONNX_MODEL_CATALOG = {
|
||||
"zipformer-bilingual-zh-en-2023-02-20": {
|
||||
kind: "stt-online",
|
||||
archiveUrl:
|
||||
"https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2",
|
||||
extractedDir: "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20",
|
||||
requiredFiles: [
|
||||
"encoder-epoch-99-avg-1.onnx",
|
||||
"decoder-epoch-99-avg-1.onnx",
|
||||
"joiner-epoch-99-avg-1.onnx",
|
||||
"tokens.txt",
|
||||
],
|
||||
description: "Streaming Zipformer transducer (fast, good accuracy).",
|
||||
aliases: ["zipformer", "zipformer-bilingual"],
|
||||
},
|
||||
"paraformer-bilingual-zh-en": {
|
||||
kind: "stt-online",
|
||||
archiveUrl:
|
||||
"https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2",
|
||||
extractedDir: "sherpa-onnx-streaming-paraformer-bilingual-zh-en",
|
||||
requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "tokens.txt"],
|
||||
description: "Streaming Paraformer (often strong accuracy; heavier).",
|
||||
aliases: ["paraformer"],
|
||||
},
|
||||
"parakeet-tdt-0.6b-v2-int8": {
|
||||
kind: "stt-offline",
|
||||
archiveUrl:
|
||||
@@ -46,27 +21,8 @@ export const SHERPA_ONNX_MODEL_CATALOG = {
|
||||
extractedDir: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8",
|
||||
requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", "tokens.txt"],
|
||||
description: "NVIDIA Parakeet TDT v2 (offline NeMo transducer, English).",
|
||||
aliases: ["parakeet-v2", "parakeet-tdt-v2"],
|
||||
defaultFor: "stt",
|
||||
},
|
||||
"parakeet-tdt-0.6b-v3-int8": {
|
||||
kind: "stt-offline",
|
||||
archiveUrl:
|
||||
"https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2",
|
||||
extractedDir: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8",
|
||||
requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", "tokens.txt"],
|
||||
description: "NVIDIA Parakeet TDT v3 (offline NeMo transducer, multilingual).",
|
||||
aliases: ["parakeet", "parakeet-v3", "parakeet-tdt"],
|
||||
},
|
||||
"kitten-nano-en-v0_1-fp16": {
|
||||
kind: "tts",
|
||||
archiveUrl:
|
||||
"https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2",
|
||||
extractedDir: "kitten-nano-en-v0_1-fp16",
|
||||
requiredFiles: ["model.fp16.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"],
|
||||
description: "KittenTTS (small, fast English TTS).",
|
||||
aliases: ["kitten"],
|
||||
},
|
||||
"kokoro-en-v0_19": {
|
||||
kind: "tts",
|
||||
archiveUrl:
|
||||
@@ -74,54 +30,8 @@ export const SHERPA_ONNX_MODEL_CATALOG = {
|
||||
extractedDir: "kokoro-en-v0_19",
|
||||
requiredFiles: ["model.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"],
|
||||
description: "Kokoro TTS (higher quality; larger).",
|
||||
aliases: ["kokoro"],
|
||||
defaultFor: "tts",
|
||||
},
|
||||
"pocket-tts-onnx-int8": {
|
||||
kind: "tts",
|
||||
extractedDir: "pocket-tts-onnx-int8",
|
||||
downloadFiles: [
|
||||
{
|
||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/mimi_encoder.onnx",
|
||||
relPath: "onnx/mimi_encoder.onnx",
|
||||
},
|
||||
{
|
||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/text_conditioner.onnx",
|
||||
relPath: "onnx/text_conditioner.onnx",
|
||||
},
|
||||
{
|
||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/flow_lm_main_int8.onnx",
|
||||
relPath: "onnx/flow_lm_main_int8.onnx",
|
||||
},
|
||||
{
|
||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/flow_lm_flow_int8.onnx",
|
||||
relPath: "onnx/flow_lm_flow_int8.onnx",
|
||||
},
|
||||
{
|
||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/mimi_decoder_int8.onnx",
|
||||
relPath: "onnx/mimi_decoder_int8.onnx",
|
||||
},
|
||||
{
|
||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/tokenizer.model",
|
||||
relPath: "tokenizer.model",
|
||||
},
|
||||
{
|
||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/reference_sample.wav",
|
||||
relPath: "reference_sample.wav",
|
||||
},
|
||||
],
|
||||
requiredFiles: [
|
||||
"onnx/mimi_encoder.onnx",
|
||||
"onnx/text_conditioner.onnx",
|
||||
"onnx/flow_lm_main_int8.onnx",
|
||||
"onnx/flow_lm_flow_int8.onnx",
|
||||
"onnx/mimi_decoder_int8.onnx",
|
||||
"tokenizer.model",
|
||||
"reference_sample.wav",
|
||||
],
|
||||
description: "Pocket TTS ONNX (INT8) with streaming decode support (via onnxruntime).",
|
||||
aliases: ["pocket", "pocket-tts"],
|
||||
},
|
||||
} as const satisfies Record<string, SherpaOnnxCatalogEntry>;
|
||||
|
||||
export type SherpaOnnxModelId = keyof typeof SHERPA_ONNX_MODEL_CATALOG;
|
||||
@@ -131,7 +41,7 @@ type ModelIdByKind<K extends SherpaOnnxModelKind> = {
|
||||
[Id in SherpaOnnxModelId]: (typeof SHERPA_ONNX_MODEL_CATALOG)[Id]["kind"] extends K ? Id : never;
|
||||
}[SherpaOnnxModelId];
|
||||
|
||||
export type LocalSttModelId = ModelIdByKind<"stt-online"> | ModelIdByKind<"stt-offline">;
|
||||
export type LocalSttModelId = ModelIdByKind<"stt-offline">;
|
||||
export type LocalTtsModelId = ModelIdByKind<"tts">;
|
||||
|
||||
const ALL_MODEL_IDS: SherpaOnnxModelId[] = Object.keys(SHERPA_ONNX_MODEL_CATALOG).filter(
|
||||
@@ -166,48 +76,22 @@ function resolveDefaultModelId(role: DefaultModelRole): SherpaOnnxModelId {
|
||||
export const DEFAULT_LOCAL_STT_MODEL = resolveDefaultModelId("stt");
|
||||
export const DEFAULT_LOCAL_TTS_MODEL = resolveDefaultModelId("tts");
|
||||
|
||||
function buildAliasMap<T extends SherpaOnnxModelId>(modelIds: readonly T[]): Record<string, T> {
|
||||
const aliasMap: Record<string, T> = {};
|
||||
for (const modelId of modelIds) {
|
||||
const aliases = SHERPA_ONNX_MODEL_CATALOG[modelId].aliases ?? [];
|
||||
for (const alias of aliases) {
|
||||
aliasMap[alias.trim().toLowerCase()] = modelId;
|
||||
}
|
||||
}
|
||||
return aliasMap;
|
||||
}
|
||||
|
||||
function createAliasedModelIdSchema<T extends string>(params: {
|
||||
modelIds: readonly T[];
|
||||
aliases: Record<string, T>;
|
||||
}): z.ZodType<T, z.ZodTypeDef, string> {
|
||||
const validIds = new Set<string>(params.modelIds);
|
||||
function createModelIdSchema<T extends string>(
|
||||
modelIds: readonly T[],
|
||||
): z.ZodType<T, z.ZodTypeDef, string> {
|
||||
const validIds = new Set<string>(modelIds);
|
||||
return z
|
||||
.string()
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.refine(
|
||||
(value): value is T =>
|
||||
validIds.has(value) || Object.prototype.hasOwnProperty.call(params.aliases, value),
|
||||
{
|
||||
message: "Invalid model id",
|
||||
},
|
||||
)
|
||||
.transform((value) => params.aliases[value] ?? value);
|
||||
.refine((value) => validIds.has(value), {
|
||||
message: "Invalid model id",
|
||||
})
|
||||
.transform((value) => value as T);
|
||||
}
|
||||
|
||||
const STT_MODEL_ALIASES = buildAliasMap(LOCAL_STT_MODEL_IDS);
|
||||
const TTS_MODEL_ALIASES = buildAliasMap(LOCAL_TTS_MODEL_IDS);
|
||||
|
||||
export const LocalSttModelIdSchema = createAliasedModelIdSchema({
|
||||
modelIds: LOCAL_STT_MODEL_IDS,
|
||||
aliases: STT_MODEL_ALIASES,
|
||||
});
|
||||
|
||||
export const LocalTtsModelIdSchema = createAliasedModelIdSchema({
|
||||
modelIds: LOCAL_TTS_MODEL_IDS,
|
||||
aliases: TTS_MODEL_ALIASES,
|
||||
});
|
||||
export const LocalSttModelIdSchema = createModelIdSchema(LOCAL_STT_MODEL_IDS);
|
||||
export const LocalTtsModelIdSchema = createModelIdSchema(LOCAL_TTS_MODEL_IDS);
|
||||
|
||||
export type SherpaOnnxModelSpec = SherpaOnnxCatalogEntry & {
|
||||
id: SherpaOnnxModelId;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { describe, expect, test, vi } from "vitest";
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
@@ -15,69 +15,27 @@ const logger = pino({ level: "silent" });
|
||||
describe("sherpa model downloader", () => {
|
||||
test("getSherpaOnnxModelDir maps modelId to extractedDir", () => {
|
||||
const modelsDir = "/tmp/models";
|
||||
expect(getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v3-int8")).toContain(
|
||||
"sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8",
|
||||
);
|
||||
expect(getSherpaOnnxModelDir(modelsDir, "pocket-tts-onnx-int8")).toContain(
|
||||
"pocket-tts-onnx-int8",
|
||||
expect(getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v2-int8")).toContain(
|
||||
"sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8",
|
||||
);
|
||||
expect(getSherpaOnnxModelDir(modelsDir, "kokoro-en-v0_19")).toContain("kokoro-en-v0_19");
|
||||
});
|
||||
|
||||
test("ensureSherpaOnnxModel succeeds without downloading when files exist", async () => {
|
||||
const modelsDir = makeTmpDir();
|
||||
const modelDir = getSherpaOnnxModelDir(modelsDir, "kitten-nano-en-v0_1-fp16");
|
||||
const modelDir = getSherpaOnnxModelDir(modelsDir, "kokoro-en-v0_19");
|
||||
|
||||
mkdirSync(path.join(modelDir, "espeak-ng-data"), { recursive: true });
|
||||
writeFileSync(path.join(modelDir, "model.fp16.onnx"), "x");
|
||||
writeFileSync(path.join(modelDir, "model.onnx"), "x");
|
||||
writeFileSync(path.join(modelDir, "voices.bin"), "x");
|
||||
writeFileSync(path.join(modelDir, "tokens.txt"), "x");
|
||||
|
||||
const out = await ensureSherpaOnnxModel({
|
||||
modelsDir,
|
||||
modelId: "kitten-nano-en-v0_1-fp16",
|
||||
modelId: "kokoro-en-v0_19",
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(out).toBe(modelDir);
|
||||
});
|
||||
|
||||
test("ensureSherpaOnnxModel logs lifecycle events without progress spam", async () => {
|
||||
const modelsDir = makeTmpDir();
|
||||
const infoMessages: string[] = [];
|
||||
|
||||
const loggerWithSpy = {
|
||||
child: () => loggerWithSpy,
|
||||
info: (_obj?: unknown, msg?: string) => {
|
||||
if (typeof msg === "string") {
|
||||
infoMessages.push(msg);
|
||||
}
|
||||
},
|
||||
error: () => undefined,
|
||||
} as unknown as pino.Logger;
|
||||
|
||||
const originalFetch = globalThis.fetch;
|
||||
const payload = Buffer.alloc(128 * 1024, 7);
|
||||
const fetchMock = vi.fn(async () => {
|
||||
return new Response(payload, {
|
||||
status: 200,
|
||||
headers: { "content-length": String(payload.length) },
|
||||
});
|
||||
});
|
||||
globalThis.fetch = fetchMock as typeof fetch;
|
||||
|
||||
try {
|
||||
await ensureSherpaOnnxModel({
|
||||
modelsDir,
|
||||
modelId: "pocket-tts-onnx-int8",
|
||||
logger: loggerWithSpy,
|
||||
});
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
|
||||
expect(fetchMock).toHaveBeenCalled();
|
||||
expect(infoMessages).toContain("Starting model download");
|
||||
expect(infoMessages).toContain("Model download completed");
|
||||
expect(infoMessages).not.toContain("Downloading model artifact");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -111,92 +111,55 @@ export async function ensureSherpaOnnxModel(
|
||||
logger.info({ modelsDir: options.modelsDir }, "Starting model download");
|
||||
|
||||
try {
|
||||
if (spec.archiveUrl) {
|
||||
const downloadsDir = path.join(options.modelsDir, ".downloads");
|
||||
const archiveFilename = path.basename(new URL(spec.archiveUrl).pathname);
|
||||
const archivePath = path.join(downloadsDir, archiveFilename);
|
||||
const downloadsDir = path.join(options.modelsDir, ".downloads");
|
||||
const archiveFilename = path.basename(new URL(spec.archiveUrl).pathname);
|
||||
const archivePath = path.join(downloadsDir, archiveFilename);
|
||||
|
||||
if (!(await isNonEmptyFile(archivePath))) {
|
||||
await downloadToFile({
|
||||
url: spec.archiveUrl,
|
||||
outputPath: archivePath,
|
||||
});
|
||||
}
|
||||
|
||||
logger.info(
|
||||
{
|
||||
modelId: options.modelId,
|
||||
archivePath,
|
||||
modelDir,
|
||||
},
|
||||
"Extracting model archive",
|
||||
);
|
||||
await extractTarArchive(archivePath, options.modelsDir);
|
||||
|
||||
logger.info(
|
||||
{
|
||||
modelId: options.modelId,
|
||||
modelDir,
|
||||
},
|
||||
"Verifying downloaded model files",
|
||||
);
|
||||
if (!(await hasRequiredFiles(modelDir, spec.requiredFiles))) {
|
||||
throw new Error(
|
||||
`Downloaded and extracted ${archiveFilename}, but required files are still missing in ${modelDir}.`,
|
||||
);
|
||||
}
|
||||
|
||||
logger.info(
|
||||
{
|
||||
modelId: options.modelId,
|
||||
archivePath,
|
||||
},
|
||||
"Finalizing model artifacts",
|
||||
);
|
||||
try {
|
||||
await rm(archivePath, { force: true });
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
|
||||
logger.info({ modelDir }, "Model download completed");
|
||||
return modelDir;
|
||||
if (!(await isNonEmptyFile(archivePath))) {
|
||||
await downloadToFile({
|
||||
url: spec.archiveUrl,
|
||||
outputPath: archivePath,
|
||||
});
|
||||
}
|
||||
|
||||
if (spec.downloadFiles && spec.downloadFiles.length > 0) {
|
||||
await mkdir(modelDir, { recursive: true });
|
||||
logger.info(
|
||||
{
|
||||
modelId: options.modelId,
|
||||
archivePath,
|
||||
modelDir,
|
||||
},
|
||||
"Extracting model archive",
|
||||
);
|
||||
await extractTarArchive(archivePath, options.modelsDir);
|
||||
|
||||
await Promise.all(
|
||||
spec.downloadFiles.map(async (file) => {
|
||||
const dst = path.join(modelDir, file.relPath);
|
||||
if (await isNonEmptyFile(dst)) {
|
||||
return;
|
||||
}
|
||||
await downloadToFile({
|
||||
url: file.url,
|
||||
outputPath: dst,
|
||||
});
|
||||
}),
|
||||
logger.info(
|
||||
{
|
||||
modelId: options.modelId,
|
||||
modelDir,
|
||||
},
|
||||
"Verifying downloaded model files",
|
||||
);
|
||||
if (!(await hasRequiredFiles(modelDir, spec.requiredFiles))) {
|
||||
throw new Error(
|
||||
`Downloaded and extracted ${archiveFilename}, but required files are still missing in ${modelDir}.`,
|
||||
);
|
||||
|
||||
logger.info(
|
||||
{
|
||||
modelId: options.modelId,
|
||||
modelDir,
|
||||
},
|
||||
"Verifying downloaded model files",
|
||||
);
|
||||
if (!(await hasRequiredFiles(modelDir, spec.requiredFiles))) {
|
||||
throw new Error(
|
||||
`Downloaded files for ${options.modelId}, but required files are still missing in ${modelDir}.`,
|
||||
);
|
||||
}
|
||||
|
||||
logger.info({ modelDir }, "Model download completed");
|
||||
return modelDir;
|
||||
}
|
||||
|
||||
throw new Error(`Model spec for ${options.modelId} has no archiveUrl or downloadFiles`);
|
||||
logger.info(
|
||||
{
|
||||
modelId: options.modelId,
|
||||
archivePath,
|
||||
},
|
||||
"Finalizing model artifacts",
|
||||
);
|
||||
try {
|
||||
await rm(archivePath, { force: true });
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
|
||||
logger.info({ modelDir }, "Model download completed");
|
||||
return modelDir;
|
||||
} catch (error) {
|
||||
logger.error({ err: error }, "Model download failed");
|
||||
throw error;
|
||||
|
||||
@@ -1,157 +0,0 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import type pino from "pino";
|
||||
|
||||
import { loadSherpaOnnx } from "./sherpa-onnx-loader.js";
|
||||
|
||||
export type SherpaOnlineRecognizerModel =
|
||||
| {
|
||||
kind: "transducer";
|
||||
encoder: string;
|
||||
decoder: string;
|
||||
joiner: string;
|
||||
tokens: string;
|
||||
modelType?: string;
|
||||
}
|
||||
| {
|
||||
kind: "paraformer";
|
||||
encoder: string;
|
||||
decoder: string;
|
||||
tokens: string;
|
||||
};
|
||||
|
||||
export interface SherpaOnlineRecognizerConfig {
|
||||
model: SherpaOnlineRecognizerModel;
|
||||
numThreads?: number;
|
||||
provider?: "cpu";
|
||||
debug?: 0 | 1;
|
||||
sampleRate?: number;
|
||||
featureDim?: number;
|
||||
decodingMethod?: "greedy_search";
|
||||
maxActivePaths?: number;
|
||||
enableEndpoint?: 0 | 1;
|
||||
rule1MinTrailingSilence?: number;
|
||||
rule2MinTrailingSilence?: number;
|
||||
rule3MinUtteranceLength?: number;
|
||||
}
|
||||
|
||||
function assertFileExists(filePath: string, label: string): void {
|
||||
if (!existsSync(filePath)) {
|
||||
throw new Error(`Missing ${label}: ${filePath}`);
|
||||
}
|
||||
}
|
||||
|
||||
function validateModelFiles(model: SherpaOnlineRecognizerModel): void {
|
||||
if (model.kind === "transducer") {
|
||||
assertFileExists(model.encoder, "transducer encoder");
|
||||
assertFileExists(model.decoder, "transducer decoder");
|
||||
assertFileExists(model.joiner, "transducer joiner");
|
||||
assertFileExists(model.tokens, "tokens");
|
||||
} else {
|
||||
assertFileExists(model.encoder, "paraformer encoder");
|
||||
assertFileExists(model.decoder, "paraformer decoder");
|
||||
assertFileExists(model.tokens, "tokens");
|
||||
}
|
||||
}
|
||||
|
||||
function buildModelConfig(model: SherpaOnlineRecognizerModel): object {
|
||||
if (model.kind === "transducer") {
|
||||
return {
|
||||
transducer: {
|
||||
encoder: model.encoder,
|
||||
decoder: model.decoder,
|
||||
joiner: model.joiner,
|
||||
},
|
||||
tokens: model.tokens,
|
||||
modelType: model.modelType ?? "zipformer",
|
||||
};
|
||||
}
|
||||
return {
|
||||
paraformer: {
|
||||
encoder: model.encoder,
|
||||
decoder: model.decoder,
|
||||
},
|
||||
tokens: model.tokens,
|
||||
};
|
||||
}
|
||||
|
||||
export interface SherpaOnlineStreamNative {
|
||||
acceptWaveform: (sampleRate: number, samples: Float32Array) => void;
|
||||
free?: () => void;
|
||||
}
|
||||
|
||||
export interface SherpaOnlineRecognizerNative {
|
||||
config?: { featConfig?: { sampleRate?: number } };
|
||||
createStream: () => SherpaOnlineStreamNative;
|
||||
isReady: (stream: SherpaOnlineStreamNative) => boolean;
|
||||
decode: (stream: SherpaOnlineStreamNative) => void;
|
||||
getResult: (stream: SherpaOnlineStreamNative) => { text?: string } | string | undefined;
|
||||
reset?: (stream: SherpaOnlineStreamNative) => void;
|
||||
free?: () => void;
|
||||
}
|
||||
|
||||
export class SherpaOnlineRecognizerEngine {
|
||||
public readonly recognizer: SherpaOnlineRecognizerNative;
|
||||
public readonly sampleRate: number;
|
||||
private readonly logger: pino.Logger;
|
||||
|
||||
constructor(config: SherpaOnlineRecognizerConfig, logger: pino.Logger) {
|
||||
this.logger = logger.child({
|
||||
module: "speech",
|
||||
provider: "local",
|
||||
component: "online-recognizer",
|
||||
});
|
||||
|
||||
const { model } = config;
|
||||
validateModelFiles(model);
|
||||
|
||||
const sherpa = loadSherpaOnnx();
|
||||
const modelConfig = buildModelConfig(model);
|
||||
|
||||
const featConfig = {
|
||||
sampleRate: config.sampleRate ?? 16000,
|
||||
featureDim: config.featureDim ?? 80,
|
||||
};
|
||||
|
||||
const recognizerConfig = {
|
||||
featConfig,
|
||||
modelConfig: {
|
||||
...modelConfig,
|
||||
// NOTE: In the WASM-backed `sherpa-onnx` npm package, online recognizers
|
||||
// error when `numThreads > 1`. Keep the default conservative.
|
||||
numThreads: config.numThreads ?? 1,
|
||||
provider: config.provider ?? "cpu",
|
||||
debug: config.debug ?? 0,
|
||||
},
|
||||
decodingMethod: config.decodingMethod ?? "greedy_search",
|
||||
maxActivePaths: config.maxActivePaths ?? 4,
|
||||
enableEndpoint: config.enableEndpoint ?? 0,
|
||||
rule1MinTrailingSilence: config.rule1MinTrailingSilence ?? 2.4,
|
||||
rule2MinTrailingSilence: config.rule2MinTrailingSilence ?? 1.2,
|
||||
rule3MinUtteranceLength: config.rule3MinUtteranceLength ?? 20,
|
||||
};
|
||||
|
||||
this.recognizer = sherpa.createOnlineRecognizer(
|
||||
recognizerConfig,
|
||||
) as SherpaOnlineRecognizerNative;
|
||||
const sr = this.recognizer?.config?.featConfig?.sampleRate;
|
||||
this.sampleRate =
|
||||
typeof sr === "number" && Number.isFinite(sr) && sr > 0 ? sr : featConfig.sampleRate;
|
||||
|
||||
this.logger.info(
|
||||
{ sampleRate: this.sampleRate, modelKind: model.kind, numThreads: config.numThreads ?? 2 },
|
||||
"Sherpa online recognizer initialized",
|
||||
);
|
||||
}
|
||||
|
||||
createStream(): SherpaOnlineStreamNative {
|
||||
return this.recognizer.createStream();
|
||||
}
|
||||
|
||||
free(): void {
|
||||
try {
|
||||
this.recognizer?.free?.();
|
||||
} catch (err) {
|
||||
this.logger.warn({ err }, "Failed to free sherpa recognizer");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
import { createRequire } from "node:module";
|
||||
|
||||
export interface SherpaOnnxModule {
|
||||
createOnlineRecognizer: (config: unknown) => unknown;
|
||||
createOfflineRecognizer: (config: unknown) => unknown;
|
||||
createOfflineTts: (config: unknown) => unknown;
|
||||
}
|
||||
|
||||
let cached: SherpaOnnxModule | null = null;
|
||||
|
||||
export function loadSherpaOnnx(): SherpaOnnxModule {
|
||||
if (cached) {
|
||||
return cached;
|
||||
}
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
cached = require("sherpa-onnx") as SherpaOnnxModule;
|
||||
return cached;
|
||||
}
|
||||
@@ -1,146 +0,0 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
|
||||
import type { StreamingTranscriptionSession } from "../../../speech-provider.js";
|
||||
import { pcm16lePeakAbs, pcm16leToFloat32 } from "../../../audio.js";
|
||||
import {
|
||||
SherpaOnlineRecognizerEngine,
|
||||
type SherpaOnlineStreamNative,
|
||||
} from "./sherpa-online-recognizer.js";
|
||||
|
||||
export class SherpaRealtimeTranscriptionSession
|
||||
extends EventEmitter
|
||||
implements StreamingTranscriptionSession
|
||||
{
|
||||
private readonly engine: SherpaOnlineRecognizerEngine;
|
||||
private stream: SherpaOnlineStreamNative | null = null;
|
||||
private connected = false;
|
||||
|
||||
public readonly requiredSampleRate: number;
|
||||
private currentSegmentId: string | null = null;
|
||||
private previousSegmentId: string | null = null;
|
||||
private lastPartialText = "";
|
||||
private readonly tailPaddingMs: number;
|
||||
|
||||
constructor(params: { engine: SherpaOnlineRecognizerEngine; tailPaddingMs?: number }) {
|
||||
super();
|
||||
this.engine = params.engine;
|
||||
this.requiredSampleRate = this.engine.sampleRate;
|
||||
this.tailPaddingMs = params.tailPaddingMs ?? 500;
|
||||
}
|
||||
|
||||
async connect(): Promise<void> {
|
||||
if (this.connected) {
|
||||
return;
|
||||
}
|
||||
this.stream = this.engine.createStream();
|
||||
this.currentSegmentId = uuidv4();
|
||||
this.connected = true;
|
||||
}
|
||||
|
||||
appendPcm16(pcm16le: Buffer): void {
|
||||
if (!this.connected || !this.stream || !this.currentSegmentId) {
|
||||
this.emit("error", new Error("Sherpa realtime session not connected"));
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const peak = pcm16lePeakAbs(pcm16le);
|
||||
const peakFloat = peak / 32768.0;
|
||||
const targetPeak = 0.6;
|
||||
const maxGain = 50;
|
||||
const gain =
|
||||
peakFloat > 0 && peakFloat < targetPeak ? Math.min(maxGain, targetPeak / peakFloat) : 1;
|
||||
const floatSamples = pcm16leToFloat32(pcm16le, gain);
|
||||
this.stream.acceptWaveform(this.engine.sampleRate, floatSamples);
|
||||
|
||||
while (this.engine.recognizer.isReady(this.stream)) {
|
||||
this.engine.recognizer.decode(this.stream);
|
||||
}
|
||||
|
||||
const rawResult = this.engine.recognizer.getResult(this.stream);
|
||||
const text = (
|
||||
(typeof rawResult === "object" && rawResult && "text" in rawResult
|
||||
? rawResult.text
|
||||
: undefined) ?? ""
|
||||
).trim();
|
||||
if (text !== this.lastPartialText) {
|
||||
this.lastPartialText = text;
|
||||
this.emit("transcript", {
|
||||
segmentId: this.currentSegmentId,
|
||||
transcript: text,
|
||||
isFinal: false,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
this.emit("error", err instanceof Error ? err : new Error(String(err)));
|
||||
}
|
||||
}
|
||||
|
||||
commit(): void {
|
||||
if (!this.connected || !this.stream || !this.currentSegmentId) {
|
||||
this.emit("error", new Error("Sherpa realtime session not connected"));
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const padSamples = Math.max(
|
||||
0,
|
||||
Math.round((this.engine.sampleRate * this.tailPaddingMs) / 1000),
|
||||
);
|
||||
if (padSamples > 0) {
|
||||
this.stream.acceptWaveform(this.engine.sampleRate, new Float32Array(padSamples));
|
||||
}
|
||||
|
||||
while (this.engine.recognizer.isReady(this.stream)) {
|
||||
this.engine.recognizer.decode(this.stream);
|
||||
}
|
||||
|
||||
const rawFinal = this.engine.recognizer.getResult(this.stream);
|
||||
const finalText = (
|
||||
(typeof rawFinal === "object" && rawFinal && "text" in rawFinal
|
||||
? rawFinal.text
|
||||
: undefined) ?? ""
|
||||
).trim();
|
||||
const segmentId = this.currentSegmentId;
|
||||
const previousSegmentId = this.previousSegmentId;
|
||||
|
||||
this.emit("committed", { segmentId, previousSegmentId });
|
||||
this.emit("transcript", { segmentId, transcript: finalText, isFinal: true });
|
||||
|
||||
this.previousSegmentId = segmentId;
|
||||
this.currentSegmentId = uuidv4();
|
||||
this.lastPartialText = "";
|
||||
this.engine.recognizer.reset?.(this.stream);
|
||||
} catch (err) {
|
||||
this.emit("error", err instanceof Error ? err : new Error(String(err)));
|
||||
}
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
if (!this.connected || !this.stream) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
this.engine.recognizer.reset?.(this.stream);
|
||||
this.currentSegmentId = uuidv4();
|
||||
this.lastPartialText = "";
|
||||
} catch (err) {
|
||||
this.emit("error", err instanceof Error ? err : new Error(String(err)));
|
||||
}
|
||||
}
|
||||
|
||||
close(): void {
|
||||
if (!this.stream) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
this.stream.free?.();
|
||||
} catch {
|
||||
// ignore
|
||||
} finally {
|
||||
this.stream = null;
|
||||
this.connected = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,184 +0,0 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import type pino from "pino";
|
||||
|
||||
import type {
|
||||
SpeechToTextProvider,
|
||||
StreamingTranscriptionSession,
|
||||
TranscriptionResult,
|
||||
} from "../../../speech-provider.js";
|
||||
import { Pcm16MonoResampler } from "../../../../agent/pcm16-resampler.js";
|
||||
import {
|
||||
parsePcm16MonoWav,
|
||||
parsePcmRateFromFormat,
|
||||
pcm16lePeakAbs,
|
||||
pcm16leToFloat32,
|
||||
} from "../../../audio.js";
|
||||
import { SherpaOnlineRecognizerEngine } from "./sherpa-online-recognizer.js";
|
||||
|
||||
export interface SherpaSttConfig {
|
||||
engine: SherpaOnlineRecognizerEngine;
|
||||
silencePeakThreshold?: number;
|
||||
tailPaddingMs?: number;
|
||||
}
|
||||
|
||||
export class SherpaOnnxSTT implements SpeechToTextProvider {
|
||||
private readonly engine: SherpaOnlineRecognizerEngine;
|
||||
private readonly silencePeakThreshold: number;
|
||||
private readonly tailPaddingMs: number;
|
||||
private readonly logger: pino.Logger;
|
||||
public readonly id = "local" as const;
|
||||
|
||||
constructor(config: SherpaSttConfig, logger: pino.Logger) {
|
||||
this.engine = config.engine;
|
||||
this.silencePeakThreshold = config.silencePeakThreshold ?? 300;
|
||||
this.tailPaddingMs = config.tailPaddingMs ?? 500;
|
||||
this.logger = logger.child({ module: "speech", provider: "local", component: "stt" });
|
||||
}
|
||||
|
||||
public createSession(params: {
|
||||
logger: pino.Logger;
|
||||
language?: string;
|
||||
prompt?: string;
|
||||
}): StreamingTranscriptionSession {
|
||||
const emitter = new EventEmitter();
|
||||
void params;
|
||||
const requiredSampleRate = this.engine.sampleRate;
|
||||
let connected = false;
|
||||
let segmentId = uuidv4();
|
||||
let previousSegmentId: string | null = null;
|
||||
let pcm16: Buffer = Buffer.alloc(0);
|
||||
|
||||
return {
|
||||
requiredSampleRate,
|
||||
async connect() {
|
||||
connected = true;
|
||||
},
|
||||
appendPcm16(chunk: Buffer) {
|
||||
if (!connected) {
|
||||
emitter.emit("error", new Error("STT session not connected"));
|
||||
return;
|
||||
}
|
||||
pcm16 = pcm16.length === 0 ? chunk : Buffer.concat([pcm16, chunk]);
|
||||
},
|
||||
commit: () => {
|
||||
if (!connected) {
|
||||
emitter.emit("error", new Error("STT session not connected"));
|
||||
return;
|
||||
}
|
||||
|
||||
const committedId = segmentId;
|
||||
const prev = previousSegmentId;
|
||||
emitter.emit("committed", { segmentId: committedId, previousSegmentId: prev });
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
const rt = await this.transcribeAudio(pcm16, `audio/pcm;rate=${requiredSampleRate}`);
|
||||
emitter.emit("transcript", {
|
||||
segmentId: committedId,
|
||||
transcript: rt.text,
|
||||
isFinal: true,
|
||||
language: rt.language,
|
||||
logprobs: rt.logprobs,
|
||||
avgLogprob: rt.avgLogprob,
|
||||
isLowConfidence: rt.isLowConfidence,
|
||||
});
|
||||
} catch (err) {
|
||||
emitter.emit("error", err);
|
||||
} finally {
|
||||
previousSegmentId = committedId;
|
||||
segmentId = uuidv4();
|
||||
pcm16 = Buffer.alloc(0);
|
||||
}
|
||||
})();
|
||||
},
|
||||
clear() {
|
||||
pcm16 = Buffer.alloc(0);
|
||||
segmentId = uuidv4();
|
||||
},
|
||||
close() {
|
||||
connected = false;
|
||||
pcm16 = Buffer.alloc(0);
|
||||
},
|
||||
on(event: "committed" | "transcript" | "error", handler: (payload: never) => void) {
|
||||
emitter.on(event, handler as (...args: unknown[]) => void);
|
||||
return undefined;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
public async transcribeAudio(audioBuffer: Buffer, format: string): Promise<TranscriptionResult> {
|
||||
const start = Date.now();
|
||||
|
||||
let inputRate: number;
|
||||
let pcm16: Buffer;
|
||||
|
||||
if (format.toLowerCase().includes("audio/wav")) {
|
||||
const parsed = parsePcm16MonoWav(audioBuffer);
|
||||
inputRate = parsed.sampleRate;
|
||||
pcm16 = parsed.pcm16;
|
||||
} else if (format.toLowerCase().includes("audio/pcm")) {
|
||||
inputRate = parsePcmRateFromFormat(format, this.engine.sampleRate) ?? this.engine.sampleRate;
|
||||
pcm16 = audioBuffer;
|
||||
} else {
|
||||
throw new Error(`Unsupported audio format for sherpa STT: ${format}`);
|
||||
}
|
||||
|
||||
const peak = pcm16lePeakAbs(pcm16);
|
||||
if (peak < this.silencePeakThreshold) {
|
||||
return { text: "", duration: Date.now() - start, isLowConfidence: true };
|
||||
}
|
||||
|
||||
let pcmForModel = pcm16;
|
||||
if (inputRate !== this.engine.sampleRate) {
|
||||
const resampler = new Pcm16MonoResampler({ inputRate, outputRate: this.engine.sampleRate });
|
||||
pcmForModel = resampler.processChunk(pcm16);
|
||||
inputRate = this.engine.sampleRate;
|
||||
}
|
||||
|
||||
const peakForModel = pcm16lePeakAbs(pcmForModel);
|
||||
const peakFloat = peakForModel / 32768.0;
|
||||
const targetPeak = 0.6;
|
||||
const maxGain = 50;
|
||||
const gain =
|
||||
peakFloat > 0 && peakFloat < targetPeak ? Math.min(maxGain, targetPeak / peakFloat) : 1;
|
||||
|
||||
const stream = this.engine.createStream();
|
||||
try {
|
||||
const floatSamples = pcm16leToFloat32(pcmForModel, gain);
|
||||
stream.acceptWaveform(inputRate, floatSamples);
|
||||
|
||||
while (this.engine.recognizer.isReady(stream)) {
|
||||
this.engine.recognizer.decode(stream);
|
||||
}
|
||||
|
||||
const padSamples = Math.max(
|
||||
0,
|
||||
Math.round((this.engine.sampleRate * this.tailPaddingMs) / 1000),
|
||||
);
|
||||
if (padSamples > 0) {
|
||||
stream.acceptWaveform(this.engine.sampleRate, new Float32Array(padSamples));
|
||||
}
|
||||
|
||||
while (this.engine.recognizer.isReady(stream)) {
|
||||
this.engine.recognizer.decode(stream);
|
||||
}
|
||||
|
||||
const rawResult = this.engine.recognizer.getResult(stream);
|
||||
const text = (
|
||||
(typeof rawResult === "object" && rawResult && "text" in rawResult
|
||||
? rawResult.text
|
||||
: undefined) ?? ""
|
||||
).trim();
|
||||
const duration = Date.now() - start;
|
||||
this.logger.debug({ duration, textLength: text.length }, "Sherpa transcription complete");
|
||||
return { text, duration, ...(text.length === 0 ? { isLowConfidence: true } : {}) };
|
||||
} finally {
|
||||
try {
|
||||
stream.free?.();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,7 @@ import type { SpeechStreamResult, TextToSpeechProvider } from "../../../speech-p
|
||||
import { chunkBuffer, float32ToPcm16le } from "../../../audio.js";
|
||||
import { loadSherpaOnnxNode } from "./sherpa-onnx-node-loader.js";
|
||||
|
||||
export type SherpaTtsPreset = "kokoro-en-v0_19" | "kitten-nano-en-v0_1-fp16";
|
||||
export type SherpaTtsPreset = "kokoro-en-v0_19";
|
||||
|
||||
export interface SherpaTtsConfig {
|
||||
preset: SherpaTtsPreset;
|
||||
@@ -41,9 +41,6 @@ export class SherpaOnnxTTS implements TextToSpeechProvider {
|
||||
private readonly logger: pino.Logger;
|
||||
|
||||
constructor(config: SherpaTtsConfig, logger: pino.Logger) {
|
||||
if (config.preset !== "kokoro-en-v0_19" && config.preset !== "kitten-nano-en-v0_1-fp16") {
|
||||
throw new Error(`Unsupported Sherpa TTS preset: ${config.preset}`);
|
||||
}
|
||||
this.logger = logger.child({ module: "speech", provider: "local", component: "tts" });
|
||||
this.speakerId = config.speakerId ?? 0;
|
||||
this.speed = config.speed ?? 1.0;
|
||||
@@ -53,8 +50,7 @@ export class SherpaOnnxTTS implements TextToSpeechProvider {
|
||||
throw new Error("sherpa-onnx-node OfflineTts is unavailable");
|
||||
}
|
||||
|
||||
const modelFile = config.preset === "kokoro-en-v0_19" ? "model.onnx" : "model.fp16.onnx";
|
||||
const modelPath = `${config.modelDir}/${modelFile}`;
|
||||
const modelPath = `${config.modelDir}/model.onnx`;
|
||||
const voicesPath = `${config.modelDir}/voices.bin`;
|
||||
const tokensPath = `${config.modelDir}/tokens.txt`;
|
||||
const dataDir = `${config.modelDir}/espeak-ng-data`;
|
||||
@@ -64,26 +60,15 @@ export class SherpaOnnxTTS implements TextToSpeechProvider {
|
||||
assertFileExists(tokensPath, "TTS tokens");
|
||||
assertFileExists(dataDir, "TTS espeak-ng dataDir");
|
||||
|
||||
const modelConfig =
|
||||
config.preset === "kokoro-en-v0_19"
|
||||
? {
|
||||
kokoro: {
|
||||
model: modelPath,
|
||||
voices: voicesPath,
|
||||
tokens: tokensPath,
|
||||
dataDir,
|
||||
lengthScale: config.lengthScale ?? 1.0,
|
||||
},
|
||||
}
|
||||
: {
|
||||
kitten: {
|
||||
model: modelPath,
|
||||
voices: voicesPath,
|
||||
tokens: tokensPath,
|
||||
dataDir,
|
||||
lengthScale: config.lengthScale ?? 1.0,
|
||||
},
|
||||
};
|
||||
const modelConfig = {
|
||||
kokoro: {
|
||||
model: modelPath,
|
||||
voices: voicesPath,
|
||||
tokens: tokensPath,
|
||||
dataDir,
|
||||
lengthScale: config.lengthScale ?? 1.0,
|
||||
},
|
||||
};
|
||||
|
||||
const offlineTtsConfig = {
|
||||
model: modelConfig,
|
||||
|
||||
@@ -5,28 +5,15 @@ import path from "node:path";
|
||||
import pino from "pino";
|
||||
|
||||
import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./model-downloader.js";
|
||||
import type { SherpaOnnxModelId } from "./model-catalog.js";
|
||||
import { createDaemonTestContext } from "../../../../test-utils/index.js";
|
||||
import { parsePcm16MonoWav, wordSimilarity } from "../../../../test-utils/dictation-e2e.js";
|
||||
import { SherpaOnnxTTS } from "./sherpa-tts.js";
|
||||
import { PocketTtsOnnxTTS } from "../pocket/pocket-tts-onnx.js";
|
||||
import { SherpaOnlineRecognizerEngine } from "./sherpa-online-recognizer.js";
|
||||
import { SherpaOnnxSTT } from "./sherpa-stt.js";
|
||||
import { SherpaOfflineRecognizerEngine } from "./sherpa-offline-recognizer.js";
|
||||
import { SherpaOnnxParakeetSTT } from "./sherpa-parakeet-stt.js";
|
||||
|
||||
const RUN = process.env.PASEO_SPEECH_E2E_DOWNLOAD === "1";
|
||||
const downloadTest = RUN ? test : test.skip;
|
||||
|
||||
type ModelSet = "zipformer-kitten" | "parakeet-pocket";
|
||||
|
||||
function getModelSet(): ModelSet {
|
||||
const raw = (process.env.PASEO_SPEECH_E2E_MODEL_SET ?? "parakeet-pocket").trim().toLowerCase();
|
||||
if (raw === "zipformer-kitten" || raw === "zipformer") return "zipformer-kitten";
|
||||
if (raw === "parakeet-pocket" || raw === "parakeet") return "parakeet-pocket";
|
||||
throw new Error(`Unknown PASEO_SPEECH_E2E_MODEL_SET: ${raw}`);
|
||||
}
|
||||
|
||||
async function readFixtureWav(): Promise<Buffer> {
|
||||
const fixturePath = path.resolve(process.cwd(), "..", "app", "e2e", "fixtures", "recording.wav");
|
||||
return import("node:fs/promises").then((fs) => fs.readFile(fixturePath));
|
||||
@@ -92,27 +79,20 @@ function toAudioPcmFormat(format: string): string {
|
||||
const rest = trimmed.replace(/^pcm;?/i, "");
|
||||
return rest ? `audio/pcm;${rest}` : "audio/pcm";
|
||||
}
|
||||
// Fall back to treating it as a suffix (e.g. "rate=24000")
|
||||
return `audio/pcm;${trimmed}`;
|
||||
}
|
||||
|
||||
downloadTest(
|
||||
"downloads models and runs streaming STT + streaming TTS",
|
||||
"downloads default local speech models and runs STT + TTS",
|
||||
async () => {
|
||||
const logger = pino({ level: "silent" });
|
||||
const set = getModelSet();
|
||||
|
||||
const paseoHomeRoot = mkdtempSync(path.join(tmpdir(), "paseo-speech-download-"));
|
||||
const modelsDir = path.join(paseoHomeRoot, ".paseo", "models", "local-speech");
|
||||
|
||||
const modelIds: SherpaOnnxModelId[] =
|
||||
set === "parakeet-pocket"
|
||||
? ["parakeet-tdt-0.6b-v3-int8", "pocket-tts-onnx-int8"]
|
||||
: ["zipformer-bilingual-zh-en-2023-02-20", "kitten-nano-en-v0_1-fp16"];
|
||||
|
||||
await ensureSherpaOnnxModels({
|
||||
modelsDir,
|
||||
modelIds,
|
||||
modelIds: ["parakeet-tdt-0.6b-v2-int8", "kokoro-en-v0_19"],
|
||||
logger,
|
||||
});
|
||||
|
||||
@@ -128,16 +108,10 @@ downloadTest(
|
||||
local: {
|
||||
modelsDir,
|
||||
models: {
|
||||
dictationStt:
|
||||
set === "parakeet-pocket"
|
||||
? "parakeet-tdt-0.6b-v3-int8"
|
||||
: "zipformer-bilingual-zh-en-2023-02-20",
|
||||
voiceStt:
|
||||
set === "parakeet-pocket"
|
||||
? "parakeet-tdt-0.6b-v3-int8"
|
||||
: "zipformer-bilingual-zh-en-2023-02-20",
|
||||
voiceTts:
|
||||
set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16",
|
||||
dictationStt: "parakeet-tdt-0.6b-v2-int8",
|
||||
voiceStt: "parakeet-tdt-0.6b-v2-int8",
|
||||
voiceTts: "kokoro-en-v0_19",
|
||||
voiceTtsSpeakerId: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -149,7 +123,6 @@ downloadTest(
|
||||
expect(sampleRate).toBe(16000);
|
||||
const format = "audio/pcm;rate=16000;bits=16";
|
||||
|
||||
// Streaming STT: dictation path (verifies websocket streaming + partials + final)
|
||||
const dictationId = `dict-download-${Date.now()}`;
|
||||
let partialCount = 0;
|
||||
const unsubscribe = ctx.client.on("dictation_stream_partial", (message) => {
|
||||
@@ -159,15 +132,14 @@ downloadTest(
|
||||
});
|
||||
|
||||
await ctx.client.startDictationStream(dictationId, format);
|
||||
const chunkBytes = 3200; // 100ms @ 16kHz mono PCM16
|
||||
const chunkBytes = 3200;
|
||||
let seq = 0;
|
||||
for (let offset = 0; offset < pcm16.length; offset += chunkBytes) {
|
||||
const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
|
||||
ctx.client.sendDictationStreamChunk(dictationId, seq, chunk.toString("base64"), format);
|
||||
seq += 1;
|
||||
}
|
||||
const finalSeq = seq - 1;
|
||||
const dictationFinal = await ctx.client.finishDictationStream(dictationId, finalSeq);
|
||||
const dictationFinal = await ctx.client.finishDictationStream(dictationId, seq - 1);
|
||||
unsubscribe();
|
||||
|
||||
expect(dictationFinal.text.toLowerCase()).toContain("voice note");
|
||||
@@ -175,7 +147,6 @@ downloadTest(
|
||||
expect(wordSimilarity(dictationFinal.text, baseline)).toBeGreaterThan(0.45);
|
||||
expect(partialCount).toBeGreaterThan(0);
|
||||
|
||||
// Voice-mode STT: chunked upload until isLast=true
|
||||
const transcriptionPromise = waitForSignal<string>(30000, (resolve, reject) => {
|
||||
const offResult = ctx.client.on("transcription_result", (message) => {
|
||||
if (message.type !== "transcription_result") return;
|
||||
@@ -218,80 +189,39 @@ downloadTest(
|
||||
await ctx.client.setVoiceMode(false);
|
||||
rmSync(voiceCwd, { recursive: true, force: true });
|
||||
|
||||
// Streaming TTS: generate locally from downloaded model and validate chunking.
|
||||
const ttsText = "This is a voice note.";
|
||||
if (set === "parakeet-pocket") {
|
||||
const modelDir = getSherpaOnnxModelDir(modelsDir, "pocket-tts-onnx-int8");
|
||||
const tts = await PocketTtsOnnxTTS.create(
|
||||
{ modelDir, precision: "int8", targetChunkMs: 50 },
|
||||
logger,
|
||||
);
|
||||
const { stream, format: ttsFormat } = await tts.synthesizeSpeech(ttsText);
|
||||
const { chunks, combined } = await readAllChunks(stream);
|
||||
const ttsModelDir = getSherpaOnnxModelDir(modelsDir, "kokoro-en-v0_19");
|
||||
const tts = new SherpaOnnxTTS(
|
||||
{ preset: "kokoro-en-v0_19", modelDir: ttsModelDir, speakerId: 0 },
|
||||
logger,
|
||||
);
|
||||
const { stream, format: ttsFormat } = await tts.synthesizeSpeech(ttsText);
|
||||
const { chunks, combined } = await readAllChunks(stream);
|
||||
tts.free();
|
||||
|
||||
expect(ttsFormat).toMatch(/rate=\d+/);
|
||||
expect(chunks.length).toBeGreaterThan(3);
|
||||
expect(combined.byteLength).toBeGreaterThan(2000);
|
||||
expect(ttsFormat).toMatch(/rate=\d+/);
|
||||
expect(chunks.length).toBeGreaterThan(3);
|
||||
expect(combined.byteLength).toBeGreaterThan(2000);
|
||||
|
||||
// Round trip: TTS -> STT (offline parakeet)
|
||||
const sttModelDir = getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v3-int8");
|
||||
const engine = new SherpaOfflineRecognizerEngine(
|
||||
{
|
||||
model: {
|
||||
kind: "nemo_transducer",
|
||||
encoder: `${sttModelDir}/encoder.int8.onnx`,
|
||||
decoder: `${sttModelDir}/decoder.int8.onnx`,
|
||||
joiner: `${sttModelDir}/joiner.int8.onnx`,
|
||||
tokens: `${sttModelDir}/tokens.txt`,
|
||||
},
|
||||
numThreads: 2,
|
||||
debug: 0,
|
||||
const sttModelDir = getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v2-int8");
|
||||
const engine = new SherpaOfflineRecognizerEngine(
|
||||
{
|
||||
model: {
|
||||
kind: "nemo_transducer",
|
||||
encoder: `${sttModelDir}/encoder.int8.onnx`,
|
||||
decoder: `${sttModelDir}/decoder.int8.onnx`,
|
||||
joiner: `${sttModelDir}/joiner.int8.onnx`,
|
||||
tokens: `${sttModelDir}/tokens.txt`,
|
||||
},
|
||||
logger,
|
||||
);
|
||||
const stt = new SherpaOnnxParakeetSTT({ engine }, logger);
|
||||
const rt = await stt.transcribeAudio(combined, toAudioPcmFormat(ttsFormat));
|
||||
engine.free();
|
||||
expect(wordSimilarity(rt.text, ttsText)).toBeGreaterThan(0.25);
|
||||
} else {
|
||||
const ttsModelDir = path.join(modelsDir, "kitten-nano-en-v0_1-fp16");
|
||||
const tts = new SherpaOnnxTTS(
|
||||
{ preset: "kitten-nano-en-v0_1-fp16", modelDir: ttsModelDir },
|
||||
logger,
|
||||
);
|
||||
const { stream, format: ttsFormat } = await tts.synthesizeSpeech(ttsText);
|
||||
const { chunks, combined } = await readAllChunks(stream);
|
||||
tts.free();
|
||||
|
||||
expect(ttsFormat).toMatch(/rate=\d+/);
|
||||
expect(chunks.length).toBeGreaterThan(3);
|
||||
expect(combined.byteLength).toBeGreaterThan(2000);
|
||||
|
||||
// Round trip: TTS -> STT (online zipformer, offline segment)
|
||||
const sttModelDir = path.join(
|
||||
modelsDir,
|
||||
"sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20",
|
||||
);
|
||||
const engine = new SherpaOnlineRecognizerEngine(
|
||||
{
|
||||
model: {
|
||||
kind: "transducer",
|
||||
encoder: `${sttModelDir}/encoder-epoch-99-avg-1.onnx`,
|
||||
decoder: `${sttModelDir}/decoder-epoch-99-avg-1.onnx`,
|
||||
joiner: `${sttModelDir}/joiner-epoch-99-avg-1.onnx`,
|
||||
tokens: `${sttModelDir}/tokens.txt`,
|
||||
modelType: "zipformer",
|
||||
},
|
||||
numThreads: 1,
|
||||
debug: 0,
|
||||
},
|
||||
logger,
|
||||
);
|
||||
const stt = new SherpaOnnxSTT({ engine }, logger);
|
||||
const rt = await stt.transcribeAudio(combined, toAudioPcmFormat(ttsFormat));
|
||||
engine.free();
|
||||
expect(wordSimilarity(rt.text, ttsText)).toBeGreaterThan(0.25);
|
||||
}
|
||||
numThreads: 2,
|
||||
debug: 0,
|
||||
},
|
||||
logger,
|
||||
);
|
||||
const stt = new SherpaOnnxParakeetSTT({ engine }, logger);
|
||||
const rt = await stt.transcribeAudio(combined, toAudioPcmFormat(ttsFormat));
|
||||
engine.free();
|
||||
expect(wordSimilarity(rt.text, ttsText)).toBeGreaterThan(0.25);
|
||||
} finally {
|
||||
await ctx.cleanup();
|
||||
}
|
||||
|
||||
@@ -57,7 +57,7 @@ describe("resolveSpeechConfig", () => {
|
||||
});
|
||||
});
|
||||
|
||||
test("resolves feature-scoped local model env vars", () => {
|
||||
test("resolves feature-scoped local speech settings", () => {
|
||||
const persisted = PersistedConfigSchema.parse({
|
||||
features: {
|
||||
voiceMode: {
|
||||
@@ -70,9 +70,9 @@ describe("resolveSpeechConfig", () => {
|
||||
},
|
||||
});
|
||||
const env = {
|
||||
PASEO_DICTATION_LOCAL_STT_MODEL: "zipformer",
|
||||
PASEO_VOICE_LOCAL_STT_MODEL: "parakeet",
|
||||
PASEO_VOICE_LOCAL_TTS_MODEL: "kitten",
|
||||
PASEO_DICTATION_LOCAL_STT_MODEL: "parakeet-tdt-0.6b-v2-int8",
|
||||
PASEO_VOICE_LOCAL_STT_MODEL: "parakeet-tdt-0.6b-v2-int8",
|
||||
PASEO_VOICE_LOCAL_TTS_MODEL: "kokoro-en-v0_19",
|
||||
PASEO_VOICE_LOCAL_TTS_SPEAKER_ID: "5",
|
||||
PASEO_VOICE_LOCAL_TTS_SPEED: "1.35",
|
||||
PASEO_DICTATION_LANGUAGE: "es",
|
||||
@@ -93,9 +93,9 @@ describe("resolveSpeechConfig", () => {
|
||||
expect(result.speech.local).toEqual({
|
||||
modelsDir: "/tmp/models",
|
||||
models: {
|
||||
dictationStt: "zipformer-bilingual-zh-en-2023-02-20",
|
||||
voiceStt: "parakeet-tdt-0.6b-v3-int8",
|
||||
voiceTts: "kitten-nano-en-v0_1-fp16",
|
||||
dictationStt: "parakeet-tdt-0.6b-v2-int8",
|
||||
voiceStt: "parakeet-tdt-0.6b-v2-int8",
|
||||
voiceTts: "kokoro-en-v0_19",
|
||||
voiceTtsSpeakerId: 5,
|
||||
voiceTtsSpeed: 1.35,
|
||||
},
|
||||
@@ -120,9 +120,9 @@ describe("resolveSpeechConfig", () => {
|
||||
explicit: true,
|
||||
enabled: true,
|
||||
});
|
||||
expect(result.speech.local?.models.dictationStt).toBe("zipformer-bilingual-zh-en-2023-02-20");
|
||||
expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8");
|
||||
expect(result.speech.local?.models.voiceTts).toBe("kitten-nano-en-v0_1-fp16");
|
||||
expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v2-int8");
|
||||
expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v2-int8");
|
||||
expect(result.speech.local?.models.voiceTts).toBe("kokoro-en-v0_19");
|
||||
expect(result.speech.local?.models.voiceTtsSpeakerId).toBe(5);
|
||||
expect(result.speech.local?.models.voiceTtsSpeed).toBe(1.35);
|
||||
expect(result.speech.sttLanguages).toEqual({
|
||||
@@ -164,25 +164,6 @@ describe("resolveSpeechConfig", () => {
|
||||
});
|
||||
});
|
||||
|
||||
test("ignores deprecated shared local model env vars", () => {
|
||||
const persisted = PersistedConfigSchema.parse({});
|
||||
const env = {
|
||||
PASEO_LOCAL_STT_MODEL: "zipformer-bilingual-zh-en-2023-02-20",
|
||||
PASEO_LOCAL_TTS_MODEL: "kitten-nano-en-v0_1-fp16",
|
||||
} as NodeJS.ProcessEnv;
|
||||
|
||||
const result = resolveSpeechConfig({
|
||||
paseoHome: "/tmp/paseo-home",
|
||||
env,
|
||||
persisted,
|
||||
});
|
||||
|
||||
expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v2-int8");
|
||||
expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v2-int8");
|
||||
expect(result.speech.local?.models.voiceTts).toBe("kokoro-en-v0_19");
|
||||
expect(result.speech.local?.models.voiceTtsSpeakerId).toBe(0);
|
||||
});
|
||||
|
||||
test("respects disabled dictation and voice mode feature flags", () => {
|
||||
const persisted = PersistedConfigSchema.parse({
|
||||
features: {
|
||||
|
||||
@@ -293,8 +293,8 @@ function createReadySpeechReadinessSnapshot(): SpeechReadinessSnapshot {
|
||||
function createDownloadInProgressSpeechReadinessSnapshot(): SpeechReadinessSnapshot {
|
||||
return {
|
||||
generatedAt: "2026-02-14T00:00:00.000Z",
|
||||
requiredLocalModelIds: ["sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20"],
|
||||
missingLocalModelIds: ["sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20"],
|
||||
requiredLocalModelIds: ["parakeet-tdt-0.6b-v2-int8"],
|
||||
missingLocalModelIds: ["parakeet-tdt-0.6b-v2-int8"],
|
||||
download: {
|
||||
inProgress: true,
|
||||
error: null,
|
||||
@@ -320,9 +320,9 @@ function createDownloadInProgressSpeechReadinessSnapshot(): SpeechReadinessSnaps
|
||||
available: false,
|
||||
reasonCode: "model_download_in_progress",
|
||||
message:
|
||||
"Voice features are unavailable while models download in the background (sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20).",
|
||||
"Voice features are unavailable while models download in the background (parakeet-tdt-0.6b-v2-int8).",
|
||||
retryable: true,
|
||||
missingModelIds: ["sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20"],
|
||||
missingModelIds: ["parakeet-tdt-0.6b-v2-int8"],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user