mirror of
https://github.com/getpaseo/paseo.git
synced 2026-07-29 12:01:31 +00:00
Local streaming STT/TTS with Parakeet + Pocket TTS
- Add provider-agnostic streaming speech interfaces - Local STT via Sherpa (Zipformer/Paraformer) + Parakeet v3 (offline) - Local TTS via Pocket TTS ONNX (onnxruntime-node) with chunked streaming - Auto-download models into PASEO_HOME and add Node download script - Add E2E download test that round-trips TTS->STT
This commit is contained in:
Mohamed Boudra
31
README.md
31
README.md
@@ -32,6 +32,37 @@ npm install -g @getpaseo/cli && paseo
|
||||
|
||||
Then open the app and connect to your daemon.
|
||||
|
||||
## Local speech (STT/TTS)
|
||||
|
||||
Paseo can run dictation + voice mode STT/TTS fully locally via `sherpa-onnx`.
|
||||
|
||||
When the daemon starts with a local speech provider selected, it will download any missing model files automatically (unless `PASEO_SHERPA_ONNX_AUTO_DOWNLOAD=0`).
|
||||
|
||||
```bash
|
||||
npm run speech:download --workspace=@getpaseo/server
|
||||
```
|
||||
|
||||
Optional configuration:
|
||||
|
||||
- `PASEO_SHERPA_ONNX_MODELS_DIR` (defaults to `~/.paseo/models/sherpa-onnx`)
|
||||
- `PASEO_SHERPA_ONNX_AUTO_DOWNLOAD` (`1` by default; set `0` to disable automatic downloads on daemon start)
|
||||
- `PASEO_SHERPA_STT_PRESET` (`zipformer`, `paraformer`, or `parakeet` for NVIDIA Parakeet TDT v3)
|
||||
- `PASEO_SHERPA_TTS_PRESET` (`pocket-tts` (Kyutai Pocket TTS), `kitten`, or `kokoro`)
|
||||
- `PASEO_DICTATION_STT_PROVIDER`, `PASEO_VOICE_STT_PROVIDER`, `PASEO_VOICE_TTS_PROVIDER` (`sherpa` or `openai`)
|
||||
|
||||
To see all supported local model IDs:
|
||||
|
||||
```bash
|
||||
npm run speech:models --workspace=@getpaseo/server
|
||||
```
|
||||
|
||||
Optional: run an end-to-end test that downloads real models and exercises streaming STT + streaming TTS:
|
||||
|
||||
```bash
|
||||
PASEO_SPEECH_E2E_DOWNLOAD=1 PASEO_SPEECH_E2E_MODEL_SET=parakeet-pocket \
|
||||
npx vitest run --workspace=@getpaseo/server src/server/speech/sherpa/speech-download.e2e.test.ts
|
||||
```
|
||||
|
||||
## Documentation
|
||||
|
||||
See [paseo.sh/docs](https://paseo.sh/docs) for full documentation.
|
||||
|
||||
269
package-lock.json
generated
269
package-lock.json
generated
@@ -5621,6 +5621,16 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@sctg/sentencepiece-js": {
|
||||
"version": "1.3.3",
|
||||
"resolved": "https://registry.npmjs.org/@sctg/sentencepiece-js/-/sentencepiece-js-1.3.3.tgz",
|
||||
"integrity": "sha512-iPnzR2HGjdQQG2SpTyPH3wnnpgQ2aS14B3I2jHjomkoaMprM3Sn+WOQblHwByetnvn3n14y3IwjwogVUHA2cVA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"app-root-path": "^3.1.0",
|
||||
"buffer": "^6.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@segment/loosely-validate-event": {
|
||||
"version": "2.0.0",
|
||||
"dev": true,
|
||||
@@ -5739,9 +5749,6 @@
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
@@ -6995,6 +7002,15 @@
|
||||
"node": ">=0.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/adm-zip": {
|
||||
"version": "0.5.16",
|
||||
"resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.16.tgz",
|
||||
"integrity": "sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/agent-base": {
|
||||
"version": "6.0.2",
|
||||
"dev": true,
|
||||
@@ -7157,6 +7173,15 @@
|
||||
"url": "https://github.com/sponsors/jonschlinkert"
|
||||
}
|
||||
},
|
||||
"node_modules/app-root-path": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/app-root-path/-/app-root-path-3.1.0.tgz",
|
||||
"integrity": "sha512-biN3PwB2gUtjaYy/isrU3aNWI5w+fAfvHkSvCKeQGxhmYpwKFUxudR3Yya+KqVRHBmEDYh+/lTozYCFbmzX4nA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 6.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/arg": {
|
||||
"version": "5.0.2",
|
||||
"license": "MIT"
|
||||
@@ -7791,6 +7816,13 @@
|
||||
"version": "1.0.0",
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/boolean": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz",
|
||||
"integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==",
|
||||
"deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/bplist-creator": {
|
||||
"version": "0.1.0",
|
||||
"license": "MIT",
|
||||
@@ -8879,6 +8911,12 @@
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/detect-node": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz",
|
||||
"integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/detect-node-es": {
|
||||
"version": "1.1.0",
|
||||
"license": "MIT"
|
||||
@@ -9741,6 +9779,12 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/es6-error": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz",
|
||||
"integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/esbuild": {
|
||||
"version": "0.25.11",
|
||||
"hasInstallScript": true,
|
||||
@@ -11752,6 +11796,62 @@
|
||||
"version": "0.4.1",
|
||||
"license": "BSD-2-Clause"
|
||||
},
|
||||
"node_modules/global-agent": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz",
|
||||
"integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==",
|
||||
"license": "BSD-3-Clause",
|
||||
"dependencies": {
|
||||
"boolean": "^3.0.1",
|
||||
"es6-error": "^4.1.1",
|
||||
"matcher": "^3.0.0",
|
||||
"roarr": "^2.15.3",
|
||||
"semver": "^7.3.2",
|
||||
"serialize-error": "^7.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/global-agent/node_modules/semver": {
|
||||
"version": "7.7.4",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz",
|
||||
"integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==",
|
||||
"license": "ISC",
|
||||
"bin": {
|
||||
"semver": "bin/semver.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/global-agent/node_modules/serialize-error": {
|
||||
"version": "7.0.1",
|
||||
"resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz",
|
||||
"integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"type-fest": "^0.13.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/global-agent/node_modules/type-fest": {
|
||||
"version": "0.13.1",
|
||||
"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz",
|
||||
"integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==",
|
||||
"license": "(MIT OR CC0-1.0)",
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/global-dirs": {
|
||||
"version": "0.1.1",
|
||||
"license": "MIT",
|
||||
@@ -11775,7 +11875,6 @@
|
||||
},
|
||||
"node_modules/globalthis": {
|
||||
"version": "1.0.4",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"define-properties": "^1.2.1",
|
||||
@@ -13075,6 +13174,12 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/json-stringify-safe": {
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
|
||||
"integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==",
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/json5": {
|
||||
"version": "2.2.3",
|
||||
"license": "MIT",
|
||||
@@ -13397,9 +13502,6 @@
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"license": "MPL-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
@@ -13560,6 +13662,18 @@
|
||||
"version": "1.3.0",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/matcher": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz",
|
||||
"integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"escape-string-regexp": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/math-intrinsics": {
|
||||
"version": "1.1.0",
|
||||
"license": "MIT",
|
||||
@@ -14755,6 +14869,29 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/onnxruntime-common": {
|
||||
"version": "1.24.1",
|
||||
"resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.24.1.tgz",
|
||||
"integrity": "sha512-UnV15u4p4XxoIV+jFP4hXPsW93s3QrwLSpi20HUDYHoTfI4z4sjzex3L4XDOxGGZJ/M/catrwAG2go958UQq0w==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/onnxruntime-node": {
|
||||
"version": "1.24.1",
|
||||
"resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.24.1.tgz",
|
||||
"integrity": "sha512-Ex/oUXKdhDoxvlNxBT3oYtW0MH88yYpPlXQeVQUXpcJQmN24usd/8RCoPLN5kCHwDsiZ+nqsnjciyFRl423dQw==",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"os": [
|
||||
"win32",
|
||||
"darwin",
|
||||
"linux"
|
||||
],
|
||||
"dependencies": {
|
||||
"adm-zip": "^0.5.16",
|
||||
"global-agent": "^3.0.0",
|
||||
"onnxruntime-common": "1.24.1"
|
||||
}
|
||||
},
|
||||
"node_modules/open": {
|
||||
"version": "8.4.2",
|
||||
"license": "MIT",
|
||||
@@ -16785,6 +16922,29 @@
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/roarr": {
|
||||
"version": "2.15.4",
|
||||
"resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz",
|
||||
"integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==",
|
||||
"license": "BSD-3-Clause",
|
||||
"dependencies": {
|
||||
"boolean": "^3.0.1",
|
||||
"detect-node": "^2.0.4",
|
||||
"globalthis": "^1.0.1",
|
||||
"json-stringify-safe": "^5.0.1",
|
||||
"semver-compare": "^1.0.0",
|
||||
"sprintf-js": "^1.1.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/roarr/node_modules/sprintf-js": {
|
||||
"version": "1.1.3",
|
||||
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz",
|
||||
"integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==",
|
||||
"license": "BSD-3-Clause"
|
||||
},
|
||||
"node_modules/rollup": {
|
||||
"version": "4.52.5",
|
||||
"license": "MIT",
|
||||
@@ -16992,6 +17152,12 @@
|
||||
"semver": "bin/semver.js"
|
||||
}
|
||||
},
|
||||
"node_modules/semver-compare": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz",
|
||||
"integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/send": {
|
||||
"version": "0.19.1",
|
||||
"license": "MIT",
|
||||
@@ -17312,6 +17478,91 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/sherpa-onnx": {
|
||||
"version": "1.12.23",
|
||||
"resolved": "https://registry.npmjs.org/sherpa-onnx/-/sherpa-onnx-1.12.23.tgz",
|
||||
"integrity": "sha512-QwgDwSIXx4DgbBE7h7VUXEVLO2vf7w3aJX3ng6WoTAcfB2NW7So4Zp4GtGemwgxZ6sfEh02JEVXUidejkX7V0A==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/sherpa-onnx-darwin-arm64": {
|
||||
"version": "1.12.23",
|
||||
"resolved": "https://registry.npmjs.org/sherpa-onnx-darwin-arm64/-/sherpa-onnx-darwin-arm64-1.12.23.tgz",
|
||||
"integrity": "sha512-zbjNUUH/IXhjRyRJ9mpcWVOGIVr31a/qXBPsfOYc7U8cgwcq33Vmj2OzoLYWQF6T+puqCAE4nMxFAxJvdZekhg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
]
|
||||
},
|
||||
"node_modules/sherpa-onnx-darwin-x64": {
|
||||
"version": "1.12.23",
|
||||
"resolved": "https://registry.npmjs.org/sherpa-onnx-darwin-x64/-/sherpa-onnx-darwin-x64-1.12.23.tgz",
|
||||
"integrity": "sha512-hgPhBG85bgULJo5xS5xvh1A9+D6GBPwhZ8QCvWvh7YP1kF9UZTF1r5hWBOzZiR2Ax0gQMbxVvw1t/NpR2N7HWA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
]
|
||||
},
|
||||
"node_modules/sherpa-onnx-linux-x64": {
|
||||
"version": "1.12.23",
|
||||
"resolved": "https://registry.npmjs.org/sherpa-onnx-linux-x64/-/sherpa-onnx-linux-x64-1.12.23.tgz",
|
||||
"integrity": "sha512-pUZIdDvPtyRXQDGo9R9MIBf2AFUzfgcGmutoulsEdH3hpK6JteR7Z/5pfrZIIqe/O99djAjEHK4AlwLHC2jiZw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/sherpa-onnx-node": {
|
||||
"version": "1.12.23",
|
||||
"resolved": "https://registry.npmjs.org/sherpa-onnx-node/-/sherpa-onnx-node-1.12.23.tgz",
|
||||
"integrity": "sha512-09SRixVSjsajxeCV8Hy9R5J4IHPtw7vNgaIcEokdh/LpU7sY+e12z9uHHIMMMgNiInyGEH74wIwjLXms+W7qRA==",
|
||||
"license": "Apache-2.0",
|
||||
"optionalDependencies": {
|
||||
"sherpa-onnx-darwin-arm64": "^1.12.23",
|
||||
"sherpa-onnx-darwin-x64": "^1.12.23",
|
||||
"sherpa-onnx-linux-arm64": "^1.12.23",
|
||||
"sherpa-onnx-linux-x64": "^1.12.23",
|
||||
"sherpa-onnx-win-ia32": "^1.12.23",
|
||||
"sherpa-onnx-win-x64": "^1.12.23"
|
||||
}
|
||||
},
|
||||
"node_modules/sherpa-onnx-win-ia32": {
|
||||
"version": "1.12.23",
|
||||
"resolved": "https://registry.npmjs.org/sherpa-onnx-win-ia32/-/sherpa-onnx-win-ia32-1.12.23.tgz",
|
||||
"integrity": "sha512-MyLsK7r6dd7paglyTgb8UHTXTEFqOzA91u6VDV64Lq8rDGuOFVYioxX7vlwmGe1A9o7VhuOPNaKcRjEPtVDhBQ==",
|
||||
"cpu": [
|
||||
"ia32"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
]
|
||||
},
|
||||
"node_modules/sherpa-onnx-win-x64": {
|
||||
"version": "1.12.23",
|
||||
"resolved": "https://registry.npmjs.org/sherpa-onnx-win-x64/-/sherpa-onnx-win-x64-1.12.23.tgz",
|
||||
"integrity": "sha512-pdHEYMJiYy8+xzH2WkBVS4/hnRwqjY8FaWnjs0NBgQZnPmc/k4M+TAiauTOuFDNK4GPwFQnjwrCGx6jI9AOkOg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
]
|
||||
},
|
||||
"node_modules/side-channel": {
|
||||
"version": "1.1.0",
|
||||
"license": "MIT",
|
||||
@@ -20036,6 +20287,7 @@
|
||||
"@modelcontextprotocol/sdk": "^1.20.1",
|
||||
"@opencode-ai/sdk": "^1.1.12",
|
||||
"@openrouter/ai-sdk-provider": "^1.2.0",
|
||||
"@sctg/sentencepiece-js": "^1.1.0",
|
||||
"@xterm/headless": "^6.0.0",
|
||||
"ai": "^5.0.76",
|
||||
"ajv": "^8.17.1",
|
||||
@@ -20044,11 +20296,14 @@
|
||||
"express-basic-auth": "^1.2.1",
|
||||
"mnemonic-id": "^3.2.7",
|
||||
"node-pty": "^1.0.0",
|
||||
"onnxruntime-node": "^1.23.0",
|
||||
"openai": "^4.20.0",
|
||||
"pino": "^10.2.0",
|
||||
"pino-pretty": "^13.1.3",
|
||||
"playwright": "^1.56.1",
|
||||
"qrcode": "^1.5.4",
|
||||
"sherpa-onnx": "^1.12.23",
|
||||
"sherpa-onnx-node": "^1.12.23",
|
||||
"strip-ansi": "^7.1.2",
|
||||
"tiny-invariant": "^1.3.3",
|
||||
"uuid": "^9.0.1",
|
||||
|
||||
2
packages/app/e2e/fixtures/recording.baseline.txt
Normal file
2
packages/app/e2e/fixtures/recording.baseline.txt
Normal file
@@ -0,0 +1,2 @@
|
||||
This is a voice note.
|
||||
|
||||
@@ -14,32 +14,60 @@ interface QueuedAudio {
|
||||
}
|
||||
|
||||
/**
|
||||
* Resample PCM16 audio from 24kHz to 16kHz
|
||||
* OpenAI returns 24kHz, Speechmatics expects 16kHz
|
||||
* Resample PCM16 audio between sample rates.
|
||||
* Speechmatics expects 16kHz.
|
||||
*/
|
||||
function resamplePcm24kTo16k(pcm24k: Uint8Array): Uint8Array {
|
||||
// PCM16 = 2 bytes per sample
|
||||
const samples24k = pcm24k.length / 2;
|
||||
const samples16k = Math.floor((samples24k * 16000) / 24000);
|
||||
|
||||
const pcm16k = new Uint8Array(samples16k * 2);
|
||||
const ratio = 24000 / 16000; // 1.5
|
||||
|
||||
for (let i = 0; i < samples16k; i++) {
|
||||
const srcIndex = Math.floor(i * ratio) * 2;
|
||||
if (srcIndex + 1 < pcm24k.length) {
|
||||
pcm16k[i * 2] = pcm24k[srcIndex];
|
||||
pcm16k[i * 2 + 1] = pcm24k[srcIndex + 1];
|
||||
}
|
||||
function resamplePcm16(pcm: Uint8Array, fromRate: number, toRate: number): Uint8Array {
|
||||
if (fromRate === toRate) {
|
||||
return pcm;
|
||||
}
|
||||
|
||||
console.log("[AudioPlayer] Resampled PCM:", {
|
||||
input24k: pcm24k.length,
|
||||
output16k: pcm16k.length,
|
||||
durationMs: samples16k / 16,
|
||||
});
|
||||
const inputSamples = Math.floor(pcm.length / 2);
|
||||
const outputSamples = Math.floor((inputSamples * toRate) / fromRate);
|
||||
const out = new Uint8Array(outputSamples * 2);
|
||||
|
||||
return pcm16k;
|
||||
const ratio = fromRate / toRate;
|
||||
|
||||
const readInt16 = (sampleIndex: number): number => {
|
||||
const i = sampleIndex * 2;
|
||||
if (i + 1 >= pcm.length) {
|
||||
return 0;
|
||||
}
|
||||
const lo = pcm[i]!;
|
||||
const hi = pcm[i + 1]!;
|
||||
let value = (hi << 8) | lo;
|
||||
if (value & 0x8000) {
|
||||
value = value - 0x10000;
|
||||
}
|
||||
return value;
|
||||
};
|
||||
|
||||
const writeInt16 = (sampleIndex: number, value: number): void => {
|
||||
const clamped = Math.max(-32768, Math.min(32767, Math.round(value)));
|
||||
const i = sampleIndex * 2;
|
||||
out[i] = clamped & 0xff;
|
||||
out[i + 1] = (clamped >> 8) & 0xff;
|
||||
};
|
||||
|
||||
for (let i = 0; i < outputSamples; i++) {
|
||||
const srcPos = i * ratio;
|
||||
const i0 = Math.floor(srcPos);
|
||||
const frac = srcPos - i0;
|
||||
const s0 = readInt16(i0);
|
||||
const s1 = readInt16(Math.min(inputSamples - 1, i0 + 1));
|
||||
writeInt16(i, s0 + (s1 - s0) * frac);
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
function parsePcmSampleRate(mimeType: string): number | null {
|
||||
const match = /rate=(\d+)/i.exec(mimeType);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
const rate = Number(match[1]);
|
||||
return Number.isFinite(rate) && rate > 0 ? rate : null;
|
||||
}
|
||||
|
||||
export interface AudioPlayerOptions {
|
||||
@@ -204,12 +232,12 @@ export function useAudioPlayer(options?: AudioPlayerOptions) {
|
||||
console.log("[AudioPlayer] Resuming playback engine...");
|
||||
resumePlayback();
|
||||
|
||||
// Get PCM data from blob (server now sends PCM format)
|
||||
// Get PCM data from blob (server sends PCM16)
|
||||
const arrayBuffer = await audioData.arrayBuffer();
|
||||
let pcm24k = new Uint8Array(arrayBuffer);
|
||||
const pcm = new Uint8Array(arrayBuffer);
|
||||
|
||||
// Resample from 24kHz (OpenAI) to 16kHz (Speechmatics)
|
||||
const pcm16k = resamplePcm24kTo16k(pcm24k);
|
||||
const inputRate = parsePcmSampleRate(audioData.type || "") ?? 24000;
|
||||
const pcm16k = resamplePcm16(pcm, inputRate, 16000);
|
||||
|
||||
// Calculate total duration
|
||||
const samples = pcm16k.length / 2; // 16-bit = 2 bytes per sample
|
||||
|
||||
@@ -14,6 +14,8 @@
|
||||
"start": "NODE_ENV=production node dist/server/index.js",
|
||||
"typecheck": "tsc -p tsconfig.server.typecheck.json --noEmit",
|
||||
"generate:config-schema": "tsx scripts/generate-config-schema.ts",
|
||||
"speech:models": "tsx scripts/list-speech-models.ts",
|
||||
"speech:download": "tsx scripts/download-speech-models.ts",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest",
|
||||
"test:ui": "vitest --ui",
|
||||
@@ -48,6 +50,10 @@
|
||||
"pino-pretty": "^13.1.3",
|
||||
"playwright": "^1.56.1",
|
||||
"qrcode": "^1.5.4",
|
||||
"@sctg/sentencepiece-js": "^1.1.0",
|
||||
"onnxruntime-node": "^1.23.0",
|
||||
"sherpa-onnx": "^1.12.23",
|
||||
"sherpa-onnx-node": "^1.12.23",
|
||||
"strip-ansi": "^7.1.2",
|
||||
"tiny-invariant": "^1.3.3",
|
||||
"uuid": "^9.0.1",
|
||||
|
||||
42
packages/server/scripts/download-speech-models.ts
Normal file
42
packages/server/scripts/download-speech-models.ts
Normal file
@@ -0,0 +1,42 @@
|
||||
import { resolvePaseoHome } from "../src/server/paseo-home.js";
|
||||
import { createRootLogger } from "../src/server/logger.js";
|
||||
import { ensureSherpaOnnxModels } from "../src/server/speech/sherpa/model-downloader.js";
|
||||
import type { SherpaOnnxModelId } from "../src/server/speech/sherpa/model-catalog.js";
|
||||
|
||||
function parseArgs(argv: string[]): { modelsDir: string; modelIds: SherpaOnnxModelId[] } {
|
||||
const home = resolvePaseoHome();
|
||||
let modelsDir = process.env.PASEO_SHERPA_ONNX_MODELS_DIR || `${home}/models/sherpa-onnx`;
|
||||
const modelIds: SherpaOnnxModelId[] = [];
|
||||
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const arg = argv[i];
|
||||
if (arg === "--models-dir") {
|
||||
modelsDir = argv[i + 1] ?? modelsDir;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--model") {
|
||||
const id = argv[i + 1] as SherpaOnnxModelId | undefined;
|
||||
if (!id) {
|
||||
throw new Error("--model requires a value");
|
||||
}
|
||||
modelIds.push(id);
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (modelIds.length === 0) {
|
||||
const stt = (process.env.PASEO_SHERPA_STT_PRESET || "zipformer-bilingual-zh-en-2023-02-20") as SherpaOnnxModelId;
|
||||
const tts = (process.env.PASEO_SHERPA_TTS_PRESET || "pocket-tts-onnx-int8") as SherpaOnnxModelId;
|
||||
modelIds.push(stt, tts);
|
||||
}
|
||||
|
||||
return { modelsDir, modelIds };
|
||||
}
|
||||
|
||||
const logger = createRootLogger({ level: "info", format: "pretty" });
|
||||
|
||||
const { modelsDir, modelIds } = parseArgs(process.argv.slice(2));
|
||||
await ensureSherpaOnnxModels({ modelsDir, modelIds, autoDownload: true, logger });
|
||||
logger.info({ modelsDir, modelIds }, "Done downloading speech models");
|
||||
11
packages/server/scripts/list-speech-models.ts
Normal file
11
packages/server/scripts/list-speech-models.ts
Normal file
@@ -0,0 +1,11 @@
|
||||
import { listSherpaOnnxModels } from "../src/server/speech/sherpa/model-catalog.js";
|
||||
|
||||
const models = listSherpaOnnxModels()
|
||||
.slice()
|
||||
.sort((a, b) => a.kind.localeCompare(b.kind) || a.id.localeCompare(b.id));
|
||||
|
||||
for (const m of models) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`${m.kind}\t${m.id}\t${m.description}`);
|
||||
}
|
||||
|
||||
@@ -169,7 +169,7 @@ describe("Codex app-server provider (integration)", () => {
|
||||
cleanup();
|
||||
rmSync(cwd, { recursive: true, force: true });
|
||||
}
|
||||
}, 60000);
|
||||
}, 120000);
|
||||
|
||||
test.runIf(isCodexInstalled())("listCommands includes custom prompts and executeCommand runs them", async () => {
|
||||
const cleanup = useTempCodexSessionDir();
|
||||
@@ -730,7 +730,9 @@ describe("Codex app-server provider (integration)", () => {
|
||||
if (captured) {
|
||||
expect(sawPermissionResolved).toBe(true);
|
||||
}
|
||||
expect(readFileSync(targetPath, "utf8").trim()).toBe("ok");
|
||||
const text = await waitForFileToContainText(targetPath, "ok", { timeoutMs: 5000 });
|
||||
expect(timelineItems.some((item) => hasApplyPatchFile(item, "approval-test.txt"))).toBe(true);
|
||||
expect(text?.trim()).toBe("ok");
|
||||
} finally {
|
||||
cleanup();
|
||||
rmSync(cwd, { recursive: true, force: true });
|
||||
|
||||
42
packages/server/src/server/agent/stt-manager.test.ts
Normal file
42
packages/server/src/server/agent/stt-manager.test.ts
Normal file
@@ -0,0 +1,42 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import pino from "pino";
|
||||
|
||||
import { STTManager } from "./stt-manager.js";
|
||||
import type { SpeechToTextProvider, TranscriptionResult } from "../speech/speech-provider.js";
|
||||
|
||||
class FakeStt implements SpeechToTextProvider {
|
||||
constructor(private readonly result: TranscriptionResult) {}
|
||||
|
||||
async transcribeAudio(): Promise<TranscriptionResult> {
|
||||
return this.result;
|
||||
}
|
||||
}
|
||||
|
||||
describe("STTManager", () => {
|
||||
it("returns empty text for low-confidence transcriptions", async () => {
|
||||
const manager = new STTManager(
|
||||
"s1",
|
||||
pino({ level: "silent" }),
|
||||
new FakeStt({ text: "um", isLowConfidence: true, avgLogprob: -10 })
|
||||
);
|
||||
|
||||
const result = await manager.transcribe(Buffer.from("x"), "audio/wav", { label: "t" });
|
||||
expect(result.text).toBe("");
|
||||
expect(result.isLowConfidence).toBe(true);
|
||||
expect(result.byteLength).toBe(1);
|
||||
});
|
||||
|
||||
it("passes through normal transcriptions", async () => {
|
||||
const manager = new STTManager(
|
||||
"s1",
|
||||
pino({ level: "silent" }),
|
||||
new FakeStt({ text: "hello world", language: "en", isLowConfidence: false })
|
||||
);
|
||||
|
||||
const result = await manager.transcribe(Buffer.from("abc"), "audio/wav");
|
||||
expect(result.text).toBe("hello world");
|
||||
expect(result.language).toBe("en");
|
||||
expect(result.byteLength).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type pino from "pino";
|
||||
import type { OpenAISTT, TranscriptionResult } from "./stt-openai.js";
|
||||
import type { SpeechToTextProvider, TranscriptionResult } from "../speech/speech-provider.js";
|
||||
import { maybePersistDebugAudio } from "./stt-debug.js";
|
||||
|
||||
interface TranscriptionMetadata {
|
||||
@@ -21,9 +21,9 @@ export interface SessionTranscriptionResult extends TranscriptionResult {
|
||||
export class STTManager {
|
||||
private readonly sessionId: string;
|
||||
private readonly logger: pino.Logger;
|
||||
private readonly stt: OpenAISTT | null;
|
||||
private readonly stt: SpeechToTextProvider | null;
|
||||
|
||||
constructor(sessionId: string, logger: pino.Logger, stt: OpenAISTT | null) {
|
||||
constructor(sessionId: string, logger: pino.Logger, stt: SpeechToTextProvider | null) {
|
||||
this.sessionId = sessionId;
|
||||
this.logger = logger.child({ module: "agent", component: "stt-manager", sessionId });
|
||||
this.stt = stt;
|
||||
|
||||
@@ -5,6 +5,9 @@ import { join } from "path";
|
||||
import { tmpdir } from "os";
|
||||
import { v4 } from "uuid";
|
||||
import { inferAudioExtension } from "./audio-utils.js";
|
||||
import type { LogprobToken, TranscriptionResult } from "../speech/speech-provider.js";
|
||||
|
||||
export type { LogprobToken, TranscriptionResult };
|
||||
|
||||
export interface STTConfig {
|
||||
apiKey: string;
|
||||
@@ -12,21 +15,6 @@ export interface STTConfig {
|
||||
confidenceThreshold?: number; // Default: -3.0
|
||||
}
|
||||
|
||||
export interface LogprobToken {
|
||||
token: string;
|
||||
logprob: number;
|
||||
bytes?: number[];
|
||||
}
|
||||
|
||||
export interface TranscriptionResult {
|
||||
text: string;
|
||||
language?: string;
|
||||
duration?: number;
|
||||
logprobs?: LogprobToken[];
|
||||
avgLogprob?: number;
|
||||
isLowConfidence?: boolean;
|
||||
}
|
||||
|
||||
function isObject(value: unknown): value is { [key: string]: unknown } {
|
||||
return typeof value === "object" && value !== null;
|
||||
}
|
||||
|
||||
47
packages/server/src/server/agent/tts-manager.test.ts
Normal file
47
packages/server/src/server/agent/tts-manager.test.ts
Normal file
@@ -0,0 +1,47 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import pino from "pino";
|
||||
import { Readable } from "node:stream";
|
||||
|
||||
import { TTSManager } from "./tts-manager.js";
|
||||
import type { TextToSpeechProvider } from "../speech/speech-provider.js";
|
||||
import type { SessionOutboundMessage } from "../messages.js";
|
||||
|
||||
class FakeTts implements TextToSpeechProvider {
|
||||
async synthesizeSpeech(): Promise<{ stream: Readable; format: string }> {
|
||||
return {
|
||||
stream: Readable.from([Buffer.from("a"), Buffer.from("b")]),
|
||||
format: "pcm;rate=24000",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
describe("TTSManager", () => {
|
||||
it("emits chunks and resolves once confirmed", async () => {
|
||||
const manager = new TTSManager("s1", pino({ level: "silent" }), new FakeTts());
|
||||
const abort = new AbortController();
|
||||
const emitted: SessionOutboundMessage[] = [];
|
||||
|
||||
const task = manager.generateAndWaitForPlayback(
|
||||
"hello",
|
||||
(msg) => {
|
||||
emitted.push(msg);
|
||||
if (msg.type === "audio_output") {
|
||||
manager.confirmAudioPlayed(msg.payload.id);
|
||||
}
|
||||
},
|
||||
abort.signal,
|
||||
true
|
||||
);
|
||||
|
||||
await task;
|
||||
|
||||
const audioMsgs = emitted.filter((m) => m.type === "audio_output");
|
||||
expect(audioMsgs).toHaveLength(2);
|
||||
const groupId = (audioMsgs[0] as any).payload.groupId;
|
||||
expect(groupId).toBeTruthy();
|
||||
expect((audioMsgs[0] as any).payload.chunkIndex).toBe(0);
|
||||
expect((audioMsgs[1] as any).payload.chunkIndex).toBe(1);
|
||||
expect((audioMsgs[1] as any).payload.isLastChunk).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type pino from "pino";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import type { OpenAITTS } from "./tts-openai.js";
|
||||
import type { TextToSpeechProvider } from "../speech/speech-provider.js";
|
||||
import type { SessionOutboundMessage } from "../messages.js";
|
||||
|
||||
interface PendingPlayback {
|
||||
@@ -17,9 +17,9 @@ interface PendingPlayback {
|
||||
export class TTSManager {
|
||||
private pendingPlaybacks: Map<string, PendingPlayback> = new Map();
|
||||
private readonly logger: pino.Logger;
|
||||
private readonly tts: OpenAITTS | null;
|
||||
private readonly tts: TextToSpeechProvider | null;
|
||||
|
||||
constructor(sessionId: string, logger: pino.Logger, tts: OpenAITTS | null) {
|
||||
constructor(sessionId: string, logger: pino.Logger, tts: TextToSpeechProvider | null) {
|
||||
this.logger = logger.child({ module: "agent", component: "tts-manager", sessionId });
|
||||
this.tts = tts;
|
||||
}
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import type pino from "pino";
|
||||
import OpenAI from "openai";
|
||||
import { Readable } from "stream";
|
||||
import { Readable } from "node:stream";
|
||||
import type { SpeechStreamResult } from "../speech/speech-provider.js";
|
||||
|
||||
export type { SpeechStreamResult };
|
||||
|
||||
export interface TTSConfig {
|
||||
apiKey: string;
|
||||
@@ -9,11 +12,6 @@ export interface TTSConfig {
|
||||
responseFormat?: "mp3" | "opus" | "aac" | "flac" | "wav" | "pcm";
|
||||
}
|
||||
|
||||
export interface SpeechStreamResult {
|
||||
stream: Readable;
|
||||
format: string;
|
||||
}
|
||||
|
||||
export class OpenAITTS {
|
||||
private readonly openaiClient: OpenAI;
|
||||
private readonly config: TTSConfig;
|
||||
|
||||
@@ -42,6 +42,18 @@ import { VoiceAssistantWebSocketServer } from "./websocket-server.js";
|
||||
import { DownloadTokenStore } from "./file-download/token-store.js";
|
||||
import { OpenAISTT, type STTConfig } from "./agent/stt-openai.js";
|
||||
import { OpenAITTS, type TTSConfig } from "./agent/tts-openai.js";
|
||||
import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js";
|
||||
import type { RealtimeTranscriptionSessionFactory } from "./dictation/dictation-stream-manager.js";
|
||||
import { SherpaOnlineRecognizerEngine } from "./speech/sherpa/sherpa-online-recognizer.js";
|
||||
import { SherpaOfflineRecognizerEngine } from "./speech/sherpa/sherpa-offline-recognizer.js";
|
||||
import { SherpaOnnxSTT } from "./speech/sherpa/sherpa-stt.js";
|
||||
import { SherpaOnnxParakeetSTT } from "./speech/sherpa/sherpa-parakeet-stt.js";
|
||||
import { SherpaOnnxTTS } from "./speech/sherpa/sherpa-tts.js";
|
||||
import { SherpaRealtimeTranscriptionSession } from "./speech/sherpa/sherpa-realtime-session.js";
|
||||
import { SherpaParakeetRealtimeTranscriptionSession } from "./speech/sherpa/sherpa-parakeet-realtime-session.js";
|
||||
import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./speech/sherpa/model-downloader.js";
|
||||
import type { SherpaOnnxModelId } from "./speech/sherpa/model-catalog.js";
|
||||
import { PocketTtsOnnxTTS } from "./speech/pocket/pocket-tts-onnx.js";
|
||||
import { AgentManager } from "./agent/agent-manager.js";
|
||||
import { AgentStorage } from "./agent/agent-storage.js";
|
||||
import { attachAgentStoragePersistence } from "./persistence-hooks.js";
|
||||
@@ -72,6 +84,26 @@ export type PaseoOpenAIConfig = {
|
||||
tts?: Partial<TTSConfig> & { apiKey?: string };
|
||||
};
|
||||
|
||||
export type PaseoSherpaOnnxConfig = {
|
||||
modelsDir: string;
|
||||
autoDownload?: boolean;
|
||||
stt?: {
|
||||
preset?: string;
|
||||
};
|
||||
tts?: {
|
||||
preset?: string;
|
||||
speakerId?: number;
|
||||
speed?: number;
|
||||
};
|
||||
};
|
||||
|
||||
export type PaseoSpeechConfig = {
|
||||
dictationSttProvider?: "openai" | "sherpa";
|
||||
voiceSttProvider?: "openai" | "sherpa";
|
||||
voiceTtsProvider?: "openai" | "sherpa";
|
||||
sherpaOnnx?: PaseoSherpaOnnxConfig;
|
||||
};
|
||||
|
||||
export type PaseoDaemonConfig = {
|
||||
listen: string;
|
||||
paseoHome: string;
|
||||
@@ -87,6 +119,7 @@ export type PaseoDaemonConfig = {
|
||||
relayPublicEndpoint?: string;
|
||||
appBaseUrl?: string;
|
||||
openai?: PaseoOpenAIConfig;
|
||||
speech?: PaseoSpeechConfig;
|
||||
openrouterApiKey?: string | null;
|
||||
voiceLlmModel?: string | null;
|
||||
dictationFinalTimeoutMs?: number;
|
||||
@@ -382,42 +415,282 @@ export async function createPaseoDaemon(
|
||||
}
|
||||
|
||||
|
||||
let sttService: OpenAISTT | null = null;
|
||||
let ttsService: OpenAITTS | null = null;
|
||||
let sttService: SpeechToTextProvider | null = null;
|
||||
let ttsService: TextToSpeechProvider | null = null;
|
||||
let dictationSessionFactory: RealtimeTranscriptionSessionFactory | undefined;
|
||||
|
||||
let sherpaOnline: SherpaOnlineRecognizerEngine | null = null;
|
||||
let sherpaOffline: SherpaOfflineRecognizerEngine | null = null;
|
||||
let sherpaTts: TextToSpeechProvider | null = null;
|
||||
|
||||
const openaiApiKey = config.openai?.apiKey;
|
||||
if (openaiApiKey) {
|
||||
const speechConfig = config.speech ?? null;
|
||||
const sherpaConfig = speechConfig?.sherpaOnnx ?? null;
|
||||
|
||||
const wantsSherpaDictation = (speechConfig?.dictationSttProvider ?? "openai") === "sherpa";
|
||||
const wantsSherpaVoiceStt = (speechConfig?.voiceSttProvider ?? "openai") === "sherpa";
|
||||
const wantsSherpaVoiceTts = (speechConfig?.voiceTtsProvider ?? "openai") === "sherpa";
|
||||
|
||||
if ((wantsSherpaDictation || wantsSherpaVoiceStt || wantsSherpaVoiceTts) && sherpaConfig) {
|
||||
const autoDownload = sherpaConfig.autoDownload ?? (process.env.VITEST ? false : true);
|
||||
let sttPreset = (sherpaConfig.stt?.preset ?? "zipformer-bilingual-zh-en-2023-02-20").trim();
|
||||
if (
|
||||
sttPreset !== "zipformer-bilingual-zh-en-2023-02-20" &&
|
||||
sttPreset !== "paraformer-bilingual-zh-en" &&
|
||||
sttPreset !== "parakeet-tdt-0.6b-v3-int8"
|
||||
) {
|
||||
logger.warn(
|
||||
{ sttPreset },
|
||||
"Unknown Sherpa STT preset; falling back to zipformer-bilingual-zh-en-2023-02-20"
|
||||
);
|
||||
sttPreset = "zipformer-bilingual-zh-en-2023-02-20";
|
||||
}
|
||||
|
||||
let ttsPreset = (sherpaConfig.tts?.preset ?? "pocket-tts-onnx-int8").trim();
|
||||
if (
|
||||
ttsPreset !== "kitten-nano-en-v0_1-fp16" &&
|
||||
ttsPreset !== "kokoro-en-v0_19" &&
|
||||
ttsPreset !== "pocket-tts-onnx-int8"
|
||||
) {
|
||||
logger.warn(
|
||||
{ ttsPreset },
|
||||
"Unknown Sherpa TTS preset; falling back to kitten-nano-en-v0_1-fp16"
|
||||
);
|
||||
ttsPreset = "kitten-nano-en-v0_1-fp16";
|
||||
}
|
||||
|
||||
const modelIds: SherpaOnnxModelId[] = [];
|
||||
if (wantsSherpaDictation || wantsSherpaVoiceStt) {
|
||||
modelIds.push(sttPreset as SherpaOnnxModelId);
|
||||
}
|
||||
if (wantsSherpaVoiceTts) {
|
||||
modelIds.push(ttsPreset as SherpaOnnxModelId);
|
||||
}
|
||||
|
||||
try {
|
||||
await ensureSherpaOnnxModels({
|
||||
modelsDir: sherpaConfig.modelsDir,
|
||||
modelIds,
|
||||
autoDownload,
|
||||
logger,
|
||||
});
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
{
|
||||
err,
|
||||
modelsDir: sherpaConfig.modelsDir,
|
||||
autoDownload,
|
||||
hint:
|
||||
"Run: npm run dev --workspace=@getpaseo/server, then run: " +
|
||||
"`tsx packages/server/scripts/download-speech-models.ts --models-dir <DIR> --model <MODEL_ID>`",
|
||||
},
|
||||
"Failed to ensure local speech models"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if ((wantsSherpaDictation || wantsSherpaVoiceStt) && sherpaConfig) {
|
||||
let preset = (sherpaConfig.stt?.preset ?? "zipformer-bilingual-zh-en-2023-02-20").trim();
|
||||
if (
|
||||
preset !== "zipformer-bilingual-zh-en-2023-02-20" &&
|
||||
preset !== "paraformer-bilingual-zh-en" &&
|
||||
preset !== "parakeet-tdt-0.6b-v3-int8"
|
||||
) {
|
||||
logger.warn(
|
||||
{ preset },
|
||||
"Unknown Sherpa STT preset; falling back to zipformer-bilingual-zh-en-2023-02-20"
|
||||
);
|
||||
preset = "zipformer-bilingual-zh-en-2023-02-20";
|
||||
}
|
||||
const base = sherpaConfig.modelsDir;
|
||||
|
||||
try {
|
||||
if (preset === "parakeet-tdt-0.6b-v3-int8") {
|
||||
const modelDir = getSherpaOnnxModelDir(base, "parakeet-tdt-0.6b-v3-int8");
|
||||
sherpaOffline = new SherpaOfflineRecognizerEngine(
|
||||
{
|
||||
model: {
|
||||
kind: "nemo_transducer",
|
||||
encoder: `${modelDir}/encoder.int8.onnx`,
|
||||
decoder: `${modelDir}/decoder.int8.onnx`,
|
||||
joiner: `${modelDir}/joiner.int8.onnx`,
|
||||
tokens: `${modelDir}/tokens.txt`,
|
||||
},
|
||||
numThreads: 2,
|
||||
debug: 0,
|
||||
},
|
||||
logger
|
||||
);
|
||||
} else {
|
||||
const model =
|
||||
preset === "paraformer-bilingual-zh-en"
|
||||
? {
|
||||
kind: "paraformer" as const,
|
||||
encoder: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/encoder.int8.onnx`,
|
||||
decoder: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/decoder.int8.onnx`,
|
||||
tokens: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/tokens.txt`,
|
||||
}
|
||||
: {
|
||||
kind: "transducer" as const,
|
||||
encoder: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/encoder-epoch-99-avg-1.onnx`,
|
||||
decoder: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/decoder-epoch-99-avg-1.onnx`,
|
||||
joiner: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/joiner-epoch-99-avg-1.onnx`,
|
||||
tokens: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt`,
|
||||
modelType: "zipformer",
|
||||
};
|
||||
|
||||
sherpaOnline = new SherpaOnlineRecognizerEngine(
|
||||
{
|
||||
model,
|
||||
numThreads: 1,
|
||||
debug: 0,
|
||||
},
|
||||
logger
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
{
|
||||
err,
|
||||
modelsDir: sherpaConfig.modelsDir,
|
||||
preset,
|
||||
hint: `Run: tsx packages/server/scripts/download-speech-models.ts --models-dir '${sherpaConfig.modelsDir}' --model '${preset}'`,
|
||||
},
|
||||
"Failed to initialize Sherpa STT (models missing or invalid)"
|
||||
);
|
||||
sherpaOnline = null;
|
||||
sherpaOffline = null;
|
||||
}
|
||||
} else if (wantsSherpaDictation || wantsSherpaVoiceStt) {
|
||||
logger.warn(
|
||||
{ configured: Boolean(sherpaConfig) },
|
||||
"Sherpa STT selected but no sherpaOnnx config found; STT will be unavailable"
|
||||
);
|
||||
}
|
||||
|
||||
if (wantsSherpaVoiceTts && sherpaConfig) {
|
||||
let preset = (sherpaConfig.tts?.preset ?? "pocket-tts-onnx-int8").trim();
|
||||
if (
|
||||
preset !== "kitten-nano-en-v0_1-fp16" &&
|
||||
preset !== "kokoro-en-v0_19" &&
|
||||
preset !== "pocket-tts-onnx-int8"
|
||||
) {
|
||||
logger.warn(
|
||||
{ preset },
|
||||
"Unknown Sherpa TTS preset; falling back to kitten-nano-en-v0_1-fp16"
|
||||
);
|
||||
preset = "kitten-nano-en-v0_1-fp16";
|
||||
}
|
||||
try {
|
||||
if (preset === "pocket-tts-onnx-int8") {
|
||||
const modelDir = getSherpaOnnxModelDir(sherpaConfig.modelsDir, "pocket-tts-onnx-int8");
|
||||
sherpaTts = await PocketTtsOnnxTTS.create(
|
||||
{
|
||||
modelDir,
|
||||
precision: "int8",
|
||||
targetChunkMs: 50,
|
||||
},
|
||||
logger
|
||||
);
|
||||
} else {
|
||||
const modelDir = `${sherpaConfig.modelsDir}/${preset}`;
|
||||
sherpaTts = new SherpaOnnxTTS(
|
||||
{
|
||||
preset: preset as any,
|
||||
modelDir,
|
||||
speakerId: sherpaConfig.tts?.speakerId,
|
||||
speed: sherpaConfig.tts?.speed,
|
||||
},
|
||||
logger
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
{
|
||||
err,
|
||||
preset,
|
||||
hint: `Run: tsx packages/server/scripts/download-speech-models.ts --models-dir '${sherpaConfig.modelsDir}' --model '${preset}'`,
|
||||
},
|
||||
"Failed to initialize Sherpa TTS (models missing or invalid)"
|
||||
);
|
||||
sherpaTts = null;
|
||||
}
|
||||
} else if (wantsSherpaVoiceTts) {
|
||||
logger.warn(
|
||||
{ configured: Boolean(sherpaConfig) },
|
||||
"Sherpa TTS selected but no sherpaOnnx config found; TTS will be unavailable"
|
||||
);
|
||||
}
|
||||
|
||||
if (wantsSherpaVoiceStt && sherpaOffline) {
|
||||
sttService = new SherpaOnnxParakeetSTT({ engine: sherpaOffline }, logger);
|
||||
} else if (wantsSherpaVoiceStt && sherpaOnline) {
|
||||
sttService = new SherpaOnnxSTT({ engine: sherpaOnline }, logger);
|
||||
}
|
||||
|
||||
if (wantsSherpaVoiceTts && sherpaTts) {
|
||||
ttsService = sherpaTts;
|
||||
}
|
||||
|
||||
if (wantsSherpaDictation && sherpaOnline) {
|
||||
dictationSessionFactory = () => new SherpaRealtimeTranscriptionSession({ engine: sherpaOnline! });
|
||||
} else if (wantsSherpaDictation && sherpaOffline) {
|
||||
dictationSessionFactory = () => new SherpaParakeetRealtimeTranscriptionSession({ engine: sherpaOffline! });
|
||||
}
|
||||
|
||||
const voiceSttProvider = speechConfig?.voiceSttProvider ?? "openai";
|
||||
const voiceTtsProvider = speechConfig?.voiceTtsProvider ?? "openai";
|
||||
const dictationSttProvider = speechConfig?.dictationSttProvider ?? "openai";
|
||||
|
||||
const needsOpenAiStt = !sttService && voiceSttProvider === "openai";
|
||||
const needsOpenAiTts = !ttsService && voiceTtsProvider === "openai";
|
||||
const needsOpenAiDictation =
|
||||
dictationSttProvider === "openai" || (dictationSttProvider === "sherpa" && !dictationSessionFactory);
|
||||
|
||||
const fallbackOpenAiStt = !sttService && voiceSttProvider === "sherpa" && Boolean(openaiApiKey);
|
||||
const fallbackOpenAiTts = !ttsService && voiceTtsProvider === "sherpa" && Boolean(openaiApiKey);
|
||||
|
||||
if ((needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation || fallbackOpenAiStt || fallbackOpenAiTts) && openaiApiKey) {
|
||||
logger.info("OpenAI client initialized");
|
||||
|
||||
const sttApiKey = config.openai?.stt?.apiKey ?? openaiApiKey;
|
||||
if (sttApiKey) {
|
||||
const { apiKey: _sttApiKey, ...sttConfig } = config.openai?.stt ?? {};
|
||||
sttService = new OpenAISTT(
|
||||
{
|
||||
apiKey: sttApiKey,
|
||||
...sttConfig,
|
||||
},
|
||||
logger
|
||||
);
|
||||
if (fallbackOpenAiStt) {
|
||||
logger.warn("Falling back to OpenAI STT because Sherpa STT is unavailable");
|
||||
}
|
||||
if (needsOpenAiStt || fallbackOpenAiStt) {
|
||||
const sttApiKey = config.openai?.stt?.apiKey ?? openaiApiKey;
|
||||
if (sttApiKey) {
|
||||
const { apiKey: _sttApiKey, ...sttConfig } = config.openai?.stt ?? {};
|
||||
sttService = new OpenAISTT(
|
||||
{
|
||||
apiKey: sttApiKey,
|
||||
...sttConfig,
|
||||
},
|
||||
logger
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const ttsApiKey = config.openai?.tts?.apiKey ?? openaiApiKey;
|
||||
if (ttsApiKey) {
|
||||
const { apiKey: _ttsApiKey, ...ttsConfig } = config.openai?.tts ?? {};
|
||||
ttsService = new OpenAITTS(
|
||||
{
|
||||
apiKey: ttsApiKey,
|
||||
voice: "alloy",
|
||||
model: "tts-1",
|
||||
responseFormat: "pcm",
|
||||
...ttsConfig,
|
||||
},
|
||||
logger
|
||||
);
|
||||
if (fallbackOpenAiTts) {
|
||||
logger.warn("Falling back to OpenAI TTS because Sherpa TTS is unavailable");
|
||||
}
|
||||
|
||||
} else {
|
||||
logger.warn("OPENAI_API_KEY not set - LLM, STT, and TTS features will not work");
|
||||
if (needsOpenAiTts || fallbackOpenAiTts) {
|
||||
const ttsApiKey = config.openai?.tts?.apiKey ?? openaiApiKey;
|
||||
if (ttsApiKey) {
|
||||
const { apiKey: _ttsApiKey, ...ttsConfig } = config.openai?.tts ?? {};
|
||||
ttsService = new OpenAITTS(
|
||||
{
|
||||
apiKey: ttsApiKey,
|
||||
voice: "alloy",
|
||||
model: "tts-1",
|
||||
responseFormat: "pcm",
|
||||
...ttsConfig,
|
||||
},
|
||||
logger
|
||||
);
|
||||
}
|
||||
}
|
||||
} else if (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation || fallbackOpenAiStt || fallbackOpenAiTts) {
|
||||
logger.warn("OPENAI_API_KEY not set - OpenAI STT/TTS/dictation fallback is unavailable");
|
||||
}
|
||||
|
||||
const wsServer = new VoiceAssistantWebSocketServer(
|
||||
@@ -439,6 +712,7 @@ export async function createPaseoDaemon(
|
||||
{
|
||||
openaiApiKey: config.openai?.apiKey ?? null,
|
||||
finalTimeoutMs: config.dictationFinalTimeoutMs,
|
||||
...(dictationSessionFactory ? { sessionFactory: dictationSessionFactory } : {}),
|
||||
}
|
||||
);
|
||||
|
||||
@@ -533,6 +807,11 @@ export async function createPaseoDaemon(
|
||||
await agentStorage.flush().catch(() => undefined);
|
||||
await shutdownProviders(logger);
|
||||
terminalManager.killAll();
|
||||
if (sherpaTts && typeof (sherpaTts as any).free === "function") {
|
||||
(sherpaTts as any).free();
|
||||
}
|
||||
sherpaOnline?.free();
|
||||
sherpaOffline?.free();
|
||||
await relayTransport?.stop().catch(() => undefined);
|
||||
await wsServer.close();
|
||||
await new Promise<void>((resolve) => {
|
||||
|
||||
@@ -76,6 +76,51 @@ function parseOpenAIConfig(
|
||||
};
|
||||
}
|
||||
|
||||
function parseSpeechProviderId(value: unknown): "openai" | "sherpa" | null {
|
||||
if (typeof value !== "string") {
|
||||
return null;
|
||||
}
|
||||
const normalized = value.trim().toLowerCase();
|
||||
if (!normalized) {
|
||||
return null;
|
||||
}
|
||||
if (normalized === "openai") return "openai";
|
||||
if (normalized === "sherpa" || normalized === "sherpa-onnx" || normalized === "local") {
|
||||
return "sherpa";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function normalizeSherpaSttPreset(value: string): string {
|
||||
const raw = value.trim();
|
||||
const normalized = raw.toLowerCase();
|
||||
if (normalized === "zipformer" || normalized === "zipformer-bilingual") {
|
||||
return "zipformer-bilingual-zh-en-2023-02-20";
|
||||
}
|
||||
if (normalized === "paraformer") {
|
||||
return "paraformer-bilingual-zh-en";
|
||||
}
|
||||
if (normalized === "parakeet" || normalized === "parakeet-v3" || normalized === "parakeet-tdt") {
|
||||
return "parakeet-tdt-0.6b-v3-int8";
|
||||
}
|
||||
return raw;
|
||||
}
|
||||
|
||||
function normalizeSherpaTtsPreset(value: string): string {
|
||||
const raw = value.trim();
|
||||
const normalized = raw.toLowerCase();
|
||||
if (normalized === "pocket" || normalized === "pocket-tts") {
|
||||
return "pocket-tts-onnx-int8";
|
||||
}
|
||||
if (normalized === "kitten") {
|
||||
return "kitten-nano-en-v0_1-fp16";
|
||||
}
|
||||
if (normalized === "kokoro") {
|
||||
return "kokoro-en-v0_19";
|
||||
}
|
||||
return raw;
|
||||
}
|
||||
|
||||
export function loadConfig(
|
||||
paseoHome: string,
|
||||
options?: {
|
||||
@@ -137,6 +182,70 @@ export function loadConfig(
|
||||
voiceTtsVoice: persisted.features?.voiceMode?.tts?.voice,
|
||||
});
|
||||
|
||||
const dictationSttProvider =
|
||||
parseSpeechProviderId(env.PASEO_DICTATION_STT_PROVIDER) ??
|
||||
parseSpeechProviderId(persisted.features?.dictation?.stt?.provider) ??
|
||||
"sherpa";
|
||||
|
||||
const voiceSttProvider =
|
||||
parseSpeechProviderId(env.PASEO_VOICE_STT_PROVIDER) ??
|
||||
parseSpeechProviderId(persisted.features?.voiceMode?.stt?.provider) ??
|
||||
"sherpa";
|
||||
|
||||
const voiceTtsProvider =
|
||||
parseSpeechProviderId(env.PASEO_VOICE_TTS_PROVIDER) ??
|
||||
parseSpeechProviderId(persisted.features?.voiceMode?.tts?.provider) ??
|
||||
"sherpa";
|
||||
|
||||
const shouldConfigureSherpa =
|
||||
dictationSttProvider === "sherpa" ||
|
||||
voiceSttProvider === "sherpa" ||
|
||||
voiceTtsProvider === "sherpa" ||
|
||||
typeof env.PASEO_SHERPA_ONNX_MODELS_DIR === "string" ||
|
||||
Boolean(persisted.providers?.sherpaOnnx);
|
||||
|
||||
const sherpaModelsDir =
|
||||
(env.PASEO_SHERPA_ONNX_MODELS_DIR ?? persisted.providers?.sherpaOnnx?.modelsDir)?.trim() ||
|
||||
path.join(paseoHome, "models", "sherpa-onnx");
|
||||
|
||||
const sherpaOnnx = shouldConfigureSherpa
|
||||
? {
|
||||
modelsDir: sherpaModelsDir,
|
||||
autoDownload:
|
||||
env.PASEO_SHERPA_ONNX_AUTO_DOWNLOAD !== undefined
|
||||
? env.PASEO_SHERPA_ONNX_AUTO_DOWNLOAD === "1"
|
||||
: persisted.providers?.sherpaOnnx?.autoDownload ??
|
||||
// In tests we should never hit the network unexpectedly.
|
||||
Boolean(env.VITEST) === false,
|
||||
stt: {
|
||||
preset: normalizeSherpaSttPreset(
|
||||
(env.PASEO_SHERPA_STT_PRESET ?? persisted.providers?.sherpaOnnx?.stt?.preset)?.trim() ||
|
||||
(persisted.features?.voiceMode?.stt?.preset ??
|
||||
persisted.features?.dictation?.stt?.preset)?.trim() ||
|
||||
"zipformer-bilingual-zh-en-2023-02-20"
|
||||
),
|
||||
},
|
||||
tts: {
|
||||
preset: normalizeSherpaTtsPreset(
|
||||
(env.PASEO_SHERPA_TTS_PRESET ??
|
||||
persisted.providers?.sherpaOnnx?.tts?.preset ??
|
||||
persisted.features?.voiceMode?.tts?.preset)?.trim() ||
|
||||
(env.VITEST ? "kitten-nano-en-v0_1-fp16" : "pocket-tts-onnx-int8")
|
||||
),
|
||||
speakerId:
|
||||
env.PASEO_SHERPA_TTS_SPEAKER_ID !== undefined
|
||||
? Number.parseInt(env.PASEO_SHERPA_TTS_SPEAKER_ID, 10)
|
||||
: persisted.providers?.sherpaOnnx?.tts?.speakerId ??
|
||||
persisted.features?.voiceMode?.tts?.speakerId,
|
||||
speed:
|
||||
env.PASEO_SHERPA_TTS_SPEED !== undefined
|
||||
? Number.parseFloat(env.PASEO_SHERPA_TTS_SPEED)
|
||||
: persisted.providers?.sherpaOnnx?.tts?.speed ??
|
||||
persisted.features?.voiceMode?.tts?.speed,
|
||||
},
|
||||
}
|
||||
: undefined;
|
||||
|
||||
const openrouterApiKey =
|
||||
env.OPENROUTER_API_KEY ?? persisted.providers?.openrouter?.apiKey ?? null;
|
||||
const voiceLlmModel = persisted.features?.voiceMode?.llm?.model ?? null;
|
||||
@@ -158,6 +267,12 @@ export function loadConfig(
|
||||
relayPublicEndpoint,
|
||||
appBaseUrl,
|
||||
openai,
|
||||
speech: {
|
||||
dictationSttProvider,
|
||||
voiceSttProvider,
|
||||
voiceTtsProvider,
|
||||
...(sherpaOnnx ? { sherpaOnnx } : {}),
|
||||
},
|
||||
openrouterApiKey,
|
||||
voiceLlmModel,
|
||||
};
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach } from "vitest";
|
||||
import { mkdtempSync, writeFileSync, rmSync, existsSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { tmpdir, homedir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { execSync } from "node:child_process";
|
||||
|
||||
@@ -13,12 +13,45 @@ import { getFullAccessConfig, getAskModeConfig } from "./daemon-e2e/agent-config
|
||||
import {
|
||||
chunkPcm16,
|
||||
parsePcm16MonoWav,
|
||||
requireEnv,
|
||||
transcribeBaselineOpenAI,
|
||||
wordSimilarity,
|
||||
} from "./test-utils/dictation-e2e.js";
|
||||
|
||||
const hasOpenAICredentials = !!process.env.OPENAI_API_KEY;
|
||||
const openaiApiKey = process.env.OPENAI_API_KEY ?? null;
|
||||
|
||||
const sherpaModelsDir =
|
||||
process.env.PASEO_SHERPA_ONNX_MODELS_DIR ??
|
||||
path.join(homedir(), ".paseo", "models", "sherpa-onnx");
|
||||
|
||||
function hasSherpaZipformerModels(modelsDir: string): boolean {
|
||||
return (
|
||||
existsSync(
|
||||
path.join(
|
||||
modelsDir,
|
||||
"sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20",
|
||||
"encoder-epoch-99-avg-1.onnx"
|
||||
)
|
||||
) &&
|
||||
existsSync(
|
||||
path.join(
|
||||
modelsDir,
|
||||
"sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20",
|
||||
"tokens.txt"
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function hasSherpaKittenModels(modelsDir: string): boolean {
|
||||
return (
|
||||
existsSync(path.join(modelsDir, "kitten-nano-en-v0_1-fp16", "model.fp16.onnx")) &&
|
||||
existsSync(path.join(modelsDir, "kitten-nano-en-v0_1-fp16", "voices.bin")) &&
|
||||
existsSync(path.join(modelsDir, "kitten-nano-en-v0_1-fp16", "tokens.txt"))
|
||||
);
|
||||
}
|
||||
|
||||
const hasLocalSpeech = hasSherpaZipformerModels(sherpaModelsDir) && hasSherpaKittenModels(sherpaModelsDir);
|
||||
const hasAnySpeech = hasLocalSpeech || Boolean(openaiApiKey);
|
||||
const speechTest = hasAnySpeech ? test : test.skip;
|
||||
|
||||
function tmpCwd(): string {
|
||||
return mkdtempSync(path.join(tmpdir(), "daemon-client-"));
|
||||
@@ -55,14 +88,27 @@ function waitForSignal<T>(
|
||||
});
|
||||
}
|
||||
|
||||
(hasOpenAICredentials ? describe : describe.skip)("daemon client E2E", () => {
|
||||
describe("daemon client E2E", () => {
|
||||
let ctx: DaemonTestContext;
|
||||
|
||||
beforeAll(async () => {
|
||||
const openaiApiKey = process.env.OPENAI_API_KEY ?? "";
|
||||
ctx = await createDaemonTestContext({
|
||||
dictationFinalTimeoutMs: 5000,
|
||||
openai: { apiKey: openaiApiKey },
|
||||
...(openaiApiKey ? { openai: { apiKey: openaiApiKey } } : {}),
|
||||
speech: {
|
||||
dictationSttProvider: "sherpa",
|
||||
voiceSttProvider: "sherpa",
|
||||
voiceTtsProvider: "sherpa",
|
||||
sherpaOnnx: {
|
||||
modelsDir: sherpaModelsDir,
|
||||
stt: {
|
||||
preset: process.env.PASEO_SHERPA_STT_PRESET ?? "zipformer-bilingual-zh-en-2023-02-20",
|
||||
},
|
||||
tts: {
|
||||
preset: process.env.PASEO_SHERPA_TTS_PRESET ?? "kitten-nano-en-v0_1-fp16",
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
}, 60000);
|
||||
|
||||
@@ -569,11 +615,9 @@ function waitForSignal<T>(
|
||||
120000
|
||||
);
|
||||
|
||||
test(
|
||||
speechTest(
|
||||
"voice mode buffers audio until isLast and emits transcription_result",
|
||||
async () => {
|
||||
requireEnv("OPENAI_API_KEY");
|
||||
|
||||
await ctx.client.setVoiceConversation(true, `voice-${Date.now()}`);
|
||||
|
||||
const transcription = waitForSignal(30_000, (resolve) => {
|
||||
@@ -675,11 +719,9 @@ function waitForSignal<T>(
|
||||
90_000
|
||||
);
|
||||
|
||||
test(
|
||||
"streams dictation PCM and returns final transcript via OpenAI Realtime transcription",
|
||||
speechTest(
|
||||
"streams dictation PCM and returns final transcript",
|
||||
async () => {
|
||||
requireEnv("OPENAI_API_KEY");
|
||||
|
||||
const fixturePath = path.resolve(
|
||||
process.cwd(),
|
||||
"..",
|
||||
@@ -713,11 +755,9 @@ function waitForSignal<T>(
|
||||
30_000
|
||||
);
|
||||
|
||||
test(
|
||||
"realtime dictation transcript is similar to baseline (OpenAI transcriptions API)",
|
||||
speechTest(
|
||||
"realtime dictation transcript is similar to baseline fixture",
|
||||
async () => {
|
||||
const apiKey = requireEnv("OPENAI_API_KEY");
|
||||
|
||||
const fixturePath = path.resolve(
|
||||
process.cwd(),
|
||||
"..",
|
||||
@@ -732,14 +772,17 @@ function waitForSignal<T>(
|
||||
const dictationId = `dict-baseline-${Date.now()}`;
|
||||
const format = "audio/pcm;rate=16000;bits=16";
|
||||
|
||||
const baseline = await transcribeBaselineOpenAI({
|
||||
apiKey,
|
||||
wav,
|
||||
model: process.env.STT_MODEL ?? "whisper-1",
|
||||
prompt:
|
||||
process.env.OPENAI_REALTIME_DICTATION_TRANSCRIPTION_PROMPT ??
|
||||
"Transcribe only what the speaker says. Do not add words. Preserve punctuation and casing. If the audio is silence or non-speech noise, return an empty transcript.",
|
||||
});
|
||||
const baselinePath = path.resolve(
|
||||
process.cwd(),
|
||||
"..",
|
||||
"app",
|
||||
"e2e",
|
||||
"fixtures",
|
||||
"recording.baseline.txt"
|
||||
);
|
||||
const baseline = await import("node:fs/promises")
|
||||
.then((fs) => fs.readFile(baselinePath, "utf-8"))
|
||||
.then((text) => text.trim());
|
||||
|
||||
await ctx.client.startDictationStream(dictationId, format);
|
||||
|
||||
@@ -755,16 +798,14 @@ function waitForSignal<T>(
|
||||
const result = await ctx.client.finishDictationStream(dictationId, finalSeq);
|
||||
|
||||
expect(result.dictationId).toBe(dictationId);
|
||||
expect(wordSimilarity(result.text, baseline)).toBeGreaterThan(0.8);
|
||||
expect(wordSimilarity(result.text, baseline)).toBeGreaterThan(0.6);
|
||||
},
|
||||
30_000
|
||||
);
|
||||
|
||||
test(
|
||||
speechTest(
|
||||
"fails fast if dictation finishes without sending required chunks",
|
||||
async () => {
|
||||
requireEnv("OPENAI_API_KEY");
|
||||
|
||||
const dictationId = `dict-missing-chunks-${Date.now()}`;
|
||||
const format = "audio/pcm;rate=16000;bits=16";
|
||||
|
||||
|
||||
@@ -151,3 +151,34 @@ describe("DictationStreamManager (semantic VAD grace fallback)", () => {
|
||||
expect(final?.payload.text).toBe("hi there");
|
||||
});
|
||||
});
|
||||
|
||||
describe("DictationStreamManager (provider-agnostic session factory)", () => {
|
||||
it("can start with a custom session factory even without OPENAI_API_KEY", async () => {
|
||||
const original = process.env.OPENAI_API_KEY;
|
||||
delete process.env.OPENAI_API_KEY;
|
||||
|
||||
try {
|
||||
const session = new FakeRealtimeSession();
|
||||
const factory: RealtimeTranscriptionSessionFactory = () => session;
|
||||
const emitted: Array<{ type: string; payload: any }> = [];
|
||||
const manager = new DictationStreamManager({
|
||||
logger: pino({ level: "silent" }),
|
||||
emit: (msg) => emitted.push(msg),
|
||||
sessionId: "s1",
|
||||
openaiApiKey: null,
|
||||
sessionFactory: factory,
|
||||
});
|
||||
|
||||
await manager.handleStart("d-local", "audio/pcm;rate=16000;bits=16");
|
||||
|
||||
expect(session.connected).toBe(true);
|
||||
expect(emitted.find((msg) => msg.type === "dictation_stream_error")).toBeUndefined();
|
||||
} finally {
|
||||
if (original !== undefined) {
|
||||
process.env.OPENAI_API_KEY = original;
|
||||
} else {
|
||||
delete process.env.OPENAI_API_KEY;
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -11,7 +11,7 @@ import { OpenAIRealtimeTranscriptionSession } from "../agent/openai-realtime-tra
|
||||
|
||||
const PCM_CHANNELS = 1;
|
||||
const PCM_BITS_PER_SAMPLE = 16;
|
||||
const DICTATION_PCM_OUTPUT_RATE = 24000;
|
||||
const OPENAI_DICTATION_PCM_OUTPUT_RATE = 24000;
|
||||
const DEFAULT_DICTATION_FINAL_TIMEOUT_MS = 10000;
|
||||
const DICTATION_VAD_GRACE_TIMEOUT_MS = Number.parseInt(
|
||||
process.env.OPENAI_REALTIME_DICTATION_VAD_GRACE_TIMEOUT_MS ?? "2000",
|
||||
@@ -106,7 +106,7 @@ export type RealtimeTranscriptionSession = {
|
||||
};
|
||||
|
||||
export type RealtimeTranscriptionSessionFactory = (params: {
|
||||
apiKey: string;
|
||||
apiKey?: string | null;
|
||||
logger: pino.Logger;
|
||||
transcriptionModel: string;
|
||||
language?: string;
|
||||
@@ -149,6 +149,7 @@ type DictationStreamState = {
|
||||
inputFormat: string;
|
||||
openai: RealtimeTranscriptionSession;
|
||||
inputRate: number;
|
||||
outputRate: number;
|
||||
resampler: Pcm16MonoResampler | null;
|
||||
debugAudioChunks: Buffer[];
|
||||
debugRecordingPath: string | null;
|
||||
@@ -194,6 +195,7 @@ export class DictationStreamManager {
|
||||
private readonly openaiApiKey: string | null;
|
||||
private readonly finalTimeoutMs: number;
|
||||
private readonly createSession: RealtimeTranscriptionSessionFactory;
|
||||
private readonly requiresApiKey: boolean;
|
||||
private readonly streams = new Map<string, DictationStreamState>();
|
||||
|
||||
constructor(params: {
|
||||
@@ -209,10 +211,18 @@ export class DictationStreamManager {
|
||||
this.sessionId = params.sessionId;
|
||||
this.openaiApiKey = params.openaiApiKey ?? null;
|
||||
this.finalTimeoutMs = params.finalTimeoutMs ?? DEFAULT_DICTATION_FINAL_TIMEOUT_MS;
|
||||
this.requiresApiKey = !params.sessionFactory;
|
||||
this.createSession =
|
||||
params.sessionFactory ??
|
||||
((factoryParams) =>
|
||||
new OpenAIRealtimeTranscriptionSession(factoryParams));
|
||||
((factoryParams) => {
|
||||
if (!factoryParams.apiKey) {
|
||||
throw new Error("OPENAI_API_KEY not set");
|
||||
}
|
||||
return new OpenAIRealtimeTranscriptionSession({
|
||||
...factoryParams,
|
||||
apiKey: factoryParams.apiKey,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
public cleanupAll(): void {
|
||||
@@ -225,7 +235,7 @@ export class DictationStreamManager {
|
||||
this.cleanupDictationStream(dictationId);
|
||||
|
||||
const apiKey = this.openaiApiKey ?? process.env.OPENAI_API_KEY;
|
||||
if (!apiKey) {
|
||||
if (this.requiresApiKey && !apiKey) {
|
||||
this.failDictationStream(dictationId, "OPENAI_API_KEY not set", false);
|
||||
return;
|
||||
}
|
||||
@@ -238,7 +248,7 @@ export class DictationStreamManager {
|
||||
|
||||
const turnDetection = parseDictationTurnDetection();
|
||||
const openai = this.createSession({
|
||||
apiKey,
|
||||
apiKey: apiKey ?? null,
|
||||
logger: this.logger.child({ dictationId }),
|
||||
transcriptionModel,
|
||||
language: "en",
|
||||
@@ -338,18 +348,21 @@ export class DictationStreamManager {
|
||||
this.logger
|
||||
);
|
||||
|
||||
const outputRate = this.requiresApiKey ? OPENAI_DICTATION_PCM_OUTPUT_RATE : inputRate;
|
||||
|
||||
this.streams.set(dictationId, {
|
||||
dictationId,
|
||||
sessionId: this.sessionId,
|
||||
inputFormat: format,
|
||||
openai,
|
||||
inputRate,
|
||||
outputRate,
|
||||
resampler:
|
||||
inputRate === DICTATION_PCM_OUTPUT_RATE
|
||||
inputRate === outputRate
|
||||
? null
|
||||
: new Pcm16MonoResampler({
|
||||
inputRate,
|
||||
outputRate: DICTATION_PCM_OUTPUT_RATE,
|
||||
outputRate,
|
||||
}),
|
||||
debugAudioChunks: [],
|
||||
debugRecordingPath: null,
|
||||
@@ -510,7 +523,7 @@ export class DictationStreamManager {
|
||||
const pcmBuffer = Buffer.concat(state.debugAudioChunks);
|
||||
const wavBuffer = convertPCMToWavBuffer(
|
||||
pcmBuffer,
|
||||
DICTATION_PCM_OUTPUT_RATE,
|
||||
state.outputRate,
|
||||
PCM_CHANNELS,
|
||||
PCM_BITS_PER_SAMPLE
|
||||
);
|
||||
@@ -610,7 +623,7 @@ export class DictationStreamManager {
|
||||
} else {
|
||||
const silenceBytes = Math.max(
|
||||
0,
|
||||
Math.round((DICTATION_PCM_OUTPUT_RATE * 2 * DICTATION_FLUSH_SILENCE_MS) / 1000)
|
||||
Math.round((state.outputRate * 2 * DICTATION_FLUSH_SILENCE_MS) / 1000)
|
||||
);
|
||||
if (silenceBytes > 0) {
|
||||
this.logger.debug(
|
||||
|
||||
@@ -17,10 +17,32 @@ const ProviderCredentialsSchema = z
|
||||
})
|
||||
.strict();
|
||||
|
||||
const SherpaOnnxProviderSchema = z
|
||||
.object({
|
||||
modelsDir: z.string().min(1).optional(),
|
||||
autoDownload: z.boolean().optional(),
|
||||
stt: z
|
||||
.object({
|
||||
preset: z.string().min(1).optional(),
|
||||
})
|
||||
.strict()
|
||||
.optional(),
|
||||
tts: z
|
||||
.object({
|
||||
preset: z.string().min(1).optional(),
|
||||
speakerId: z.number().int().optional(),
|
||||
speed: z.number().optional(),
|
||||
})
|
||||
.strict()
|
||||
.optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
const ProvidersSchema = z
|
||||
.object({
|
||||
openai: ProviderCredentialsSchema.optional(),
|
||||
openrouter: ProviderCredentialsSchema.optional(),
|
||||
sherpaOnnx: SherpaOnnxProviderSchema.optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
@@ -28,8 +50,9 @@ const FeatureDictationSchema = z
|
||||
.object({
|
||||
stt: z
|
||||
.object({
|
||||
provider: z.enum(["openai"]).optional(),
|
||||
provider: z.enum(["openai", "sherpa"]).optional(),
|
||||
model: z.string().min(1).optional(),
|
||||
preset: z.string().min(1).optional(),
|
||||
confidenceThreshold: z.number().optional(),
|
||||
})
|
||||
.strict()
|
||||
@@ -48,16 +71,20 @@ const FeatureVoiceModeSchema = z
|
||||
.optional(),
|
||||
stt: z
|
||||
.object({
|
||||
provider: z.enum(["openai"]).optional(),
|
||||
provider: z.enum(["openai", "sherpa"]).optional(),
|
||||
model: z.string().min(1).optional(),
|
||||
preset: z.string().min(1).optional(),
|
||||
})
|
||||
.strict()
|
||||
.optional(),
|
||||
tts: z
|
||||
.object({
|
||||
provider: z.enum(["openai"]).optional(),
|
||||
provider: z.enum(["openai", "sherpa"]).optional(),
|
||||
model: z.enum(["tts-1", "tts-1-hd"]).optional(),
|
||||
voice: z.enum(["alloy", "echo", "fable", "onyx", "nova", "shimmer"]).optional(),
|
||||
preset: z.string().min(1).optional(),
|
||||
speakerId: z.number().int().optional(),
|
||||
speed: z.number().optional(),
|
||||
})
|
||||
.strict()
|
||||
.optional(),
|
||||
|
||||
@@ -33,12 +33,12 @@ import { getSystemPrompt } from "./agent/system-prompt.js";
|
||||
import { getAllTools } from "./agent/llm-openai.js";
|
||||
import { TTSManager } from "./agent/tts-manager.js";
|
||||
import { STTManager } from "./agent/stt-manager.js";
|
||||
import type { OpenAISTT } from "./agent/stt-openai.js";
|
||||
import type { OpenAITTS } from "./agent/tts-openai.js";
|
||||
import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js";
|
||||
import { maybePersistTtsDebugAudio } from "./agent/tts-debug.js";
|
||||
import { isPaseoDictationDebugEnabled } from "./agent/recordings-debug.js";
|
||||
import {
|
||||
DictationStreamManager,
|
||||
type RealtimeTranscriptionSessionFactory,
|
||||
} from "./dictation/dictation-stream-manager.js";
|
||||
import type { VoiceConversationStore } from "./voice-conversation-store.js";
|
||||
import {
|
||||
@@ -325,8 +325,8 @@ export class Session {
|
||||
agentManager: AgentManager,
|
||||
agentStorage: AgentStorage,
|
||||
createAgentMcpTransport: AgentMcpTransportFactory,
|
||||
stt: OpenAISTT | null,
|
||||
tts: OpenAITTS | null,
|
||||
stt: SpeechToTextProvider | null,
|
||||
tts: TextToSpeechProvider | null,
|
||||
terminalManager: TerminalManager | null,
|
||||
voiceConversationStore: VoiceConversationStore,
|
||||
voice?: {
|
||||
@@ -336,6 +336,7 @@ export class Session {
|
||||
dictation?: {
|
||||
openaiApiKey?: string | null;
|
||||
finalTimeoutMs?: number;
|
||||
sessionFactory?: RealtimeTranscriptionSessionFactory;
|
||||
}
|
||||
) {
|
||||
this.clientId = clientId;
|
||||
@@ -368,6 +369,7 @@ export class Session {
|
||||
emit: (msg) => this.emit(msg as unknown as SessionOutboundMessage),
|
||||
openaiApiKey: dictation?.openaiApiKey ?? null,
|
||||
finalTimeoutMs: dictation?.finalTimeoutMs,
|
||||
...(dictation?.sessionFactory ? { sessionFactory: dictation.sessionFactory } : {}),
|
||||
});
|
||||
|
||||
// Initialize agent MCP client asynchronously
|
||||
|
||||
113
packages/server/src/server/speech/audio.ts
Normal file
113
packages/server/src/server/speech/audio.ts
Normal file
@@ -0,0 +1,113 @@
|
||||
export function parsePcm16MonoWav(buffer: Buffer): { sampleRate: number; pcm16: Buffer } {
|
||||
if (buffer.toString("ascii", 0, 4) !== "RIFF" || buffer.toString("ascii", 8, 12) !== "WAVE") {
|
||||
throw new Error("Invalid WAV header");
|
||||
}
|
||||
|
||||
let offset = 12;
|
||||
let fmt:
|
||||
| { audioFormat: number; channels: number; sampleRate: number; bitsPerSample: number }
|
||||
| null = null;
|
||||
let dataChunk: Buffer | null = null;
|
||||
|
||||
while (offset + 8 <= buffer.length) {
|
||||
const id = buffer.toString("ascii", offset, offset + 4);
|
||||
const size = buffer.readUInt32LE(offset + 4);
|
||||
const payloadStart = offset + 8;
|
||||
const payloadEnd = payloadStart + size;
|
||||
if (payloadEnd > buffer.length) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (id === "fmt ") {
|
||||
const audioFormat = buffer.readUInt16LE(payloadStart);
|
||||
const channels = buffer.readUInt16LE(payloadStart + 2);
|
||||
const sampleRate = buffer.readUInt32LE(payloadStart + 4);
|
||||
const bitsPerSample = buffer.readUInt16LE(payloadStart + 14);
|
||||
fmt = { audioFormat, channels, sampleRate, bitsPerSample };
|
||||
} else if (id === "data") {
|
||||
dataChunk = buffer.subarray(payloadStart, payloadEnd);
|
||||
}
|
||||
|
||||
offset = payloadEnd + (size % 2);
|
||||
}
|
||||
|
||||
if (!fmt || !dataChunk) {
|
||||
throw new Error("Missing WAV fmt/data chunks");
|
||||
}
|
||||
if (fmt.audioFormat !== 1) {
|
||||
throw new Error(`Unsupported WAV encoding (audioFormat=${fmt.audioFormat})`);
|
||||
}
|
||||
if (fmt.channels !== 1 || fmt.bitsPerSample !== 16) {
|
||||
throw new Error(
|
||||
`Unexpected WAV format: channels=${fmt.channels} rate=${fmt.sampleRate} bits=${fmt.bitsPerSample}`
|
||||
);
|
||||
}
|
||||
if (dataChunk.length % 2 !== 0) {
|
||||
throw new Error("WAV PCM16 data length must be even");
|
||||
}
|
||||
return { sampleRate: fmt.sampleRate, pcm16: dataChunk };
|
||||
}
|
||||
|
||||
export function parsePcmRateFromFormat(format: string, fallback: number | null = null): number | null {
|
||||
const match = /(?:^|[;,\s])rate\s*=\s*(\d+)(?:$|[;,\s])/i.exec(format);
|
||||
if (!match) {
|
||||
return fallback;
|
||||
}
|
||||
const rate = Number.parseInt(match[1]!, 10);
|
||||
return Number.isFinite(rate) && rate > 0 ? rate : fallback;
|
||||
}
|
||||
|
||||
export function pcm16lePeakAbs(pcm16le: Buffer): number {
|
||||
if (pcm16le.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
if (pcm16le.length % 2 !== 0) {
|
||||
throw new Error(`PCM16 chunk byteLength must be even, got ${pcm16le.length}`);
|
||||
}
|
||||
const samples = new Int16Array(pcm16le.buffer, pcm16le.byteOffset, pcm16le.byteLength / 2);
|
||||
let peak = 0;
|
||||
for (let i = 0; i < samples.length; i += 1) {
|
||||
const v = samples[i]!;
|
||||
const abs = v < 0 ? -v : v;
|
||||
if (abs > peak) {
|
||||
peak = abs;
|
||||
if (peak >= 32767) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return peak;
|
||||
}
|
||||
|
||||
export function pcm16leToFloat32(pcm16le: Buffer, gain: number = 1): Float32Array {
|
||||
if (pcm16le.length % 2 !== 0) {
|
||||
throw new Error(`PCM16 chunk byteLength must be even, got ${pcm16le.length}`);
|
||||
}
|
||||
const int16 = new Int16Array(pcm16le.buffer, pcm16le.byteOffset, pcm16le.byteLength / 2);
|
||||
const out = new Float32Array(int16.length);
|
||||
for (let i = 0; i < int16.length; i += 1) {
|
||||
const v = (int16[i]! / 32768.0) * gain;
|
||||
out[i] = v > 1 ? 1 : v < -1 ? -1 : v;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function float32ToPcm16le(samples: Float32Array): Buffer {
|
||||
const out = new Int16Array(samples.length);
|
||||
for (let i = 0; i < samples.length; i += 1) {
|
||||
const clamped = Math.max(-1, Math.min(1, samples[i]!));
|
||||
out[i] = Math.round(clamped * 32767);
|
||||
}
|
||||
return Buffer.from(out.buffer, out.byteOffset, out.byteLength);
|
||||
}
|
||||
|
||||
export function chunkBuffer(buffer: Buffer, chunkBytes: number): Buffer[] {
|
||||
if (chunkBytes <= 0) {
|
||||
return [buffer];
|
||||
}
|
||||
const out: Buffer[] = [];
|
||||
for (let offset = 0; offset < buffer.length; offset += chunkBytes) {
|
||||
out.push(buffer.subarray(offset, Math.min(buffer.length, offset + chunkBytes)));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
559
packages/server/src/server/speech/pocket/pocket-tts-onnx.ts
Normal file
559
packages/server/src/server/speech/pocket/pocket-tts-onnx.ts
Normal file
@@ -0,0 +1,559 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { Readable } from "node:stream";
|
||||
import type pino from "pino";
|
||||
|
||||
import type { SpeechStreamResult, TextToSpeechProvider } from "../speech-provider.js";
|
||||
import { chunkBuffer, float32ToPcm16le, parsePcm16MonoWav, pcm16leToFloat32 } from "../audio.js";
|
||||
import { Pcm16MonoResampler } from "../../agent/pcm16-resampler.js";
|
||||
|
||||
type OrtModule = typeof import("onnxruntime-node");
|
||||
type OrtSession = import("onnxruntime-node").InferenceSession;
|
||||
type OrtTensor = import("onnxruntime-node").Tensor;
|
||||
|
||||
type SentencePieceProcessor = {
|
||||
encodeIds: (text: string) => number[];
|
||||
load?: (modelPath: string) => unknown;
|
||||
Load?: (modelPath: string) => unknown;
|
||||
};
|
||||
|
||||
function assertFileExists(filePath: string, label: string): void {
|
||||
if (!existsSync(filePath)) {
|
||||
throw new Error(`Missing ${label}: ${filePath}`);
|
||||
}
|
||||
}
|
||||
|
||||
function product(dims: number[]): number {
|
||||
let out = 1;
|
||||
for (const d of dims) out *= d;
|
||||
return out;
|
||||
}
|
||||
|
||||
function normalizeDims(dims: Array<number | string | null | undefined>): number[] {
|
||||
// ONNX metadata can contain dynamic dimensions as strings (e.g. "batch") or -1.
|
||||
// For state tensors we want a valid minimal shape, so coerce unknown/invalid dims to 1.
|
||||
// Preserve explicit 0 dims (some models use empty initial state buffers with shape [0]).
|
||||
return dims.map((d) => {
|
||||
if (typeof d === "number" && Number.isFinite(d)) {
|
||||
if (d === 0) return 0;
|
||||
if (d > 0) return d;
|
||||
return 1;
|
||||
}
|
||||
return 1;
|
||||
});
|
||||
}
|
||||
|
||||
function getSessionInputMeta(
|
||||
session: OrtSession,
|
||||
inputName: string
|
||||
): { type?: string; dims?: Array<number | string | null> } | undefined {
|
||||
const metaAny = (session as any).inputMetadata as unknown;
|
||||
if (Array.isArray(metaAny)) {
|
||||
const entry = metaAny.find((m) => m && typeof m === "object" && (m as any).name === inputName) as any;
|
||||
if (!entry) return undefined;
|
||||
return { type: entry.type, dims: entry.shape };
|
||||
}
|
||||
|
||||
if (metaAny && typeof metaAny === "object" && inputName in (metaAny as any)) {
|
||||
const entry = (metaAny as any)[inputName] as any;
|
||||
return { type: entry?.type, dims: entry?.dimensions ?? entry?.shape };
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function toBigInt64(values: number[]): BigInt64Array {
|
||||
const out = new BigInt64Array(values.length);
|
||||
for (let i = 0; i < values.length; i += 1) {
|
||||
out[i] = BigInt(values[i]!);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function randn(): number {
|
||||
// Box–Muller
|
||||
let u = 0;
|
||||
let v = 0;
|
||||
while (u === 0) u = Math.random();
|
||||
while (v === 0) v = Math.random();
|
||||
return Math.sqrt(-2.0 * Math.log(u)) * Math.cos(2.0 * Math.PI * v);
|
||||
}
|
||||
|
||||
function normalizeTextForPocket(text: string): string {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed) {
|
||||
throw new Error("Cannot synthesize empty text");
|
||||
}
|
||||
let out = trimmed;
|
||||
if (out.length > 0 && /[A-Za-z0-9]$/.test(out)) {
|
||||
out = `${out}.`;
|
||||
}
|
||||
if (out.length > 0 && /[a-z]/.test(out[0]!)) {
|
||||
out = out[0]!.toUpperCase() + out.slice(1);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function loadOrt(): Promise<OrtModule> {
|
||||
return (await import("onnxruntime-node")) as OrtModule;
|
||||
}
|
||||
|
||||
async function loadSentencePiece(tokenizerModelPath: string): Promise<SentencePieceProcessor> {
|
||||
const mod = await import("@sctg/sentencepiece-js");
|
||||
|
||||
const Processor =
|
||||
(mod as any).SentencePieceProcessor ??
|
||||
(mod as any).default?.SentencePieceProcessor ??
|
||||
(mod as any).default;
|
||||
|
||||
if (!Processor) {
|
||||
throw new Error("Failed to load SentencePiece processor from @sctg/sentencepiece-js");
|
||||
}
|
||||
|
||||
const sp: SentencePieceProcessor = new Processor();
|
||||
|
||||
if (typeof sp.load === "function") {
|
||||
await sp.load(tokenizerModelPath);
|
||||
} else if (typeof sp.Load === "function") {
|
||||
sp.Load(tokenizerModelPath);
|
||||
} else {
|
||||
throw new Error("SentencePiece processor does not expose load()/Load()");
|
||||
}
|
||||
|
||||
return sp;
|
||||
}
|
||||
|
||||
function getOrtProviders(ort: OrtModule, device: "auto" | "cpu" | "cuda"): string[] {
|
||||
// NOTE: onnxruntime-node uses backend names like "cpu"/"coreml"/"webgpu" (not "CPUExecutionProvider").
|
||||
if (device === "cpu") return ["cpu"];
|
||||
if (device === "cuda") return ["cuda", "cpu"];
|
||||
// auto
|
||||
// CoreML EP does not support some dynamic/zero-length shapes used by Pocket TTS (e.g. [1, 0, 32]).
|
||||
// Default to CPU to keep behavior predictable across platforms.
|
||||
void ort;
|
||||
return ["cpu"];
|
||||
}
|
||||
|
||||
function createZeroTensorForInput(ort: OrtModule, session: OrtSession, inputName: string): OrtTensor {
|
||||
const meta = getSessionInputMeta(session, inputName);
|
||||
const dims = normalizeDims(meta?.dims ?? []);
|
||||
if (dims.length === 0) {
|
||||
throw new Error(`Missing input metadata shape for ${inputName}`);
|
||||
}
|
||||
|
||||
const type = (meta?.type ?? "float32").toLowerCase();
|
||||
const size = product(dims);
|
||||
|
||||
if (type.includes("int64")) {
|
||||
return new ort.Tensor("int64", new BigInt64Array(size), dims);
|
||||
}
|
||||
if (type.includes("bool")) {
|
||||
return new ort.Tensor("bool", new Uint8Array(size), dims);
|
||||
}
|
||||
return new ort.Tensor("float32", new Float32Array(size), dims);
|
||||
}
|
||||
|
||||
function initState(session: OrtSession, ort: OrtModule): Record<string, OrtTensor> {
|
||||
const out: Record<string, OrtTensor> = {};
|
||||
for (const name of (session as any).inputNames as string[]) {
|
||||
if (name.startsWith("state_")) {
|
||||
out[name] = createZeroTensorForInput(ort, session, name);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function updateStateFromOutputs(
|
||||
state: Record<string, OrtTensor>,
|
||||
outputs: Record<string, OrtTensor>
|
||||
): void {
|
||||
for (const [name, tensor] of Object.entries(outputs)) {
|
||||
if (!name.startsWith("out_state_")) continue;
|
||||
const idx = Number.parseInt(name.replace("out_state_", ""), 10);
|
||||
if (Number.isFinite(idx)) {
|
||||
state[`state_${idx}`] = tensor;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function tensorDataFloat32(t: OrtTensor): Float32Array {
|
||||
const data = (t as any).data;
|
||||
if (data instanceof Float32Array) return data;
|
||||
if (Array.isArray(data)) return Float32Array.from(data as number[]);
|
||||
throw new Error("Unexpected tensor data type (expected Float32Array)");
|
||||
}
|
||||
|
||||
export type PocketTtsOnnxConfig = {
|
||||
modelDir: string;
|
||||
precision?: "int8" | "fp32";
|
||||
device?: "auto" | "cpu" | "cuda";
|
||||
temperature?: number;
|
||||
lsdSteps?: number;
|
||||
maxFrames?: number;
|
||||
framesAfterEos?: number;
|
||||
firstChunkFrames?: number;
|
||||
maxChunkFrames?: number;
|
||||
targetChunkMs?: number;
|
||||
referenceAudioFile?: string;
|
||||
};
|
||||
|
||||
class PocketTtsOnnxEngine {
|
||||
static readonly SAMPLE_RATE = 24000;
|
||||
static readonly SAMPLES_PER_FRAME = 1920;
|
||||
|
||||
private readonly ort: OrtModule;
|
||||
|
||||
private readonly temperature: number;
|
||||
private readonly lsdSteps: number;
|
||||
private readonly maxFrames: number;
|
||||
private readonly framesAfterEos: number;
|
||||
|
||||
private readonly firstChunkFrames: number;
|
||||
private readonly maxChunkFrames: number;
|
||||
|
||||
private readonly tokenizer: SentencePieceProcessor;
|
||||
private readonly textConditioner: OrtSession;
|
||||
private readonly flowLmMain: OrtSession;
|
||||
private readonly flowLmFlow: OrtSession;
|
||||
private readonly mimiDecoder: OrtSession;
|
||||
|
||||
private readonly stBuffers: Array<{ s: OrtTensor; t: OrtTensor }>;
|
||||
private readonly voiceEmbeddings: OrtTensor;
|
||||
|
||||
private constructor(args: {
|
||||
ort: OrtModule;
|
||||
temperature: number;
|
||||
lsdSteps: number;
|
||||
maxFrames: number;
|
||||
framesAfterEos: number;
|
||||
firstChunkFrames: number;
|
||||
maxChunkFrames: number;
|
||||
tokenizer: SentencePieceProcessor;
|
||||
textConditioner: OrtSession;
|
||||
flowLmMain: OrtSession;
|
||||
flowLmFlow: OrtSession;
|
||||
mimiDecoder: OrtSession;
|
||||
stBuffers: Array<{ s: OrtTensor; t: OrtTensor }>;
|
||||
voiceEmbeddings: OrtTensor;
|
||||
}) {
|
||||
this.ort = args.ort;
|
||||
this.temperature = args.temperature;
|
||||
this.lsdSteps = args.lsdSteps;
|
||||
this.maxFrames = args.maxFrames;
|
||||
this.framesAfterEos = args.framesAfterEos;
|
||||
this.firstChunkFrames = args.firstChunkFrames;
|
||||
this.maxChunkFrames = args.maxChunkFrames;
|
||||
this.tokenizer = args.tokenizer;
|
||||
this.textConditioner = args.textConditioner;
|
||||
this.flowLmMain = args.flowLmMain;
|
||||
this.flowLmFlow = args.flowLmFlow;
|
||||
this.mimiDecoder = args.mimiDecoder;
|
||||
this.stBuffers = args.stBuffers;
|
||||
this.voiceEmbeddings = args.voiceEmbeddings;
|
||||
}
|
||||
|
||||
static async create(config: PocketTtsOnnxConfig, logger: pino.Logger): Promise<PocketTtsOnnxEngine> {
|
||||
const log = logger.child({ module: "speech", provider: "pocket-tts", component: "onnx-engine" });
|
||||
|
||||
const modelDir = config.modelDir;
|
||||
const onnxDir = `${modelDir}/onnx`;
|
||||
const precision = config.precision ?? "int8";
|
||||
const device = config.device ?? "auto";
|
||||
const temperature = config.temperature ?? 0.7;
|
||||
const lsdSteps = config.lsdSteps ?? 10;
|
||||
const maxFrames = config.maxFrames ?? 500;
|
||||
const framesAfterEos = config.framesAfterEos ?? 3;
|
||||
const firstChunkFrames = config.firstChunkFrames ?? 2;
|
||||
const maxChunkFrames = config.maxChunkFrames ?? 15;
|
||||
|
||||
const tokenizerPath = `${modelDir}/tokenizer.model`;
|
||||
const referenceAudioFile = config.referenceAudioFile ?? `${modelDir}/reference_sample.wav`;
|
||||
|
||||
const flowMainFile = precision === "int8" ? "flow_lm_main_int8.onnx" : "flow_lm_main.onnx";
|
||||
const flowFlowFile = precision === "int8" ? "flow_lm_flow_int8.onnx" : "flow_lm_flow.onnx";
|
||||
const decoderFile = precision === "int8" ? "mimi_decoder_int8.onnx" : "mimi_decoder.onnx";
|
||||
|
||||
assertFileExists(`${onnxDir}/mimi_encoder.onnx`, "PocketTTS mimi_encoder");
|
||||
assertFileExists(`${onnxDir}/text_conditioner.onnx`, "PocketTTS text_conditioner");
|
||||
assertFileExists(`${onnxDir}/${flowMainFile}`, "PocketTTS flow_lm_main");
|
||||
assertFileExists(`${onnxDir}/${flowFlowFile}`, "PocketTTS flow_lm_flow");
|
||||
assertFileExists(`${onnxDir}/${decoderFile}`, "PocketTTS mimi_decoder");
|
||||
assertFileExists(tokenizerPath, "PocketTTS tokenizer.model");
|
||||
assertFileExists(referenceAudioFile, "PocketTTS reference_sample.wav");
|
||||
|
||||
const ort = await loadOrt();
|
||||
const providers = getOrtProviders(ort, device);
|
||||
|
||||
const [tokenizer, mimiEncoder, textConditioner, flowLmMain, flowLmFlow, mimiDecoder] = await Promise.all([
|
||||
loadSentencePiece(tokenizerPath),
|
||||
ort.InferenceSession.create(`${onnxDir}/mimi_encoder.onnx`, { executionProviders: providers }),
|
||||
ort.InferenceSession.create(`${onnxDir}/text_conditioner.onnx`, { executionProviders: providers }),
|
||||
ort.InferenceSession.create(`${onnxDir}/${flowMainFile}`, { executionProviders: providers }),
|
||||
ort.InferenceSession.create(`${onnxDir}/${flowFlowFile}`, { executionProviders: providers }),
|
||||
ort.InferenceSession.create(`${onnxDir}/${decoderFile}`, { executionProviders: providers }),
|
||||
]);
|
||||
|
||||
// Precompute flow matching time-step buffers.
|
||||
const stBuffers: Array<{ s: OrtTensor; t: OrtTensor }> = [];
|
||||
for (let j = 0; j < lsdSteps; j += 1) {
|
||||
const s = j / lsdSteps;
|
||||
const t = s + 1.0 / lsdSteps;
|
||||
stBuffers.push({
|
||||
s: new ort.Tensor("float32", new Float32Array([s]), [1, 1]),
|
||||
t: new ort.Tensor("float32", new Float32Array([t]), [1, 1]),
|
||||
});
|
||||
}
|
||||
|
||||
// Precompute reference voice embeddings once.
|
||||
const refWav = await readFile(referenceAudioFile);
|
||||
const parsed = parsePcm16MonoWav(refWav);
|
||||
let pcm16 = parsed.pcm16;
|
||||
if (parsed.sampleRate !== PocketTtsOnnxEngine.SAMPLE_RATE) {
|
||||
const resampler = new Pcm16MonoResampler({
|
||||
inputRate: parsed.sampleRate,
|
||||
outputRate: PocketTtsOnnxEngine.SAMPLE_RATE,
|
||||
});
|
||||
pcm16 = resampler.processChunk(pcm16);
|
||||
}
|
||||
const floatAudio = pcm16leToFloat32(pcm16);
|
||||
const audioTensor = new ort.Tensor(
|
||||
"float32",
|
||||
floatAudio,
|
||||
[1, 1, floatAudio.length]
|
||||
);
|
||||
|
||||
const encoded = await mimiEncoder.run({ audio: audioTensor });
|
||||
const firstOutName = (mimiEncoder as any).outputNames?.[0] as string | undefined;
|
||||
const voiceEmb = firstOutName ? (encoded as any)[firstOutName] : (Object.values(encoded)[0] as any);
|
||||
if (!voiceEmb) {
|
||||
throw new Error("PocketTTS mimi_encoder: missing output");
|
||||
}
|
||||
|
||||
log.info(
|
||||
{ precision, device, providers, lsdSteps, temperature },
|
||||
"PocketTTS ONNX initialized"
|
||||
);
|
||||
|
||||
return new PocketTtsOnnxEngine({
|
||||
ort,
|
||||
temperature,
|
||||
lsdSteps,
|
||||
maxFrames,
|
||||
framesAfterEos,
|
||||
firstChunkFrames,
|
||||
maxChunkFrames,
|
||||
tokenizer,
|
||||
textConditioner,
|
||||
flowLmMain,
|
||||
flowLmFlow,
|
||||
mimiDecoder,
|
||||
stBuffers,
|
||||
voiceEmbeddings: voiceEmb,
|
||||
});
|
||||
}
|
||||
|
||||
private tokenize(text: string): OrtTensor {
|
||||
const normalized = normalizeTextForPocket(text);
|
||||
const ids = this.tokenizer.encodeIds(normalized);
|
||||
const data = toBigInt64(ids ?? []);
|
||||
return new this.ort.Tensor("int64", data, [1, data.length]);
|
||||
}
|
||||
|
||||
private async runTextConditioner(tokenIds: OrtTensor): Promise<OrtTensor> {
|
||||
const out = await this.textConditioner.run({ token_ids: tokenIds } as any);
|
||||
const firstOutName = (this.textConditioner as any).outputNames?.[0] as string | undefined;
|
||||
const t = firstOutName ? (out as any)[firstOutName] : (Object.values(out)[0] as any);
|
||||
if (!t) throw new Error("PocketTTS text_conditioner: missing output");
|
||||
return t;
|
||||
}
|
||||
|
||||
private async *runFlowLm(textEmbeddings: OrtTensor): AsyncGenerator<Float32Array> {
|
||||
const ort = this.ort;
|
||||
const state = initState(this.flowLmMain, ort);
|
||||
|
||||
const emptySeq = new ort.Tensor("float32", new Float32Array(0), [1, 0, 32]);
|
||||
const emptyText = new ort.Tensor("float32", new Float32Array(0), [1, 0, 1024]);
|
||||
|
||||
// Voice conditioning pass
|
||||
const resVoice = await this.flowLmMain.run({
|
||||
sequence: emptySeq,
|
||||
text_embeddings: this.voiceEmbeddings,
|
||||
...state,
|
||||
} as any);
|
||||
updateStateFromOutputs(state, resVoice as any);
|
||||
|
||||
// Text conditioning pass
|
||||
const resText = await this.flowLmMain.run({
|
||||
sequence: emptySeq,
|
||||
text_embeddings: textEmbeddings,
|
||||
...state,
|
||||
} as any);
|
||||
updateStateFromOutputs(state, resText as any);
|
||||
|
||||
// Autoregressive generation
|
||||
const curr = new Float32Array(32);
|
||||
curr.fill(Number.NaN);
|
||||
let currTensor = new ort.Tensor("float32", curr, [1, 1, 32]);
|
||||
|
||||
const dt = 1.0 / this.lsdSteps;
|
||||
let eosStep: number | null = null;
|
||||
|
||||
for (let step = 0; step < this.maxFrames; step += 1) {
|
||||
const resStep = await this.flowLmMain.run({
|
||||
sequence: currTensor,
|
||||
text_embeddings: emptyText,
|
||||
...state,
|
||||
} as any);
|
||||
|
||||
const outputNames = (this.flowLmMain as any).outputNames as string[] | undefined;
|
||||
const conditioningName = outputNames?.[0] ?? Object.keys(resStep)[0]!;
|
||||
const eosName = outputNames?.[1] ?? Object.keys(resStep)[1]!;
|
||||
|
||||
const conditioning = (resStep as any)[conditioningName] as OrtTensor;
|
||||
const eos = (resStep as any)[eosName] as OrtTensor;
|
||||
if (!conditioning || !eos) {
|
||||
throw new Error("PocketTTS flow_lm_main: missing conditioning/EOS outputs");
|
||||
}
|
||||
updateStateFromOutputs(state, resStep as any);
|
||||
|
||||
const eosData = tensorDataFloat32(eos);
|
||||
if (eosData[0]! > -4.0 && eosStep === null) {
|
||||
eosStep = step;
|
||||
}
|
||||
if (eosStep !== null && step >= eosStep + this.framesAfterEos) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Flow matching with external Euler loop.
|
||||
const std = this.temperature > 0 ? Math.sqrt(this.temperature) : 0;
|
||||
const x = new Float32Array(32);
|
||||
if (std > 0) {
|
||||
for (let i = 0; i < x.length; i += 1) {
|
||||
x[i] = randn() * std;
|
||||
}
|
||||
}
|
||||
|
||||
for (const st of this.stBuffers) {
|
||||
const xTensor = new ort.Tensor("float32", x, [1, 32]);
|
||||
const flowOut = await this.flowLmFlow.run({
|
||||
c: conditioning,
|
||||
s: st.s,
|
||||
t: st.t,
|
||||
x: xTensor,
|
||||
} as any);
|
||||
const first = (this.flowLmFlow as any).outputNames?.[0] as string | undefined;
|
||||
const flowTensor = first ? (flowOut as any)[first] : (Object.values(flowOut)[0] as any);
|
||||
if (!flowTensor) throw new Error("PocketTTS flow_lm_flow: missing output");
|
||||
const delta = tensorDataFloat32(flowTensor);
|
||||
for (let i = 0; i < x.length; i += 1) {
|
||||
x[i] = x[i]! + delta[i]! * dt;
|
||||
}
|
||||
}
|
||||
|
||||
yield x;
|
||||
currTensor = new ort.Tensor("float32", x, [1, 1, 32]);
|
||||
}
|
||||
}
|
||||
|
||||
private async decodeLatentsChunk(
|
||||
frames: Float32Array[],
|
||||
state: Record<string, OrtTensor>
|
||||
): Promise<Float32Array> {
|
||||
const ort = this.ort;
|
||||
const frameCount = frames.length;
|
||||
const flattened = new Float32Array(frameCount * 32);
|
||||
for (let i = 0; i < frameCount; i += 1) {
|
||||
flattened.set(frames[i]!, i * 32);
|
||||
}
|
||||
const latent = new ort.Tensor("float32", flattened, [1, frameCount, 32]);
|
||||
|
||||
const out = await this.mimiDecoder.run({ latent, ...state } as any);
|
||||
updateStateFromOutputs(state, out as any);
|
||||
|
||||
const firstOutName = (this.mimiDecoder as any).outputNames?.[0] as string | undefined;
|
||||
const audioTensor = firstOutName ? (out as any)[firstOutName] : (Object.values(out)[0] as any);
|
||||
if (!audioTensor) {
|
||||
throw new Error("PocketTTS mimi_decoder: missing audio output");
|
||||
}
|
||||
return tensorDataFloat32(audioTensor);
|
||||
}
|
||||
|
||||
async *streamAudio(text: string): AsyncGenerator<Float32Array> {
|
||||
const tokenIds = this.tokenize(text);
|
||||
const textEmb = await this.runTextConditioner(tokenIds);
|
||||
|
||||
const decoderState = initState(this.mimiDecoder, this.ort);
|
||||
|
||||
const generated: Float32Array[] = [];
|
||||
let decodedFrames = 0;
|
||||
|
||||
for await (const latent of this.runFlowLm(textEmb)) {
|
||||
generated.push(latent);
|
||||
const pending = generated.length - decodedFrames;
|
||||
|
||||
let chunkSize = 0;
|
||||
if (decodedFrames === 0) {
|
||||
if (pending >= this.firstChunkFrames) {
|
||||
chunkSize = this.firstChunkFrames;
|
||||
}
|
||||
} else if (pending >= this.maxChunkFrames) {
|
||||
chunkSize = this.maxChunkFrames;
|
||||
}
|
||||
|
||||
if (chunkSize > 0) {
|
||||
const audio = await this.decodeLatentsChunk(
|
||||
generated.slice(decodedFrames, decodedFrames + chunkSize),
|
||||
decoderState
|
||||
);
|
||||
decodedFrames += chunkSize;
|
||||
yield audio;
|
||||
}
|
||||
}
|
||||
|
||||
if (decodedFrames < generated.length) {
|
||||
const audio = await this.decodeLatentsChunk(generated.slice(decodedFrames), decoderState);
|
||||
yield audio;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class PocketTtsOnnxTTS implements TextToSpeechProvider {
|
||||
private readonly engine: PocketTtsOnnxEngine;
|
||||
private readonly chunkMs: number;
|
||||
private readonly logger: pino.Logger;
|
||||
|
||||
private constructor(engine: PocketTtsOnnxEngine, logger: pino.Logger, chunkMs: number) {
|
||||
this.engine = engine;
|
||||
this.chunkMs = chunkMs;
|
||||
this.logger = logger.child({ module: "speech", provider: "pocket-tts", component: "tts" });
|
||||
}
|
||||
|
||||
static async create(config: PocketTtsOnnxConfig, logger: pino.Logger): Promise<PocketTtsOnnxTTS> {
|
||||
const engine = await PocketTtsOnnxEngine.create(config, logger);
|
||||
const chunkMs = config.targetChunkMs ?? 50;
|
||||
return new PocketTtsOnnxTTS(engine, logger, chunkMs);
|
||||
}
|
||||
|
||||
async synthesizeSpeech(text: string): Promise<SpeechStreamResult> {
|
||||
const start = Date.now();
|
||||
const sampleRate = PocketTtsOnnxEngine.SAMPLE_RATE;
|
||||
|
||||
const pcmChunkBytes = Math.max(2, Math.round((sampleRate * this.chunkMs) / 1000) * 2);
|
||||
|
||||
const iterable = (async function* (engine: PocketTtsOnnxEngine) {
|
||||
for await (const floatChunk of engine.streamAudio(text)) {
|
||||
const pcm = float32ToPcm16le(floatChunk);
|
||||
for (const chunk of chunkBuffer(pcm, pcmChunkBytes)) {
|
||||
yield chunk;
|
||||
}
|
||||
}
|
||||
})(this.engine);
|
||||
|
||||
this.logger.debug({ ms: Date.now() - start, textLength: text.length }, "PocketTTS stream ready");
|
||||
|
||||
return {
|
||||
stream: Readable.from(iterable),
|
||||
format: `pcm;rate=${sampleRate}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
128
packages/server/src/server/speech/sherpa/model-catalog.ts
Normal file
128
packages/server/src/server/speech/sherpa/model-catalog.ts
Normal file
@@ -0,0 +1,128 @@
|
||||
export type SherpaOnnxModelKind = "stt-online" | "stt-offline" | "tts";
|
||||
|
||||
export type SherpaOnnxModelId =
|
||||
| "zipformer-bilingual-zh-en-2023-02-20"
|
||||
| "paraformer-bilingual-zh-en"
|
||||
| "parakeet-tdt-0.6b-v3-int8"
|
||||
| "kitten-nano-en-v0_1-fp16"
|
||||
| "kokoro-en-v0_19"
|
||||
| "pocket-tts-onnx-int8";
|
||||
|
||||
export type SherpaOnnxModelSpec = {
|
||||
id: SherpaOnnxModelId;
|
||||
kind: SherpaOnnxModelKind;
|
||||
archiveUrl?: string;
|
||||
downloadFiles?: Array<{ url: string; relPath: string }>;
|
||||
extractedDir: string;
|
||||
requiredFiles: string[];
|
||||
description: string;
|
||||
};
|
||||
|
||||
export const SHERPA_ONNX_MODEL_CATALOG: Record<SherpaOnnxModelId, SherpaOnnxModelSpec> = {
|
||||
"zipformer-bilingual-zh-en-2023-02-20": {
|
||||
id: "zipformer-bilingual-zh-en-2023-02-20",
|
||||
kind: "stt-online",
|
||||
archiveUrl:
|
||||
"https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2",
|
||||
extractedDir: "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20",
|
||||
requiredFiles: [
|
||||
"encoder-epoch-99-avg-1.onnx",
|
||||
"decoder-epoch-99-avg-1.onnx",
|
||||
"joiner-epoch-99-avg-1.onnx",
|
||||
"tokens.txt",
|
||||
],
|
||||
description: "Streaming Zipformer transducer (fast, good accuracy).",
|
||||
},
|
||||
"paraformer-bilingual-zh-en": {
|
||||
id: "paraformer-bilingual-zh-en",
|
||||
kind: "stt-online",
|
||||
archiveUrl:
|
||||
"https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2",
|
||||
extractedDir: "sherpa-onnx-streaming-paraformer-bilingual-zh-en",
|
||||
requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "tokens.txt"],
|
||||
description: "Streaming Paraformer (often strong accuracy; heavier).",
|
||||
},
|
||||
"parakeet-tdt-0.6b-v3-int8": {
|
||||
id: "parakeet-tdt-0.6b-v3-int8",
|
||||
kind: "stt-offline",
|
||||
archiveUrl:
|
||||
"https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2",
|
||||
extractedDir: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8",
|
||||
requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", "tokens.txt"],
|
||||
description: "NVIDIA Parakeet TDT v3 (offline NeMo transducer, multilingual).",
|
||||
},
|
||||
"kitten-nano-en-v0_1-fp16": {
|
||||
id: "kitten-nano-en-v0_1-fp16",
|
||||
kind: "tts",
|
||||
archiveUrl:
|
||||
"https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2",
|
||||
extractedDir: "kitten-nano-en-v0_1-fp16",
|
||||
requiredFiles: ["model.fp16.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"],
|
||||
description: "KittenTTS (small, fast English TTS).",
|
||||
},
|
||||
"kokoro-en-v0_19": {
|
||||
id: "kokoro-en-v0_19",
|
||||
kind: "tts",
|
||||
archiveUrl: "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2",
|
||||
extractedDir: "kokoro-en-v0_19",
|
||||
requiredFiles: ["model.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"],
|
||||
description: "Kokoro TTS (higher quality; larger).",
|
||||
},
|
||||
"pocket-tts-onnx-int8": {
|
||||
id: "pocket-tts-onnx-int8",
|
||||
kind: "tts",
|
||||
extractedDir: "pocket-tts-onnx-int8",
|
||||
downloadFiles: [
|
||||
{
|
||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/mimi_encoder.onnx",
|
||||
relPath: "onnx/mimi_encoder.onnx",
|
||||
},
|
||||
{
|
||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/text_conditioner.onnx",
|
||||
relPath: "onnx/text_conditioner.onnx",
|
||||
},
|
||||
{
|
||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/flow_lm_main_int8.onnx",
|
||||
relPath: "onnx/flow_lm_main_int8.onnx",
|
||||
},
|
||||
{
|
||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/flow_lm_flow_int8.onnx",
|
||||
relPath: "onnx/flow_lm_flow_int8.onnx",
|
||||
},
|
||||
{
|
||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/mimi_decoder_int8.onnx",
|
||||
relPath: "onnx/mimi_decoder_int8.onnx",
|
||||
},
|
||||
{
|
||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/tokenizer.model",
|
||||
relPath: "tokenizer.model",
|
||||
},
|
||||
{
|
||||
url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/reference_sample.wav",
|
||||
relPath: "reference_sample.wav",
|
||||
},
|
||||
],
|
||||
requiredFiles: [
|
||||
"onnx/mimi_encoder.onnx",
|
||||
"onnx/text_conditioner.onnx",
|
||||
"onnx/flow_lm_main_int8.onnx",
|
||||
"onnx/flow_lm_flow_int8.onnx",
|
||||
"onnx/mimi_decoder_int8.onnx",
|
||||
"tokenizer.model",
|
||||
"reference_sample.wav",
|
||||
],
|
||||
description: "Pocket TTS ONNX (INT8) with streaming decode support (via onnxruntime).",
|
||||
},
|
||||
};
|
||||
|
||||
export function listSherpaOnnxModels(): SherpaOnnxModelSpec[] {
|
||||
return Object.values(SHERPA_ONNX_MODEL_CATALOG);
|
||||
}
|
||||
|
||||
export function getSherpaOnnxModelSpec(id: SherpaOnnxModelId): SherpaOnnxModelSpec {
|
||||
const spec = SHERPA_ONNX_MODEL_CATALOG[id];
|
||||
if (!spec) {
|
||||
throw new Error(`Unknown sherpa-onnx model id: ${id}`);
|
||||
}
|
||||
return spec;
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import pino from "pino";
|
||||
|
||||
import { ensureSherpaOnnxModel, getSherpaOnnxModelDir } from "./model-downloader.js";
|
||||
|
||||
function makeTmpDir(): string {
|
||||
return mkdtempSync(path.join(tmpdir(), "paseo-speech-models-"));
|
||||
}
|
||||
|
||||
const logger = pino({ level: "silent" });
|
||||
|
||||
describe("sherpa model downloader", () => {
|
||||
test("getSherpaOnnxModelDir maps modelId to extractedDir", () => {
|
||||
const modelsDir = "/tmp/models";
|
||||
expect(getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v3-int8")).toContain(
|
||||
"sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8"
|
||||
);
|
||||
expect(getSherpaOnnxModelDir(modelsDir, "pocket-tts-onnx-int8")).toContain(
|
||||
"pocket-tts-onnx-int8"
|
||||
);
|
||||
});
|
||||
|
||||
test("ensureSherpaOnnxModel succeeds without downloading when files exist", async () => {
|
||||
const modelsDir = makeTmpDir();
|
||||
const modelDir = getSherpaOnnxModelDir(modelsDir, "kitten-nano-en-v0_1-fp16");
|
||||
|
||||
mkdirSync(path.join(modelDir, "espeak-ng-data"), { recursive: true });
|
||||
writeFileSync(path.join(modelDir, "model.fp16.onnx"), "x");
|
||||
writeFileSync(path.join(modelDir, "voices.bin"), "x");
|
||||
writeFileSync(path.join(modelDir, "tokens.txt"), "x");
|
||||
|
||||
const out = await ensureSherpaOnnxModel({
|
||||
modelsDir,
|
||||
modelId: "kitten-nano-en-v0_1-fp16",
|
||||
autoDownload: false,
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(out).toBe(modelDir);
|
||||
});
|
||||
|
||||
test("ensureSherpaOnnxModel throws when missing and autoDownload is false", async () => {
|
||||
const modelsDir = makeTmpDir();
|
||||
await expect(
|
||||
ensureSherpaOnnxModel({
|
||||
modelsDir,
|
||||
modelId: "zipformer-bilingual-zh-en-2023-02-20",
|
||||
autoDownload: false,
|
||||
logger,
|
||||
})
|
||||
).rejects.toThrow(/auto-download/i);
|
||||
});
|
||||
});
|
||||
190
packages/server/src/server/speech/sherpa/model-downloader.ts
Normal file
190
packages/server/src/server/speech/sherpa/model-downloader.ts
Normal file
@@ -0,0 +1,190 @@
|
||||
import { createWriteStream } from "node:fs";
|
||||
import { mkdir, rename, rm, stat } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { Readable } from "node:stream";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
import { spawn } from "node:child_process";
|
||||
import type pino from "pino";
|
||||
|
||||
import { getSherpaOnnxModelSpec, type SherpaOnnxModelId } from "./model-catalog.js";
|
||||
|
||||
export type EnsureSherpaOnnxModelOptions = {
|
||||
modelsDir: string;
|
||||
modelId: SherpaOnnxModelId;
|
||||
autoDownload: boolean;
|
||||
logger: pino.Logger;
|
||||
};
|
||||
|
||||
export function getSherpaOnnxModelDir(modelsDir: string, modelId: SherpaOnnxModelId): string {
|
||||
const spec = getSherpaOnnxModelSpec(modelId);
|
||||
return path.join(modelsDir, spec.extractedDir);
|
||||
}
|
||||
|
||||
async function hasRequiredFiles(modelDir: string, requiredFiles: string[]): Promise<boolean> {
|
||||
for (const rel of requiredFiles) {
|
||||
const abs = path.join(modelDir, rel);
|
||||
try {
|
||||
const s = await stat(abs);
|
||||
if (s.isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
if (s.isFile() && s.size > 0) {
|
||||
continue;
|
||||
}
|
||||
return false;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
async function downloadToFile(url: string, outputPath: string, logger: pino.Logger): Promise<void> {
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) {
|
||||
throw new Error(`Failed to download ${url}: ${res.status} ${res.statusText}`);
|
||||
}
|
||||
if (!res.body) {
|
||||
throw new Error(`Failed to download ${url}: missing response body`);
|
||||
}
|
||||
|
||||
const tmpPath = `${outputPath}.tmp-${Date.now()}`;
|
||||
await mkdir(path.dirname(outputPath), { recursive: true });
|
||||
|
||||
const total = Number(res.headers.get("content-length") ?? "0");
|
||||
let downloaded = 0;
|
||||
let lastLoggedBucket = -1;
|
||||
|
||||
const nodeStream = Readable.fromWeb(res.body as any).on("data", (chunk: Buffer) => {
|
||||
downloaded += chunk.length;
|
||||
if (total > 0) {
|
||||
const pct = Math.floor((downloaded / total) * 100);
|
||||
const bucket = Math.min(100, Math.floor(pct / 10) * 10);
|
||||
if (bucket >= 0 && bucket <= 100 && bucket !== lastLoggedBucket) {
|
||||
lastLoggedBucket = bucket;
|
||||
logger.info({ pct: bucket, downloaded, total }, "Downloading model artifact");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
await pipeline(nodeStream, createWriteStream(tmpPath));
|
||||
await rename(tmpPath, outputPath);
|
||||
}
|
||||
|
||||
async function extractTarArchive(archivePath: string, destDir: string): Promise<void> {
|
||||
await mkdir(destDir, { recursive: true });
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const child = spawn("tar", ["xf", archivePath, "-C", destDir], { stdio: "inherit" });
|
||||
child.on("error", reject);
|
||||
child.on("exit", (code) => {
|
||||
if (code === 0) resolve();
|
||||
else reject(new Error(`tar exited with code ${code}`));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function isNonEmptyFile(filePath: string): Promise<boolean> {
|
||||
try {
|
||||
const s = await stat(filePath);
|
||||
return s.isFile() && s.size > 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function ensureSherpaOnnxModel(options: EnsureSherpaOnnxModelOptions): Promise<string> {
|
||||
const logger = options.logger.child({
|
||||
module: "speech",
|
||||
provider: "sherpa-onnx",
|
||||
component: "model-downloader",
|
||||
modelId: options.modelId,
|
||||
});
|
||||
|
||||
const spec = getSherpaOnnxModelSpec(options.modelId);
|
||||
const modelDir = path.join(options.modelsDir, spec.extractedDir);
|
||||
if (await hasRequiredFiles(modelDir, spec.requiredFiles)) {
|
||||
return modelDir;
|
||||
}
|
||||
|
||||
if (!options.autoDownload) {
|
||||
throw new Error(
|
||||
`Missing sherpa-onnx model files for ${options.modelId} in ${modelDir}. ` +
|
||||
`Set PASEO_SHERPA_ONNX_AUTO_DOWNLOAD=1 to auto-download.`
|
||||
);
|
||||
}
|
||||
|
||||
if (spec.archiveUrl) {
|
||||
logger.info({ modelsDir: options.modelsDir, url: spec.archiveUrl }, "Model files missing; downloading");
|
||||
|
||||
const downloadsDir = path.join(options.modelsDir, ".downloads");
|
||||
const archiveFilename = path.basename(new URL(spec.archiveUrl).pathname);
|
||||
const archivePath = path.join(downloadsDir, archiveFilename);
|
||||
|
||||
if (!(await isNonEmptyFile(archivePath))) {
|
||||
await downloadToFile(spec.archiveUrl, archivePath, logger);
|
||||
} else {
|
||||
logger.info({ archivePath }, "Using cached archive");
|
||||
}
|
||||
|
||||
await extractTarArchive(archivePath, options.modelsDir);
|
||||
|
||||
if (!(await hasRequiredFiles(modelDir, spec.requiredFiles))) {
|
||||
throw new Error(
|
||||
`Downloaded and extracted ${archiveFilename}, but required files are still missing in ${modelDir}.`
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
await rm(archivePath, { force: true });
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
|
||||
logger.info({ modelDir }, "Model ready");
|
||||
return modelDir;
|
||||
}
|
||||
|
||||
if (spec.downloadFiles && spec.downloadFiles.length > 0) {
|
||||
logger.info({ modelsDir: options.modelsDir, fileCount: spec.downloadFiles.length }, "Model files missing; downloading");
|
||||
await mkdir(modelDir, { recursive: true });
|
||||
|
||||
for (const file of spec.downloadFiles) {
|
||||
const dst = path.join(modelDir, file.relPath);
|
||||
if (await isNonEmptyFile(dst)) {
|
||||
continue;
|
||||
}
|
||||
await downloadToFile(file.url, dst, logger);
|
||||
}
|
||||
|
||||
if (!(await hasRequiredFiles(modelDir, spec.requiredFiles))) {
|
||||
throw new Error(
|
||||
`Downloaded files for ${options.modelId}, but required files are still missing in ${modelDir}.`
|
||||
);
|
||||
}
|
||||
|
||||
logger.info({ modelDir }, "Model ready");
|
||||
return modelDir;
|
||||
}
|
||||
|
||||
throw new Error(`Model spec for ${options.modelId} has no archiveUrl or downloadFiles`);
|
||||
}
|
||||
|
||||
export async function ensureSherpaOnnxModels(options: {
|
||||
modelsDir: string;
|
||||
modelIds: SherpaOnnxModelId[];
|
||||
autoDownload: boolean;
|
||||
logger: pino.Logger;
|
||||
}): Promise<Record<SherpaOnnxModelId, string>> {
|
||||
const uniq = Array.from(new Set(options.modelIds));
|
||||
const out: Partial<Record<SherpaOnnxModelId, string>> = {};
|
||||
for (const id of uniq) {
|
||||
out[id] = await ensureSherpaOnnxModel({
|
||||
modelsDir: options.modelsDir,
|
||||
modelId: id,
|
||||
autoDownload: options.autoDownload,
|
||||
logger: options.logger,
|
||||
});
|
||||
}
|
||||
return out as Record<SherpaOnnxModelId, string>;
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import type pino from "pino";
|
||||
|
||||
import { loadSherpaOnnxNode } from "./sherpa-onnx-node-loader.js";
|
||||
|
||||
function assertFileExists(filePath: string, label: string): void {
|
||||
if (!existsSync(filePath)) {
|
||||
throw new Error(`Missing ${label}: ${filePath}`);
|
||||
}
|
||||
}
|
||||
|
||||
export type SherpaOfflineRecognizerModel = {
|
||||
kind: "nemo_transducer";
|
||||
encoder: string;
|
||||
decoder: string;
|
||||
joiner: string;
|
||||
tokens: string;
|
||||
};
|
||||
|
||||
export type SherpaOfflineRecognizerConfig = {
|
||||
model: SherpaOfflineRecognizerModel;
|
||||
numThreads?: number;
|
||||
provider?: "cpu";
|
||||
debug?: 0 | 1;
|
||||
sampleRate?: number;
|
||||
featureDim?: number;
|
||||
decodingMethod?: "greedy_search";
|
||||
maxActivePaths?: number;
|
||||
};
|
||||
|
||||
export class SherpaOfflineRecognizerEngine {
|
||||
public readonly recognizer: any;
|
||||
public readonly sampleRate: number;
|
||||
private readonly logger: pino.Logger;
|
||||
|
||||
constructor(config: SherpaOfflineRecognizerConfig, logger: pino.Logger) {
|
||||
this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "offline-recognizer" });
|
||||
|
||||
assertFileExists(config.model.encoder, "offline encoder");
|
||||
assertFileExists(config.model.decoder, "offline decoder");
|
||||
assertFileExists(config.model.joiner, "offline joiner");
|
||||
assertFileExists(config.model.tokens, "tokens");
|
||||
|
||||
const sherpa = loadSherpaOnnxNode();
|
||||
|
||||
const recognizerConfig = {
|
||||
featConfig: {
|
||||
sampleRate: config.sampleRate ?? 16000,
|
||||
featureDim: config.featureDim ?? 80,
|
||||
},
|
||||
modelConfig: {
|
||||
transducer: {
|
||||
encoder: config.model.encoder,
|
||||
decoder: config.model.decoder,
|
||||
joiner: config.model.joiner,
|
||||
},
|
||||
tokens: config.model.tokens,
|
||||
modelType: "nemo_transducer",
|
||||
numThreads: config.numThreads ?? 1,
|
||||
provider: config.provider ?? "cpu",
|
||||
debug: config.debug ?? 0,
|
||||
},
|
||||
decodingMethod: config.decodingMethod ?? "greedy_search",
|
||||
maxActivePaths: config.maxActivePaths ?? 4,
|
||||
};
|
||||
|
||||
this.recognizer = new sherpa.OfflineRecognizer(recognizerConfig);
|
||||
const sr = this.recognizer?.config?.featConfig?.sampleRate;
|
||||
this.sampleRate = typeof sr === "number" && Number.isFinite(sr) && sr > 0
|
||||
? sr
|
||||
: recognizerConfig.featConfig.sampleRate;
|
||||
|
||||
this.logger.info(
|
||||
{ sampleRate: this.sampleRate, numThreads: recognizerConfig.modelConfig.numThreads },
|
||||
"Sherpa offline recognizer initialized"
|
||||
);
|
||||
}
|
||||
|
||||
createStream(): any {
|
||||
return this.recognizer.createStream();
|
||||
}
|
||||
|
||||
acceptWaveform(stream: any, sampleRate: number, samples: Float32Array): void {
|
||||
if (!stream || typeof stream.acceptWaveform !== "function") {
|
||||
throw new Error("Unexpected sherpa offline stream: missing acceptWaveform()");
|
||||
}
|
||||
|
||||
// sherpa-onnx-node expects: acceptWaveform({ samples, sampleRate })
|
||||
// sherpa-onnx (WASM) expects: acceptWaveform(sampleRate, samples)
|
||||
if (stream.acceptWaveform.length <= 1) {
|
||||
stream.acceptWaveform({ samples, sampleRate });
|
||||
} else {
|
||||
stream.acceptWaveform(sampleRate, samples);
|
||||
}
|
||||
}
|
||||
|
||||
free(): void {
|
||||
try {
|
||||
this.recognizer?.free?.();
|
||||
} catch (err) {
|
||||
this.logger.warn({ err }, "Failed to free sherpa offline recognizer");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import type pino from "pino";
|
||||
|
||||
import { loadSherpaOnnx } from "./sherpa-onnx-loader.js";
|
||||
|
||||
export type SherpaOnlineRecognizerModel =
|
||||
| {
|
||||
kind: "transducer";
|
||||
encoder: string;
|
||||
decoder: string;
|
||||
joiner: string;
|
||||
tokens: string;
|
||||
modelType?: string;
|
||||
}
|
||||
| {
|
||||
kind: "paraformer";
|
||||
encoder: string;
|
||||
decoder: string;
|
||||
tokens: string;
|
||||
};
|
||||
|
||||
export type SherpaOnlineRecognizerConfig = {
|
||||
model: SherpaOnlineRecognizerModel;
|
||||
numThreads?: number;
|
||||
provider?: "cpu";
|
||||
debug?: 0 | 1;
|
||||
sampleRate?: number;
|
||||
featureDim?: number;
|
||||
decodingMethod?: "greedy_search";
|
||||
maxActivePaths?: number;
|
||||
enableEndpoint?: 0 | 1;
|
||||
rule1MinTrailingSilence?: number;
|
||||
rule2MinTrailingSilence?: number;
|
||||
rule3MinUtteranceLength?: number;
|
||||
};
|
||||
|
||||
function assertFileExists(filePath: string, label: string): void {
|
||||
if (!existsSync(filePath)) {
|
||||
throw new Error(`Missing ${label}: ${filePath}`);
|
||||
}
|
||||
}
|
||||
|
||||
export class SherpaOnlineRecognizerEngine {
|
||||
public readonly recognizer: any;
|
||||
public readonly sampleRate: number;
|
||||
private readonly logger: pino.Logger;
|
||||
|
||||
constructor(config: SherpaOnlineRecognizerConfig, logger: pino.Logger) {
|
||||
this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "online-recognizer" });
|
||||
|
||||
const { model } = config;
|
||||
if (model.kind === "transducer") {
|
||||
assertFileExists(model.encoder, "transducer encoder");
|
||||
assertFileExists(model.decoder, "transducer decoder");
|
||||
assertFileExists(model.joiner, "transducer joiner");
|
||||
assertFileExists(model.tokens, "tokens");
|
||||
} else {
|
||||
assertFileExists(model.encoder, "paraformer encoder");
|
||||
assertFileExists(model.decoder, "paraformer decoder");
|
||||
assertFileExists(model.tokens, "tokens");
|
||||
}
|
||||
|
||||
const sherpa = loadSherpaOnnx();
|
||||
const modelConfig =
|
||||
model.kind === "transducer"
|
||||
? {
|
||||
transducer: {
|
||||
encoder: model.encoder,
|
||||
decoder: model.decoder,
|
||||
joiner: model.joiner,
|
||||
},
|
||||
tokens: model.tokens,
|
||||
modelType: model.modelType ?? "zipformer",
|
||||
}
|
||||
: {
|
||||
paraformer: {
|
||||
encoder: model.encoder,
|
||||
decoder: model.decoder,
|
||||
},
|
||||
tokens: model.tokens,
|
||||
};
|
||||
|
||||
const featConfig = {
|
||||
sampleRate: config.sampleRate ?? 16000,
|
||||
featureDim: config.featureDim ?? 80,
|
||||
};
|
||||
|
||||
const recognizerConfig = {
|
||||
featConfig,
|
||||
modelConfig: {
|
||||
...modelConfig,
|
||||
// NOTE: In the WASM-backed `sherpa-onnx` npm package, online recognizers
|
||||
// error when `numThreads > 1`. Keep the default conservative.
|
||||
numThreads: config.numThreads ?? 1,
|
||||
provider: config.provider ?? "cpu",
|
||||
debug: config.debug ?? 0,
|
||||
},
|
||||
decodingMethod: config.decodingMethod ?? "greedy_search",
|
||||
maxActivePaths: config.maxActivePaths ?? 4,
|
||||
enableEndpoint: config.enableEndpoint ?? 0,
|
||||
rule1MinTrailingSilence: config.rule1MinTrailingSilence ?? 2.4,
|
||||
rule2MinTrailingSilence: config.rule2MinTrailingSilence ?? 1.2,
|
||||
rule3MinUtteranceLength: config.rule3MinUtteranceLength ?? 20,
|
||||
};
|
||||
|
||||
this.recognizer = sherpa.createOnlineRecognizer(recognizerConfig);
|
||||
const sr = this.recognizer?.config?.featConfig?.sampleRate;
|
||||
this.sampleRate = typeof sr === "number" && Number.isFinite(sr) && sr > 0 ? sr : featConfig.sampleRate;
|
||||
|
||||
this.logger.info(
|
||||
{ sampleRate: this.sampleRate, modelKind: model.kind, numThreads: config.numThreads ?? 2 },
|
||||
"Sherpa online recognizer initialized"
|
||||
);
|
||||
}
|
||||
|
||||
createStream(): any {
|
||||
return this.recognizer.createStream();
|
||||
}
|
||||
|
||||
free(): void {
|
||||
try {
|
||||
this.recognizer?.free?.();
|
||||
} catch (err) {
|
||||
this.logger.warn({ err }, "Failed to free sherpa recognizer");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { createRequire } from "node:module";
|
||||
|
||||
export type SherpaOnnxModule = {
|
||||
createOnlineRecognizer: (config: any) => any;
|
||||
createOfflineRecognizer: (config: any) => any;
|
||||
createOfflineTts: (config: any) => any;
|
||||
};
|
||||
|
||||
let cached: SherpaOnnxModule | null = null;
|
||||
|
||||
export function loadSherpaOnnx(): SherpaOnnxModule {
|
||||
if (cached) {
|
||||
return cached;
|
||||
}
|
||||
const require = createRequire(import.meta.url);
|
||||
cached = require("sherpa-onnx") as SherpaOnnxModule;
|
||||
return cached;
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import { createRequire } from "node:module";
|
||||
import path from "node:path";
|
||||
|
||||
export type SherpaOnnxNodeModule = {
|
||||
OfflineRecognizer: new (config: any) => any;
|
||||
OnlineRecognizer?: new (config: any) => any;
|
||||
OfflineTts?: new (config: any) => any;
|
||||
};
|
||||
|
||||
let cached: SherpaOnnxNodeModule | null = null;
|
||||
|
||||
function platformArch(): string {
|
||||
const platform = process.platform === "win32" ? "win" : process.platform;
|
||||
return `${platform}-${process.arch}`;
|
||||
}
|
||||
|
||||
function prependLibraryPath(envKey: "DYLD_LIBRARY_PATH" | "LD_LIBRARY_PATH", dir: string): void {
|
||||
const current = process.env[envKey] ?? "";
|
||||
const parts = current.split(":").filter(Boolean);
|
||||
if (parts.includes(dir)) {
|
||||
return;
|
||||
}
|
||||
process.env[envKey] = [dir, ...parts].join(":");
|
||||
}
|
||||
|
||||
export function loadSherpaOnnxNode(): SherpaOnnxNodeModule {
|
||||
if (cached) {
|
||||
return cached;
|
||||
}
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
// sherpa-onnx-node depends on a platform-specific package (e.g. sherpa-onnx-darwin-arm64)
|
||||
// that contains the native addon + its shared libraries. On macOS/Linux we need to ensure
|
||||
// the appropriate library path env var includes that directory before requiring the addon.
|
||||
const arch = platformArch();
|
||||
const pkgName = `sherpa-onnx-${arch}`;
|
||||
|
||||
try {
|
||||
const pkgJson = require.resolve(`${pkgName}/package.json`);
|
||||
const pkgDir = path.dirname(pkgJson);
|
||||
if (process.platform === "darwin") {
|
||||
prependLibraryPath("DYLD_LIBRARY_PATH", pkgDir);
|
||||
} else if (process.platform === "linux") {
|
||||
prependLibraryPath("LD_LIBRARY_PATH", pkgDir);
|
||||
}
|
||||
} catch {
|
||||
// Best effort - if the platform package isn't present, require() below will throw a useful error.
|
||||
}
|
||||
|
||||
cached = require("sherpa-onnx-node") as SherpaOnnxNodeModule;
|
||||
return cached;
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
|
||||
import type { RealtimeTranscriptionSession } from "../../dictation/dictation-stream-manager.js";
|
||||
import { pcm16lePeakAbs, pcm16leToFloat32 } from "../audio.js";
|
||||
import { SherpaOfflineRecognizerEngine } from "./sherpa-offline-recognizer.js";
|
||||
|
||||
export class SherpaParakeetRealtimeTranscriptionSession
|
||||
extends EventEmitter
|
||||
implements RealtimeTranscriptionSession
|
||||
{
|
||||
private readonly engine: SherpaOfflineRecognizerEngine;
|
||||
private connected = false;
|
||||
|
||||
private currentItemId: string | null = null;
|
||||
private previousItemId: string | null = null;
|
||||
private lastPartialText = "";
|
||||
|
||||
private pcm16 = Buffer.alloc(0);
|
||||
private lastDecodeAt = 0;
|
||||
private decoding = false;
|
||||
private pendingDecode = false;
|
||||
private readonly minDecodeIntervalMs: number;
|
||||
|
||||
constructor(params: { engine: SherpaOfflineRecognizerEngine; minDecodeIntervalMs?: number }) {
|
||||
super();
|
||||
this.engine = params.engine;
|
||||
this.minDecodeIntervalMs = params.minDecodeIntervalMs ?? 350;
|
||||
}
|
||||
|
||||
async connect(): Promise<void> {
|
||||
if (this.connected) {
|
||||
return;
|
||||
}
|
||||
this.currentItemId = uuidv4();
|
||||
this.connected = true;
|
||||
}
|
||||
|
||||
appendPcm16Base64(base64Audio: string): void {
|
||||
if (!this.connected || !this.currentItemId) {
|
||||
this.emit("error", new Error("Parakeet realtime session not connected"));
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const chunk = Buffer.from(base64Audio, "base64");
|
||||
this.pcm16 = this.pcm16.length === 0 ? chunk : Buffer.concat([this.pcm16, chunk]);
|
||||
void this.maybeDecode(false);
|
||||
} catch (err) {
|
||||
this.emit("error", err instanceof Error ? err : new Error(String(err)));
|
||||
}
|
||||
}
|
||||
|
||||
commit(): void {
|
||||
if (!this.connected || !this.currentItemId) {
|
||||
this.emit("error", new Error("Parakeet realtime session not connected"));
|
||||
return;
|
||||
}
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
await this.maybeDecode(true);
|
||||
const finalText = this.lastPartialText;
|
||||
const itemId = this.currentItemId!;
|
||||
const previousItemId = this.previousItemId;
|
||||
|
||||
this.emit("committed", { itemId, previousItemId });
|
||||
this.emit("transcript", { itemId, transcript: finalText, isFinal: true });
|
||||
|
||||
this.previousItemId = itemId;
|
||||
this.currentItemId = uuidv4();
|
||||
this.lastPartialText = "";
|
||||
this.pcm16 = Buffer.alloc(0);
|
||||
} catch (err) {
|
||||
this.emit("error", err instanceof Error ? err : new Error(String(err)));
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
if (!this.connected) {
|
||||
return;
|
||||
}
|
||||
this.pcm16 = Buffer.alloc(0);
|
||||
this.currentItemId = uuidv4();
|
||||
this.lastPartialText = "";
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.connected = false;
|
||||
this.currentItemId = null;
|
||||
this.pcm16 = Buffer.alloc(0);
|
||||
}
|
||||
|
||||
private async maybeDecode(force: boolean): Promise<void> {
|
||||
if (!this.connected || !this.currentItemId) {
|
||||
return;
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
if (!force && now - this.lastDecodeAt < this.minDecodeIntervalMs) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.decoding) {
|
||||
this.pendingDecode = true;
|
||||
return;
|
||||
}
|
||||
|
||||
this.decoding = true;
|
||||
try {
|
||||
const text = await this.decodeNow();
|
||||
this.lastDecodeAt = Date.now();
|
||||
if (text !== this.lastPartialText) {
|
||||
this.lastPartialText = text;
|
||||
this.emit("transcript", { itemId: this.currentItemId, transcript: text, isFinal: false });
|
||||
}
|
||||
} finally {
|
||||
this.decoding = false;
|
||||
if (this.pendingDecode) {
|
||||
this.pendingDecode = false;
|
||||
await this.maybeDecode(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async decodeNow(): Promise<string> {
|
||||
if (this.pcm16.length === 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const peak = pcm16lePeakAbs(this.pcm16);
|
||||
const peakFloat = peak / 32768.0;
|
||||
const targetPeak = 0.6;
|
||||
const maxGain = 50;
|
||||
const gain = peakFloat > 0 && peakFloat < targetPeak ? Math.min(maxGain, targetPeak / peakFloat) : 1;
|
||||
|
||||
const stream = this.engine.createStream();
|
||||
try {
|
||||
const floatSamples = pcm16leToFloat32(this.pcm16, gain);
|
||||
this.engine.acceptWaveform(stream, this.engine.sampleRate, floatSamples);
|
||||
this.engine.recognizer.decode(stream);
|
||||
const result = this.engine.recognizer.getResult(stream);
|
||||
return String(result?.text ?? result ?? "").trim();
|
||||
} finally {
|
||||
try {
|
||||
stream.free?.();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
import type pino from "pino";
|
||||
|
||||
import type { SpeechToTextProvider, TranscriptionResult } from "../speech-provider.js";
|
||||
import { Pcm16MonoResampler } from "../../agent/pcm16-resampler.js";
|
||||
import { parsePcm16MonoWav, parsePcmRateFromFormat, pcm16lePeakAbs, pcm16leToFloat32 } from "../audio.js";
|
||||
import { SherpaOfflineRecognizerEngine } from "./sherpa-offline-recognizer.js";
|
||||
|
||||
export type SherpaParakeetSttConfig = {
|
||||
engine: SherpaOfflineRecognizerEngine;
|
||||
silencePeakThreshold?: number;
|
||||
};
|
||||
|
||||
export class SherpaOnnxParakeetSTT implements SpeechToTextProvider {
|
||||
private readonly engine: SherpaOfflineRecognizerEngine;
|
||||
private readonly silencePeakThreshold: number;
|
||||
private readonly logger: pino.Logger;
|
||||
|
||||
constructor(config: SherpaParakeetSttConfig, logger: pino.Logger) {
|
||||
this.engine = config.engine;
|
||||
this.silencePeakThreshold = config.silencePeakThreshold ?? 300;
|
||||
this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "parakeet-stt" });
|
||||
}
|
||||
|
||||
async transcribeAudio(audioBuffer: Buffer, format: string): Promise<TranscriptionResult> {
|
||||
const start = Date.now();
|
||||
|
||||
let inputRate: number;
|
||||
let pcm16: Buffer;
|
||||
|
||||
if (format.toLowerCase().includes("audio/wav")) {
|
||||
const parsed = parsePcm16MonoWav(audioBuffer);
|
||||
inputRate = parsed.sampleRate;
|
||||
pcm16 = parsed.pcm16;
|
||||
} else if (format.toLowerCase().includes("audio/pcm")) {
|
||||
inputRate = parsePcmRateFromFormat(format, this.engine.sampleRate) ?? this.engine.sampleRate;
|
||||
pcm16 = audioBuffer;
|
||||
} else {
|
||||
throw new Error(`Unsupported audio format for sherpa Parakeet STT: ${format}`);
|
||||
}
|
||||
|
||||
const peak = pcm16lePeakAbs(pcm16);
|
||||
if (peak < this.silencePeakThreshold) {
|
||||
return { text: "", duration: Date.now() - start, isLowConfidence: true };
|
||||
}
|
||||
|
||||
let pcmForModel = pcm16;
|
||||
if (inputRate !== this.engine.sampleRate) {
|
||||
const resampler = new Pcm16MonoResampler({ inputRate, outputRate: this.engine.sampleRate });
|
||||
pcmForModel = resampler.processChunk(pcm16);
|
||||
inputRate = this.engine.sampleRate;
|
||||
}
|
||||
|
||||
const peakForModel = pcm16lePeakAbs(pcmForModel);
|
||||
const peakFloat = peakForModel / 32768.0;
|
||||
const targetPeak = 0.6;
|
||||
const maxGain = 50;
|
||||
const gain =
|
||||
peakFloat > 0 && peakFloat < targetPeak
|
||||
? Math.min(maxGain, targetPeak / peakFloat)
|
||||
: 1;
|
||||
|
||||
const stream = this.engine.createStream();
|
||||
try {
|
||||
const floatSamples = pcm16leToFloat32(pcmForModel, gain);
|
||||
this.engine.acceptWaveform(stream, inputRate, floatSamples);
|
||||
this.engine.recognizer.decode(stream);
|
||||
const result = this.engine.recognizer.getResult(stream);
|
||||
const text = String(result?.text ?? result ?? "").trim();
|
||||
const duration = Date.now() - start;
|
||||
this.logger.debug({ duration, textLength: text.length }, "Parakeet transcription complete");
|
||||
return { text, duration, ...(text.length === 0 ? { isLowConfidence: true } : {}) };
|
||||
} finally {
|
||||
try {
|
||||
stream.free?.();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
|
||||
import type { RealtimeTranscriptionSession } from "../../dictation/dictation-stream-manager.js";
|
||||
import { pcm16lePeakAbs, pcm16leToFloat32 } from "../audio.js";
|
||||
import { SherpaOnlineRecognizerEngine } from "./sherpa-online-recognizer.js";
|
||||
|
||||
export class SherpaRealtimeTranscriptionSession
|
||||
extends EventEmitter
|
||||
implements RealtimeTranscriptionSession
|
||||
{
|
||||
private readonly engine: SherpaOnlineRecognizerEngine;
|
||||
private stream: any | null = null;
|
||||
private connected = false;
|
||||
|
||||
private currentItemId: string | null = null;
|
||||
private previousItemId: string | null = null;
|
||||
private lastPartialText = "";
|
||||
private readonly tailPaddingMs: number;
|
||||
|
||||
constructor(params: { engine: SherpaOnlineRecognizerEngine; tailPaddingMs?: number }) {
|
||||
super();
|
||||
this.engine = params.engine;
|
||||
this.tailPaddingMs = params.tailPaddingMs ?? 500;
|
||||
}
|
||||
|
||||
async connect(): Promise<void> {
|
||||
if (this.connected) {
|
||||
return;
|
||||
}
|
||||
this.stream = this.engine.createStream();
|
||||
this.currentItemId = uuidv4();
|
||||
this.connected = true;
|
||||
}
|
||||
|
||||
appendPcm16Base64(base64Audio: string): void {
|
||||
if (!this.connected || !this.stream || !this.currentItemId) {
|
||||
this.emit("error", new Error("Sherpa realtime session not connected"));
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const pcm16 = Buffer.from(base64Audio, "base64");
|
||||
const peak = pcm16lePeakAbs(pcm16);
|
||||
const peakFloat = peak / 32768.0;
|
||||
const targetPeak = 0.6;
|
||||
const maxGain = 50;
|
||||
const gain =
|
||||
peakFloat > 0 && peakFloat < targetPeak
|
||||
? Math.min(maxGain, targetPeak / peakFloat)
|
||||
: 1;
|
||||
const floatSamples = pcm16leToFloat32(pcm16, gain);
|
||||
this.stream.acceptWaveform(this.engine.sampleRate, floatSamples);
|
||||
|
||||
while (this.engine.recognizer.isReady(this.stream)) {
|
||||
this.engine.recognizer.decode(this.stream);
|
||||
}
|
||||
|
||||
const text = String(this.engine.recognizer.getResult(this.stream)?.text ?? "").trim();
|
||||
if (text !== this.lastPartialText) {
|
||||
this.lastPartialText = text;
|
||||
this.emit("transcript", { itemId: this.currentItemId, transcript: text, isFinal: false });
|
||||
}
|
||||
} catch (err) {
|
||||
this.emit("error", err instanceof Error ? err : new Error(String(err)));
|
||||
}
|
||||
}
|
||||
|
||||
commit(): void {
|
||||
if (!this.connected || !this.stream || !this.currentItemId) {
|
||||
this.emit("error", new Error("Sherpa realtime session not connected"));
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const padSamples = Math.max(0, Math.round((this.engine.sampleRate * this.tailPaddingMs) / 1000));
|
||||
if (padSamples > 0) {
|
||||
this.stream.acceptWaveform(this.engine.sampleRate, new Float32Array(padSamples));
|
||||
}
|
||||
|
||||
while (this.engine.recognizer.isReady(this.stream)) {
|
||||
this.engine.recognizer.decode(this.stream);
|
||||
}
|
||||
|
||||
const finalText = String(this.engine.recognizer.getResult(this.stream)?.text ?? "").trim();
|
||||
const itemId = this.currentItemId;
|
||||
const previousItemId = this.previousItemId;
|
||||
|
||||
this.emit("committed", { itemId, previousItemId });
|
||||
this.emit("transcript", { itemId, transcript: finalText, isFinal: true });
|
||||
|
||||
this.previousItemId = itemId;
|
||||
this.currentItemId = uuidv4();
|
||||
this.lastPartialText = "";
|
||||
this.engine.recognizer.reset(this.stream);
|
||||
} catch (err) {
|
||||
this.emit("error", err instanceof Error ? err : new Error(String(err)));
|
||||
}
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
if (!this.connected || !this.stream) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
this.engine.recognizer.reset(this.stream);
|
||||
this.currentItemId = uuidv4();
|
||||
this.lastPartialText = "";
|
||||
} catch (err) {
|
||||
this.emit("error", err instanceof Error ? err : new Error(String(err)));
|
||||
}
|
||||
}
|
||||
|
||||
close(): void {
|
||||
if (!this.stream) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
this.stream.free?.();
|
||||
} catch {
|
||||
// ignore
|
||||
} finally {
|
||||
this.stream = null;
|
||||
this.connected = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
95
packages/server/src/server/speech/sherpa/sherpa-stt.ts
Normal file
95
packages/server/src/server/speech/sherpa/sherpa-stt.ts
Normal file
@@ -0,0 +1,95 @@
|
||||
import type pino from "pino";
|
||||
|
||||
import type { SpeechToTextProvider, TranscriptionResult } from "../speech-provider.js";
|
||||
import { Pcm16MonoResampler } from "../../agent/pcm16-resampler.js";
|
||||
import { parsePcm16MonoWav, parsePcmRateFromFormat, pcm16lePeakAbs, pcm16leToFloat32 } from "../audio.js";
|
||||
import { SherpaOnlineRecognizerEngine } from "./sherpa-online-recognizer.js";
|
||||
|
||||
export type SherpaSttConfig = {
|
||||
engine: SherpaOnlineRecognizerEngine;
|
||||
silencePeakThreshold?: number;
|
||||
tailPaddingMs?: number;
|
||||
};
|
||||
|
||||
export class SherpaOnnxSTT implements SpeechToTextProvider {
|
||||
private readonly engine: SherpaOnlineRecognizerEngine;
|
||||
private readonly silencePeakThreshold: number;
|
||||
private readonly tailPaddingMs: number;
|
||||
private readonly logger: pino.Logger;
|
||||
|
||||
constructor(config: SherpaSttConfig, logger: pino.Logger) {
|
||||
this.engine = config.engine;
|
||||
this.silencePeakThreshold = config.silencePeakThreshold ?? 300;
|
||||
this.tailPaddingMs = config.tailPaddingMs ?? 500;
|
||||
this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "stt" });
|
||||
}
|
||||
|
||||
async transcribeAudio(audioBuffer: Buffer, format: string): Promise<TranscriptionResult> {
|
||||
const start = Date.now();
|
||||
|
||||
let inputRate: number;
|
||||
let pcm16: Buffer;
|
||||
|
||||
if (format.toLowerCase().includes("audio/wav")) {
|
||||
const parsed = parsePcm16MonoWav(audioBuffer);
|
||||
inputRate = parsed.sampleRate;
|
||||
pcm16 = parsed.pcm16;
|
||||
} else if (format.toLowerCase().includes("audio/pcm")) {
|
||||
inputRate = parsePcmRateFromFormat(format, this.engine.sampleRate) ?? this.engine.sampleRate;
|
||||
pcm16 = audioBuffer;
|
||||
} else {
|
||||
throw new Error(`Unsupported audio format for sherpa STT: ${format}`);
|
||||
}
|
||||
|
||||
const peak = pcm16lePeakAbs(pcm16);
|
||||
if (peak < this.silencePeakThreshold) {
|
||||
return { text: "", duration: Date.now() - start, isLowConfidence: true };
|
||||
}
|
||||
|
||||
let pcmForModel = pcm16;
|
||||
if (inputRate !== this.engine.sampleRate) {
|
||||
const resampler = new Pcm16MonoResampler({ inputRate, outputRate: this.engine.sampleRate });
|
||||
pcmForModel = resampler.processChunk(pcm16);
|
||||
inputRate = this.engine.sampleRate;
|
||||
}
|
||||
|
||||
const peakForModel = pcm16lePeakAbs(pcmForModel);
|
||||
const peakFloat = peakForModel / 32768.0;
|
||||
const targetPeak = 0.6;
|
||||
const maxGain = 50;
|
||||
const gain =
|
||||
peakFloat > 0 && peakFloat < targetPeak
|
||||
? Math.min(maxGain, targetPeak / peakFloat)
|
||||
: 1;
|
||||
|
||||
const stream = this.engine.createStream();
|
||||
try {
|
||||
const floatSamples = pcm16leToFloat32(pcmForModel, gain);
|
||||
stream.acceptWaveform(inputRate, floatSamples);
|
||||
|
||||
while (this.engine.recognizer.isReady(stream)) {
|
||||
this.engine.recognizer.decode(stream);
|
||||
}
|
||||
|
||||
const padSamples = Math.max(0, Math.round((this.engine.sampleRate * this.tailPaddingMs) / 1000));
|
||||
if (padSamples > 0) {
|
||||
stream.acceptWaveform(this.engine.sampleRate, new Float32Array(padSamples));
|
||||
}
|
||||
|
||||
while (this.engine.recognizer.isReady(stream)) {
|
||||
this.engine.recognizer.decode(stream);
|
||||
}
|
||||
|
||||
const text = String(this.engine.recognizer.getResult(stream)?.text ?? "").trim();
|
||||
const duration = Date.now() - start;
|
||||
this.logger.debug({ duration, textLength: text.length }, "Sherpa transcription complete");
|
||||
return { text, duration, ...(text.length === 0 ? { isLowConfidence: true } : {}) };
|
||||
} finally {
|
||||
try {
|
||||
stream.free?.();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
121
packages/server/src/server/speech/sherpa/sherpa-tts.ts
Normal file
121
packages/server/src/server/speech/sherpa/sherpa-tts.ts
Normal file
@@ -0,0 +1,121 @@
|
||||
import type pino from "pino";
|
||||
import { Readable } from "node:stream";
|
||||
import { existsSync } from "node:fs";
|
||||
|
||||
import type { SpeechStreamResult, TextToSpeechProvider } from "../speech-provider.js";
|
||||
import { chunkBuffer, float32ToPcm16le } from "../audio.js";
|
||||
import { loadSherpaOnnx } from "./sherpa-onnx-loader.js";
|
||||
|
||||
export type SherpaTtsPreset = "kokoro-en-v0_19" | "kitten-nano-en-v0_1-fp16";
|
||||
|
||||
export type SherpaTtsConfig = {
|
||||
preset: SherpaTtsPreset;
|
||||
modelDir: string;
|
||||
speakerId?: number;
|
||||
speed?: number;
|
||||
lengthScale?: number;
|
||||
numThreads?: number;
|
||||
};
|
||||
|
||||
function assertFileExists(filePath: string, label: string): void {
|
||||
if (!existsSync(filePath)) {
|
||||
throw new Error(`Missing ${label}: ${filePath}`);
|
||||
}
|
||||
}
|
||||
|
||||
export class SherpaOnnxTTS implements TextToSpeechProvider {
|
||||
private readonly tts: any;
|
||||
private readonly speakerId: number;
|
||||
private readonly speed: number;
|
||||
private readonly logger: pino.Logger;
|
||||
|
||||
constructor(config: SherpaTtsConfig, logger: pino.Logger) {
|
||||
if (config.preset !== "kokoro-en-v0_19" && config.preset !== "kitten-nano-en-v0_1-fp16") {
|
||||
throw new Error(`Unsupported Sherpa TTS preset: ${config.preset}`);
|
||||
}
|
||||
this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "tts" });
|
||||
this.speakerId = config.speakerId ?? 0;
|
||||
this.speed = config.speed ?? 1.0;
|
||||
|
||||
const sherpa = loadSherpaOnnx();
|
||||
|
||||
const modelFile = config.preset === "kokoro-en-v0_19" ? "model.onnx" : "model.fp16.onnx";
|
||||
const modelPath = `${config.modelDir}/${modelFile}`;
|
||||
const voicesPath = `${config.modelDir}/voices.bin`;
|
||||
const tokensPath = `${config.modelDir}/tokens.txt`;
|
||||
const dataDir = `${config.modelDir}/espeak-ng-data`;
|
||||
|
||||
assertFileExists(modelPath, "TTS model");
|
||||
assertFileExists(voicesPath, "TTS voices");
|
||||
assertFileExists(tokensPath, "TTS tokens");
|
||||
assertFileExists(dataDir, "TTS espeak-ng dataDir");
|
||||
|
||||
const modelConfigKey =
|
||||
config.preset === "kokoro-en-v0_19"
|
||||
? "offlineTtsKokoroModelConfig"
|
||||
: "offlineTtsKittenModelConfig";
|
||||
|
||||
const modelConfig = {
|
||||
[modelConfigKey]: {
|
||||
model: modelPath,
|
||||
voices: voicesPath,
|
||||
tokens: tokensPath,
|
||||
dataDir,
|
||||
lengthScale: config.lengthScale ?? 1.0,
|
||||
},
|
||||
numThreads: config.numThreads ?? 2,
|
||||
debug: 0,
|
||||
provider: "cpu",
|
||||
};
|
||||
|
||||
const offlineTtsConfig = {
|
||||
offlineTtsModelConfig: modelConfig,
|
||||
maxNumSentences: 1,
|
||||
};
|
||||
|
||||
this.tts = sherpa.createOfflineTts(offlineTtsConfig);
|
||||
this.logger.info({ preset: config.preset, modelDir: config.modelDir }, "Sherpa offline TTS initialized");
|
||||
}
|
||||
|
||||
async synthesizeSpeech(text: string): Promise<SpeechStreamResult> {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed) {
|
||||
throw new Error("Cannot synthesize empty text");
|
||||
}
|
||||
|
||||
const audio = this.tts.generate({ text: trimmed, sid: this.speakerId, speed: this.speed });
|
||||
const samples: Float32Array | null =
|
||||
audio && audio.samples instanceof Float32Array
|
||||
? audio.samples
|
||||
: audio && Array.isArray(audio.samples)
|
||||
? Float32Array.from(audio.samples as number[])
|
||||
: null;
|
||||
const sampleRate: number =
|
||||
audio && typeof audio.sampleRate === "number" && Number.isFinite(audio.sampleRate) && audio.sampleRate > 0
|
||||
? audio.sampleRate
|
||||
: typeof this.tts.sampleRate === "number"
|
||||
? this.tts.sampleRate
|
||||
: 24000;
|
||||
|
||||
if (!samples) {
|
||||
throw new Error("Unexpected sherpa TTS output: missing Float32 samples");
|
||||
}
|
||||
|
||||
const pcm16 = float32ToPcm16le(samples);
|
||||
const chunkBytes = Math.max(2, Math.round(sampleRate * 0.05) * 2); // ~50ms
|
||||
const chunks = chunkBuffer(pcm16, chunkBytes);
|
||||
|
||||
return {
|
||||
stream: Readable.from(chunks),
|
||||
format: `pcm;rate=${sampleRate}`,
|
||||
};
|
||||
}
|
||||
|
||||
free(): void {
|
||||
try {
|
||||
this.tts?.free?.();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import pino from "pino";
|
||||
|
||||
import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./model-downloader.js";
|
||||
import type { SherpaOnnxModelId } from "./model-catalog.js";
|
||||
import { createDaemonTestContext } from "../../test-utils/index.js";
|
||||
import { parsePcm16MonoWav, wordSimilarity } from "../../test-utils/dictation-e2e.js";
|
||||
import { SherpaOnnxTTS } from "./sherpa-tts.js";
|
||||
import { PocketTtsOnnxTTS } from "../pocket/pocket-tts-onnx.js";
|
||||
import { SherpaOnlineRecognizerEngine } from "./sherpa-online-recognizer.js";
|
||||
import { SherpaOnnxSTT } from "./sherpa-stt.js";
|
||||
import { SherpaOfflineRecognizerEngine } from "./sherpa-offline-recognizer.js";
|
||||
import { SherpaOnnxParakeetSTT } from "./sherpa-parakeet-stt.js";
|
||||
|
||||
const RUN = process.env.PASEO_SPEECH_E2E_DOWNLOAD === "1";
|
||||
const downloadTest = RUN ? test : test.skip;
|
||||
|
||||
type ModelSet = "zipformer-kitten" | "parakeet-pocket";
|
||||
|
||||
function getModelSet(): ModelSet {
|
||||
const raw = (process.env.PASEO_SPEECH_E2E_MODEL_SET ?? "parakeet-pocket").trim().toLowerCase();
|
||||
if (raw === "zipformer-kitten" || raw === "zipformer") return "zipformer-kitten";
|
||||
if (raw === "parakeet-pocket" || raw === "parakeet") return "parakeet-pocket";
|
||||
throw new Error(`Unknown PASEO_SPEECH_E2E_MODEL_SET: ${raw}`);
|
||||
}
|
||||
|
||||
async function readFixtureWav(): Promise<Buffer> {
|
||||
const fixturePath = path.resolve(process.cwd(), "..", "app", "e2e", "fixtures", "recording.wav");
|
||||
return import("node:fs/promises").then((fs) => fs.readFile(fixturePath));
|
||||
}
|
||||
|
||||
async function readBaseline(): Promise<string> {
|
||||
const baselinePath = path.resolve(process.cwd(), "..", "app", "e2e", "fixtures", "recording.baseline.txt");
|
||||
return import("node:fs/promises").then((fs) => fs.readFile(baselinePath, "utf-8")).then((t) => t.trim());
|
||||
}
|
||||
|
||||
async function readAllChunks(stream: NodeJS.ReadableStream): Promise<{ chunks: Buffer[]; combined: Buffer }> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of stream as any) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
}
|
||||
return { chunks, combined: Buffer.concat(chunks) };
|
||||
}
|
||||
|
||||
function waitForSignal<T>(
|
||||
timeoutMs: number,
|
||||
setup: (
|
||||
resolve: (value: T) => void,
|
||||
reject: (error: Error) => void
|
||||
) => () => void
|
||||
): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let cleanup: (() => void) | null = null;
|
||||
const timeout = setTimeout(() => {
|
||||
cleanup?.();
|
||||
reject(new Error(`Timeout waiting for event after ${timeoutMs}ms`));
|
||||
}, timeoutMs);
|
||||
|
||||
cleanup = setup(
|
||||
(value) => {
|
||||
clearTimeout(timeout);
|
||||
cleanup?.();
|
||||
resolve(value);
|
||||
},
|
||||
(error) => {
|
||||
clearTimeout(timeout);
|
||||
cleanup?.();
|
||||
reject(error);
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function toAudioPcmFormat(format: string): string {
|
||||
const trimmed = format.trim();
|
||||
const lower = trimmed.toLowerCase();
|
||||
if (lower.startsWith("audio/pcm")) {
|
||||
return trimmed;
|
||||
}
|
||||
if (lower.startsWith("pcm")) {
|
||||
const rest = trimmed.replace(/^pcm;?/i, "");
|
||||
return rest ? `audio/pcm;${rest}` : "audio/pcm";
|
||||
}
|
||||
// Fall back to treating it as a suffix (e.g. "rate=24000")
|
||||
return `audio/pcm;${trimmed}`;
|
||||
}
|
||||
|
||||
describe("speech models (download E2E)", () => {
|
||||
downloadTest(
|
||||
"downloads models and runs streaming STT + streaming TTS",
|
||||
async () => {
|
||||
const logger = pino({ level: "silent" });
|
||||
const set = getModelSet();
|
||||
|
||||
const paseoHomeRoot = mkdtempSync(path.join(tmpdir(), "paseo-speech-download-"));
|
||||
const modelsDir = path.join(paseoHomeRoot, ".paseo", "models", "sherpa-onnx");
|
||||
|
||||
const modelIds: SherpaOnnxModelId[] =
|
||||
set === "parakeet-pocket"
|
||||
? ["parakeet-tdt-0.6b-v3-int8", "pocket-tts-onnx-int8"]
|
||||
: ["zipformer-bilingual-zh-en-2023-02-20", "kitten-nano-en-v0_1-fp16"];
|
||||
|
||||
await ensureSherpaOnnxModels({
|
||||
modelsDir,
|
||||
modelIds,
|
||||
autoDownload: true,
|
||||
logger,
|
||||
});
|
||||
|
||||
const ctx = await createDaemonTestContext({
|
||||
paseoHomeRoot,
|
||||
dictationFinalTimeoutMs: 8000,
|
||||
speech: {
|
||||
dictationSttProvider: "sherpa",
|
||||
voiceSttProvider: "sherpa",
|
||||
voiceTtsProvider: "sherpa",
|
||||
sherpaOnnx: {
|
||||
modelsDir,
|
||||
autoDownload: false,
|
||||
stt: { preset: set === "parakeet-pocket" ? "parakeet-tdt-0.6b-v3-int8" : "zipformer-bilingual-zh-en-2023-02-20" },
|
||||
tts: { preset: set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16" },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
const wav = await readFixtureWav();
|
||||
const { sampleRate, pcm16 } = parsePcm16MonoWav(wav);
|
||||
expect(sampleRate).toBe(16000);
|
||||
const format = "audio/pcm;rate=16000;bits=16";
|
||||
|
||||
// Streaming STT: dictation path (verifies websocket streaming + partials + final)
|
||||
const dictationId = `dict-download-${Date.now()}`;
|
||||
let partialCount = 0;
|
||||
const unsubscribe = ctx.client.on("dictation_stream_partial", (message) => {
|
||||
if (message.type !== "dictation_stream_partial") return;
|
||||
if (message.payload.dictationId !== dictationId) return;
|
||||
partialCount += 1;
|
||||
});
|
||||
|
||||
await ctx.client.startDictationStream(dictationId, format);
|
||||
const chunkBytes = 3200; // 100ms @ 16kHz mono PCM16
|
||||
let seq = 0;
|
||||
for (let offset = 0; offset < pcm16.length; offset += chunkBytes) {
|
||||
const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
|
||||
ctx.client.sendDictationStreamChunk(dictationId, seq, chunk.toString("base64"), format);
|
||||
seq += 1;
|
||||
}
|
||||
const finalSeq = seq - 1;
|
||||
const dictationFinal = await ctx.client.finishDictationStream(dictationId, finalSeq);
|
||||
unsubscribe();
|
||||
|
||||
expect(dictationFinal.text.toLowerCase()).toContain("voice note");
|
||||
const baseline = await readBaseline();
|
||||
expect(wordSimilarity(dictationFinal.text, baseline)).toBeGreaterThan(0.45);
|
||||
expect(partialCount).toBeGreaterThan(0);
|
||||
|
||||
// Voice-mode STT: chunked upload until isLast=true
|
||||
const transcriptionPromise = waitForSignal<string>(30000, (resolve, reject) => {
|
||||
const offResult = ctx.client.on("transcription_result", (message) => {
|
||||
if (message.type !== "transcription_result") return;
|
||||
resolve(message.payload.text);
|
||||
});
|
||||
const offError = ctx.client.on("activity_log", (message) => {
|
||||
if (message.type !== "activity_log") return;
|
||||
const payload = message.payload as { type?: unknown; content?: unknown };
|
||||
if (payload.type !== "error") return;
|
||||
const content = typeof payload.content === "string" ? payload.content : null;
|
||||
if (!content) return;
|
||||
reject(new Error(content));
|
||||
});
|
||||
return () => {
|
||||
offResult();
|
||||
offError();
|
||||
};
|
||||
});
|
||||
|
||||
await ctx.client.setVoiceConversation(true, `voice-download-${Date.now()}`);
|
||||
for (let offset = 0; offset < pcm16.length; offset += chunkBytes) {
|
||||
const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
|
||||
const isLast = offset + chunkBytes >= pcm16.length;
|
||||
await ctx.client.sendVoiceAudioChunk(chunk.toString("base64"), format, isLast);
|
||||
}
|
||||
const voiceText = (await transcriptionPromise).trim().toLowerCase();
|
||||
if (voiceText.length > 0) {
|
||||
expect(voiceText).toContain("voice note");
|
||||
}
|
||||
await ctx.client.setVoiceConversation(false);
|
||||
|
||||
// Streaming TTS: generate locally from downloaded model and validate chunking.
|
||||
const ttsText = "This is a voice note.";
|
||||
if (set === "parakeet-pocket") {
|
||||
const modelDir = getSherpaOnnxModelDir(modelsDir, "pocket-tts-onnx-int8");
|
||||
const tts = await PocketTtsOnnxTTS.create({ modelDir, precision: "int8", targetChunkMs: 50 }, logger);
|
||||
const { stream, format: ttsFormat } = await tts.synthesizeSpeech(ttsText);
|
||||
const { chunks, combined } = await readAllChunks(stream);
|
||||
|
||||
expect(ttsFormat).toMatch(/rate=\d+/);
|
||||
expect(chunks.length).toBeGreaterThan(3);
|
||||
expect(combined.byteLength).toBeGreaterThan(2000);
|
||||
|
||||
// Round trip: TTS -> STT (offline parakeet)
|
||||
const sttModelDir = getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v3-int8");
|
||||
const engine = new SherpaOfflineRecognizerEngine(
|
||||
{
|
||||
model: {
|
||||
kind: "nemo_transducer",
|
||||
encoder: `${sttModelDir}/encoder.int8.onnx`,
|
||||
decoder: `${sttModelDir}/decoder.int8.onnx`,
|
||||
joiner: `${sttModelDir}/joiner.int8.onnx`,
|
||||
tokens: `${sttModelDir}/tokens.txt`,
|
||||
},
|
||||
numThreads: 2,
|
||||
debug: 0,
|
||||
},
|
||||
logger
|
||||
);
|
||||
const stt = new SherpaOnnxParakeetSTT({ engine }, logger);
|
||||
const rt = await stt.transcribeAudio(combined, toAudioPcmFormat(ttsFormat));
|
||||
engine.free();
|
||||
expect(wordSimilarity(rt.text, ttsText)).toBeGreaterThan(0.25);
|
||||
} else {
|
||||
const ttsModelDir = path.join(modelsDir, "kitten-nano-en-v0_1-fp16");
|
||||
const tts = new SherpaOnnxTTS({ preset: "kitten-nano-en-v0_1-fp16", modelDir: ttsModelDir }, logger);
|
||||
const { stream, format: ttsFormat } = await tts.synthesizeSpeech(ttsText);
|
||||
const { chunks, combined } = await readAllChunks(stream);
|
||||
tts.free();
|
||||
|
||||
expect(ttsFormat).toMatch(/rate=\d+/);
|
||||
expect(chunks.length).toBeGreaterThan(3);
|
||||
expect(combined.byteLength).toBeGreaterThan(2000);
|
||||
|
||||
// Round trip: TTS -> STT (online zipformer, offline segment)
|
||||
const sttModelDir = path.join(modelsDir, "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20");
|
||||
const engine = new SherpaOnlineRecognizerEngine(
|
||||
{
|
||||
model: {
|
||||
kind: "transducer",
|
||||
encoder: `${sttModelDir}/encoder-epoch-99-avg-1.onnx`,
|
||||
decoder: `${sttModelDir}/decoder-epoch-99-avg-1.onnx`,
|
||||
joiner: `${sttModelDir}/joiner-epoch-99-avg-1.onnx`,
|
||||
tokens: `${sttModelDir}/tokens.txt`,
|
||||
modelType: "zipformer",
|
||||
},
|
||||
numThreads: 1,
|
||||
debug: 0,
|
||||
},
|
||||
logger
|
||||
);
|
||||
const stt = new SherpaOnnxSTT({ engine }, logger);
|
||||
const rt = await stt.transcribeAudio(combined, toAudioPcmFormat(ttsFormat));
|
||||
engine.free();
|
||||
expect(wordSimilarity(rt.text, ttsText)).toBeGreaterThan(0.25);
|
||||
}
|
||||
} finally {
|
||||
await ctx.cleanup();
|
||||
}
|
||||
},
|
||||
15 * 60_000
|
||||
);
|
||||
});
|
||||
30
packages/server/src/server/speech/speech-provider.ts
Normal file
30
packages/server/src/server/speech/speech-provider.ts
Normal file
@@ -0,0 +1,30 @@
|
||||
import type { Readable } from "node:stream";
|
||||
|
||||
export interface LogprobToken {
|
||||
token: string;
|
||||
logprob: number;
|
||||
bytes?: number[];
|
||||
}
|
||||
|
||||
export interface TranscriptionResult {
|
||||
text: string;
|
||||
language?: string;
|
||||
duration?: number;
|
||||
logprobs?: LogprobToken[];
|
||||
avgLogprob?: number;
|
||||
isLowConfidence?: boolean;
|
||||
}
|
||||
|
||||
export interface SpeechToTextProvider {
|
||||
transcribeAudio(audioBuffer: Buffer, format: string): Promise<TranscriptionResult>;
|
||||
}
|
||||
|
||||
export interface SpeechStreamResult {
|
||||
stream: Readable;
|
||||
format: string;
|
||||
}
|
||||
|
||||
export interface TextToSpeechProvider {
|
||||
synthesizeSpeech(text: string): Promise<SpeechStreamResult>;
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ import path from "node:path";
|
||||
import { mkdir, mkdtemp, rm } from "node:fs/promises";
|
||||
|
||||
import pino from "pino";
|
||||
import { createPaseoDaemon, type PaseoDaemonConfig, type PaseoOpenAIConfig } from "../bootstrap.js";
|
||||
import { createPaseoDaemon, type PaseoDaemonConfig, type PaseoOpenAIConfig, type PaseoSpeechConfig } from "../bootstrap.js";
|
||||
import type { AgentClient, AgentProvider } from "../agent/agent-sdk-types.js";
|
||||
import { createTestAgentClients } from "./fake-agent-client.js";
|
||||
|
||||
@@ -20,6 +20,7 @@ type TestPaseoDaemonOptions = {
|
||||
staticDir?: string;
|
||||
cleanup?: boolean;
|
||||
openai?: PaseoOpenAIConfig;
|
||||
speech?: PaseoSpeechConfig;
|
||||
dictationFinalTimeoutMs?: number;
|
||||
};
|
||||
|
||||
@@ -76,6 +77,7 @@ export async function createTestPaseoDaemon(
|
||||
relayEndpoint: options.relayEndpoint ?? "relay.paseo.sh:443",
|
||||
appBaseUrl: "https://app.paseo.sh",
|
||||
openai: options.openai,
|
||||
speech: options.speech,
|
||||
openrouterApiKey: null,
|
||||
dictationFinalTimeoutMs: options.dictationFinalTimeoutMs,
|
||||
downloadTokenTtlMs: options.downloadTokenTtlMs,
|
||||
|
||||
@@ -6,8 +6,6 @@ import { hostname as getHostname } from "node:os";
|
||||
import type { AgentManager } from "./agent/agent-manager.js";
|
||||
import type { AgentStorage } from "./agent/agent-storage.js";
|
||||
import type { DownloadTokenStore } from "./file-download/token-store.js";
|
||||
import type { OpenAISTT } from "./agent/stt-openai.js";
|
||||
import type { OpenAITTS } from "./agent/tts-openai.js";
|
||||
import type { TerminalManager } from "../terminal/terminal-manager.js";
|
||||
import type pino from "pino";
|
||||
import {
|
||||
@@ -22,6 +20,8 @@ import type { AgentProvider } from "./agent/agent-sdk-types.js";
|
||||
import { PushTokenStore } from "./push/token-store.js";
|
||||
import { PushService } from "./push/push-service.js";
|
||||
import { VoiceConversationStore } from "./voice-conversation-store.js";
|
||||
import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js";
|
||||
import type { RealtimeTranscriptionSessionFactory } from "./dictation/dictation-stream-manager.js";
|
||||
|
||||
export type AgentMcpTransportFactory = () => Promise<Transport>;
|
||||
|
||||
@@ -67,13 +67,14 @@ export class VoiceAssistantWebSocketServer {
|
||||
private readonly pushTokenStore: PushTokenStore;
|
||||
private readonly pushService: PushService;
|
||||
private readonly createAgentMcpTransport: AgentMcpTransportFactory;
|
||||
private readonly stt: OpenAISTT | null;
|
||||
private readonly tts: OpenAITTS | null;
|
||||
private readonly stt: SpeechToTextProvider | null;
|
||||
private readonly tts: TextToSpeechProvider | null;
|
||||
private readonly terminalManager: TerminalManager | null;
|
||||
private readonly voiceConversationStore: VoiceConversationStore;
|
||||
private readonly dictation: {
|
||||
openaiApiKey?: string | null;
|
||||
finalTimeoutMs?: number;
|
||||
sessionFactory?: RealtimeTranscriptionSessionFactory;
|
||||
} | null;
|
||||
private readonly voice: {
|
||||
openrouterApiKey?: string | null;
|
||||
@@ -90,7 +91,7 @@ export class VoiceAssistantWebSocketServer {
|
||||
paseoHome: string,
|
||||
createAgentMcpTransport: AgentMcpTransportFactory,
|
||||
wsConfig: WebSocketServerConfig,
|
||||
speech?: { stt: OpenAISTT | null; tts: OpenAITTS | null },
|
||||
speech?: { stt: SpeechToTextProvider | null; tts: TextToSpeechProvider | null },
|
||||
terminalManager?: TerminalManager | null,
|
||||
voice?: {
|
||||
openrouterApiKey?: string | null;
|
||||
@@ -99,6 +100,7 @@ export class VoiceAssistantWebSocketServer {
|
||||
dictation?: {
|
||||
openaiApiKey?: string | null;
|
||||
finalTimeoutMs?: number;
|
||||
sessionFactory?: RealtimeTranscriptionSessionFactory;
|
||||
}
|
||||
) {
|
||||
this.logger = logger.child({ module: "websocket-server" });
|
||||
|
||||
95
scripts/speech/download-sherpa-models.sh
Executable file
95
scripts/speech/download-sherpa-models.sh
Executable file
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Download local speech models for Paseo (sherpa-onnx).
|
||||
|
||||
Defaults:
|
||||
- STT: sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20
|
||||
- TTS: kitten-nano-en-v0_1-fp16
|
||||
|
||||
Usage:
|
||||
scripts/speech/download-sherpa-models.sh [--models-dir DIR] [--with-kokoro] [--with-paraformer]
|
||||
|
||||
Preferred:
|
||||
npm run speech:download --workspace=@getpaseo/server
|
||||
|
||||
Notes:
|
||||
- Models are downloaded from the sherpa-onnx GitHub releases.
|
||||
- Pocket TTS is downloaded by the Node script (`npm run speech:download --workspace=@getpaseo/server`)
|
||||
because it is a file-based HuggingFace model (not a single tarball).
|
||||
- Set PASEO_SHERPA_ONNX_MODELS_DIR to override where the daemon looks.
|
||||
EOF
|
||||
}
|
||||
|
||||
MODELS_DIR=""
|
||||
WITH_KOKORO=0
|
||||
WITH_PARAFORMER=0
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--models-dir)
|
||||
MODELS_DIR="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--with-kokoro)
|
||||
WITH_KOKORO=1
|
||||
shift 1
|
||||
;;
|
||||
--with-paraformer)
|
||||
WITH_PARAFORMER=1
|
||||
shift 1
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unknown arg: $1" >&2
|
||||
usage >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "${MODELS_DIR}" ]]; then
|
||||
if [[ -n "${PASEO_SHERPA_ONNX_MODELS_DIR:-}" ]]; then
|
||||
MODELS_DIR="${PASEO_SHERPA_ONNX_MODELS_DIR}"
|
||||
elif [[ -n "${PASEO_HOME:-}" ]]; then
|
||||
MODELS_DIR="${PASEO_HOME}/models/sherpa-onnx"
|
||||
else
|
||||
MODELS_DIR="${HOME}/.paseo/models/sherpa-onnx"
|
||||
fi
|
||||
fi
|
||||
|
||||
mkdir -p "${MODELS_DIR}"
|
||||
cd "${MODELS_DIR}"
|
||||
|
||||
download_and_extract() {
|
||||
local url="$1"
|
||||
local filename
|
||||
filename="$(basename "$url")"
|
||||
|
||||
echo "Downloading ${filename}..."
|
||||
curl -fsSL -O "${url}"
|
||||
echo "Extracting ${filename}..."
|
||||
tar xf "${filename}"
|
||||
rm -f "${filename}"
|
||||
}
|
||||
|
||||
echo "NOTE: This script is deprecated. Prefer: npm run speech:download --workspace=@getpaseo/server" >&2
|
||||
|
||||
download_and_extract "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2"
|
||||
download_and_extract "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2"
|
||||
|
||||
if [[ "${WITH_PARAFORMER}" -eq 1 ]]; then
|
||||
download_and_extract "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2"
|
||||
fi
|
||||
|
||||
if [[ "${WITH_KOKORO}" -eq 1 ]]; then
|
||||
download_and_extract "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2"
|
||||
fi
|
||||
|
||||
echo "Done."
|
||||
echo "Models dir: ${MODELS_DIR}"
|
||||
Reference in New Issue
Block a user