From d791f79efeb675f6002e316b0a6c582793dcc7d4 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 6 Feb 2026 10:38:38 +0700 Subject: [PATCH] Local streaming STT/TTS with Parakeet + Pocket TTS - Add provider-agnostic streaming speech interfaces - Local STT via Sherpa (Zipformer/Paraformer) + Parakeet v3 (offline) - Local TTS via Pocket TTS ONNX (onnxruntime-node) with chunked streaming - Auto-download models into PASEO_HOME and add Node download script - Add E2E download test that round-trips TTS->STT --- README.md | 31 + package-lock.json | 269 ++++++++- .../app/e2e/fixtures/recording.baseline.txt | 2 + .../app/src/hooks/use-audio-player.native.ts | 80 ++- packages/server/package.json | 6 + .../server/scripts/download-speech-models.ts | 42 ++ packages/server/scripts/list-speech-models.ts | 11 + .../providers/codex-app-server-agent.test.ts | 6 +- .../src/server/agent/stt-manager.test.ts | 42 ++ .../server/src/server/agent/stt-manager.ts | 6 +- .../server/src/server/agent/stt-openai.ts | 18 +- .../src/server/agent/tts-manager.test.ts | 47 ++ .../server/src/server/agent/tts-manager.ts | 6 +- .../server/src/server/agent/tts-openai.ts | 10 +- packages/server/src/server/bootstrap.ts | 337 ++++++++++- packages/server/src/server/config.ts | 115 ++++ .../src/server/daemon-client.e2e.test.ts | 101 +++- .../dictation-stream-manager.test.ts | 31 + .../dictation/dictation-stream-manager.ts | 33 +- .../server/src/server/persisted-config.ts | 33 +- packages/server/src/server/session.ts | 10 +- packages/server/src/server/speech/audio.ts | 113 ++++ .../server/speech/pocket/pocket-tts-onnx.ts | 559 ++++++++++++++++++ .../src/server/speech/sherpa/model-catalog.ts | 128 ++++ .../speech/sherpa/model-downloader.test.ts | 56 ++ .../server/speech/sherpa/model-downloader.ts | 190 ++++++ .../sherpa/sherpa-offline-recognizer.ts | 104 ++++ .../speech/sherpa/sherpa-online-recognizer.ts | 127 ++++ .../speech/sherpa/sherpa-onnx-loader.ts | 18 + .../speech/sherpa/sherpa-onnx-node-loader.ts | 53 ++ .../sherpa-parakeet-realtime-session.ts | 153 +++++ .../speech/sherpa/sherpa-parakeet-stt.ts | 80 +++ .../speech/sherpa/sherpa-realtime-session.ts | 127 ++++ .../src/server/speech/sherpa/sherpa-stt.ts | 95 +++ .../src/server/speech/sherpa/sherpa-tts.ts | 121 ++++ .../speech/sherpa/speech-download.e2e.test.ts | 264 +++++++++ .../src/server/speech/speech-provider.ts | 30 + .../src/server/test-utils/paseo-daemon.ts | 4 +- .../server/src/server/websocket-server.ts | 12 +- scripts/speech/download-sherpa-models.sh | 95 +++ 40 files changed, 3421 insertions(+), 144 deletions(-) create mode 100644 packages/app/e2e/fixtures/recording.baseline.txt create mode 100644 packages/server/scripts/download-speech-models.ts create mode 100644 packages/server/scripts/list-speech-models.ts create mode 100644 packages/server/src/server/agent/stt-manager.test.ts create mode 100644 packages/server/src/server/agent/tts-manager.test.ts create mode 100644 packages/server/src/server/speech/audio.ts create mode 100644 packages/server/src/server/speech/pocket/pocket-tts-onnx.ts create mode 100644 packages/server/src/server/speech/sherpa/model-catalog.ts create mode 100644 packages/server/src/server/speech/sherpa/model-downloader.test.ts create mode 100644 packages/server/src/server/speech/sherpa/model-downloader.ts create mode 100644 packages/server/src/server/speech/sherpa/sherpa-offline-recognizer.ts create mode 100644 packages/server/src/server/speech/sherpa/sherpa-online-recognizer.ts create mode 100644 packages/server/src/server/speech/sherpa/sherpa-onnx-loader.ts create mode 100644 packages/server/src/server/speech/sherpa/sherpa-onnx-node-loader.ts create mode 100644 packages/server/src/server/speech/sherpa/sherpa-parakeet-realtime-session.ts create mode 100644 packages/server/src/server/speech/sherpa/sherpa-parakeet-stt.ts create mode 100644 packages/server/src/server/speech/sherpa/sherpa-realtime-session.ts create mode 100644 packages/server/src/server/speech/sherpa/sherpa-stt.ts create mode 100644 packages/server/src/server/speech/sherpa/sherpa-tts.ts create mode 100644 packages/server/src/server/speech/sherpa/speech-download.e2e.test.ts create mode 100644 packages/server/src/server/speech/speech-provider.ts create mode 100755 scripts/speech/download-sherpa-models.sh diff --git a/README.md b/README.md index 9773b5b2d..9a70f5d36 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,37 @@ npm install -g @getpaseo/cli && paseo Then open the app and connect to your daemon. +## Local speech (STT/TTS) + +Paseo can run dictation + voice mode STT/TTS fully locally via `sherpa-onnx`. + +When the daemon starts with a local speech provider selected, it will download any missing model files automatically (unless `PASEO_SHERPA_ONNX_AUTO_DOWNLOAD=0`). + +```bash +npm run speech:download --workspace=@getpaseo/server +``` + +Optional configuration: + +- `PASEO_SHERPA_ONNX_MODELS_DIR` (defaults to `~/.paseo/models/sherpa-onnx`) +- `PASEO_SHERPA_ONNX_AUTO_DOWNLOAD` (`1` by default; set `0` to disable automatic downloads on daemon start) +- `PASEO_SHERPA_STT_PRESET` (`zipformer`, `paraformer`, or `parakeet` for NVIDIA Parakeet TDT v3) +- `PASEO_SHERPA_TTS_PRESET` (`pocket-tts` (Kyutai Pocket TTS), `kitten`, or `kokoro`) +- `PASEO_DICTATION_STT_PROVIDER`, `PASEO_VOICE_STT_PROVIDER`, `PASEO_VOICE_TTS_PROVIDER` (`sherpa` or `openai`) + +To see all supported local model IDs: + +```bash +npm run speech:models --workspace=@getpaseo/server +``` + +Optional: run an end-to-end test that downloads real models and exercises streaming STT + streaming TTS: + +```bash +PASEO_SPEECH_E2E_DOWNLOAD=1 PASEO_SPEECH_E2E_MODEL_SET=parakeet-pocket \ + npx vitest run --workspace=@getpaseo/server src/server/speech/sherpa/speech-download.e2e.test.ts +``` + ## Documentation See [paseo.sh/docs](https://paseo.sh/docs) for full documentation. diff --git a/package-lock.json b/package-lock.json index bedc32ec7..4ec5f45a0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5621,6 +5621,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@sctg/sentencepiece-js": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@sctg/sentencepiece-js/-/sentencepiece-js-1.3.3.tgz", + "integrity": "sha512-iPnzR2HGjdQQG2SpTyPH3wnnpgQ2aS14B3I2jHjomkoaMprM3Sn+WOQblHwByetnvn3n14y3IwjwogVUHA2cVA==", + "license": "Apache-2.0", + "dependencies": { + "app-root-path": "^3.1.0", + "buffer": "^6.0.3" + } + }, "node_modules/@segment/loosely-validate-event": { "version": "2.0.0", "dev": true, @@ -5739,9 +5749,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "dev": true, "license": "MIT", "optional": true, @@ -6995,6 +7002,15 @@ "node": ">=0.4.0" } }, + "node_modules/adm-zip": { + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.16.tgz", + "integrity": "sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==", + "license": "MIT", + "engines": { + "node": ">=12.0" + } + }, "node_modules/agent-base": { "version": "6.0.2", "dev": true, @@ -7157,6 +7173,15 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/app-root-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/app-root-path/-/app-root-path-3.1.0.tgz", + "integrity": "sha512-biN3PwB2gUtjaYy/isrU3aNWI5w+fAfvHkSvCKeQGxhmYpwKFUxudR3Yya+KqVRHBmEDYh+/lTozYCFbmzX4nA==", + "license": "MIT", + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/arg": { "version": "5.0.2", "license": "MIT" @@ -7791,6 +7816,13 @@ "version": "1.0.0", "license": "ISC" }, + "node_modules/boolean": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", + "integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "license": "MIT" + }, "node_modules/bplist-creator": { "version": "0.1.0", "license": "MIT", @@ -8879,6 +8911,12 @@ "node": ">=8" } }, + "node_modules/detect-node": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", + "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", + "license": "MIT" + }, "node_modules/detect-node-es": { "version": "1.1.0", "license": "MIT" @@ -9741,6 +9779,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/es6-error": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", + "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", + "license": "MIT" + }, "node_modules/esbuild": { "version": "0.25.11", "hasInstallScript": true, @@ -11752,6 +11796,62 @@ "version": "0.4.1", "license": "BSD-2-Clause" }, + "node_modules/global-agent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz", + "integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==", + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "es6-error": "^4.1.1", + "matcher": "^3.0.0", + "roarr": "^2.15.3", + "semver": "^7.3.2", + "serialize-error": "^7.0.1" + }, + "engines": { + "node": ">=10.0" + } + }, + "node_modules/global-agent/node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/global-agent/node_modules/serialize-error": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", + "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==", + "license": "MIT", + "dependencies": { + "type-fest": "^0.13.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/global-agent/node_modules/type-fest": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", + "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/global-dirs": { "version": "0.1.1", "license": "MIT", @@ -11775,7 +11875,6 @@ }, "node_modules/globalthis": { "version": "1.0.4", - "dev": true, "license": "MIT", "dependencies": { "define-properties": "^1.2.1", @@ -13075,6 +13174,12 @@ "dev": true, "license": "MIT" }, + "node_modules/json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", + "license": "ISC" + }, "node_modules/json5": { "version": "2.2.3", "license": "MIT", @@ -13397,9 +13502,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -13560,6 +13662,18 @@ "version": "1.3.0", "license": "Apache-2.0" }, + "node_modules/matcher": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", + "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==", + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/math-intrinsics": { "version": "1.1.0", "license": "MIT", @@ -14755,6 +14869,29 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/onnxruntime-common": { + "version": "1.24.1", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.24.1.tgz", + "integrity": "sha512-UnV15u4p4XxoIV+jFP4hXPsW93s3QrwLSpi20HUDYHoTfI4z4sjzex3L4XDOxGGZJ/M/catrwAG2go958UQq0w==", + "license": "MIT" + }, + "node_modules/onnxruntime-node": { + "version": "1.24.1", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.24.1.tgz", + "integrity": "sha512-Ex/oUXKdhDoxvlNxBT3oYtW0MH88yYpPlXQeVQUXpcJQmN24usd/8RCoPLN5kCHwDsiZ+nqsnjciyFRl423dQw==", + "hasInstallScript": true, + "license": "MIT", + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "adm-zip": "^0.5.16", + "global-agent": "^3.0.0", + "onnxruntime-common": "1.24.1" + } + }, "node_modules/open": { "version": "8.4.2", "license": "MIT", @@ -16785,6 +16922,29 @@ "node": "*" } }, + "node_modules/roarr": { + "version": "2.15.4", + "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz", + "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==", + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "detect-node": "^2.0.4", + "globalthis": "^1.0.1", + "json-stringify-safe": "^5.0.1", + "semver-compare": "^1.0.0", + "sprintf-js": "^1.1.2" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/roarr/node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", + "license": "BSD-3-Clause" + }, "node_modules/rollup": { "version": "4.52.5", "license": "MIT", @@ -16992,6 +17152,12 @@ "semver": "bin/semver.js" } }, + "node_modules/semver-compare": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", + "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==", + "license": "MIT" + }, "node_modules/send": { "version": "0.19.1", "license": "MIT", @@ -17312,6 +17478,91 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/sherpa-onnx": { + "version": "1.12.23", + "resolved": "https://registry.npmjs.org/sherpa-onnx/-/sherpa-onnx-1.12.23.tgz", + "integrity": "sha512-QwgDwSIXx4DgbBE7h7VUXEVLO2vf7w3aJX3ng6WoTAcfB2NW7So4Zp4GtGemwgxZ6sfEh02JEVXUidejkX7V0A==", + "license": "Apache-2.0" + }, + "node_modules/sherpa-onnx-darwin-arm64": { + "version": "1.12.23", + "resolved": "https://registry.npmjs.org/sherpa-onnx-darwin-arm64/-/sherpa-onnx-darwin-arm64-1.12.23.tgz", + "integrity": "sha512-zbjNUUH/IXhjRyRJ9mpcWVOGIVr31a/qXBPsfOYc7U8cgwcq33Vmj2OzoLYWQF6T+puqCAE4nMxFAxJvdZekhg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/sherpa-onnx-darwin-x64": { + "version": "1.12.23", + "resolved": "https://registry.npmjs.org/sherpa-onnx-darwin-x64/-/sherpa-onnx-darwin-x64-1.12.23.tgz", + "integrity": "sha512-hgPhBG85bgULJo5xS5xvh1A9+D6GBPwhZ8QCvWvh7YP1kF9UZTF1r5hWBOzZiR2Ax0gQMbxVvw1t/NpR2N7HWA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/sherpa-onnx-linux-x64": { + "version": "1.12.23", + "resolved": "https://registry.npmjs.org/sherpa-onnx-linux-x64/-/sherpa-onnx-linux-x64-1.12.23.tgz", + "integrity": "sha512-pUZIdDvPtyRXQDGo9R9MIBf2AFUzfgcGmutoulsEdH3hpK6JteR7Z/5pfrZIIqe/O99djAjEHK4AlwLHC2jiZw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/sherpa-onnx-node": { + "version": "1.12.23", + "resolved": "https://registry.npmjs.org/sherpa-onnx-node/-/sherpa-onnx-node-1.12.23.tgz", + "integrity": "sha512-09SRixVSjsajxeCV8Hy9R5J4IHPtw7vNgaIcEokdh/LpU7sY+e12z9uHHIMMMgNiInyGEH74wIwjLXms+W7qRA==", + "license": "Apache-2.0", + "optionalDependencies": { + "sherpa-onnx-darwin-arm64": "^1.12.23", + "sherpa-onnx-darwin-x64": "^1.12.23", + "sherpa-onnx-linux-arm64": "^1.12.23", + "sherpa-onnx-linux-x64": "^1.12.23", + "sherpa-onnx-win-ia32": "^1.12.23", + "sherpa-onnx-win-x64": "^1.12.23" + } + }, + "node_modules/sherpa-onnx-win-ia32": { + "version": "1.12.23", + "resolved": "https://registry.npmjs.org/sherpa-onnx-win-ia32/-/sherpa-onnx-win-ia32-1.12.23.tgz", + "integrity": "sha512-MyLsK7r6dd7paglyTgb8UHTXTEFqOzA91u6VDV64Lq8rDGuOFVYioxX7vlwmGe1A9o7VhuOPNaKcRjEPtVDhBQ==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/sherpa-onnx-win-x64": { + "version": "1.12.23", + "resolved": "https://registry.npmjs.org/sherpa-onnx-win-x64/-/sherpa-onnx-win-x64-1.12.23.tgz", + "integrity": "sha512-pdHEYMJiYy8+xzH2WkBVS4/hnRwqjY8FaWnjs0NBgQZnPmc/k4M+TAiauTOuFDNK4GPwFQnjwrCGx6jI9AOkOg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/side-channel": { "version": "1.1.0", "license": "MIT", @@ -20036,6 +20287,7 @@ "@modelcontextprotocol/sdk": "^1.20.1", "@opencode-ai/sdk": "^1.1.12", "@openrouter/ai-sdk-provider": "^1.2.0", + "@sctg/sentencepiece-js": "^1.1.0", "@xterm/headless": "^6.0.0", "ai": "^5.0.76", "ajv": "^8.17.1", @@ -20044,11 +20296,14 @@ "express-basic-auth": "^1.2.1", "mnemonic-id": "^3.2.7", "node-pty": "^1.0.0", + "onnxruntime-node": "^1.23.0", "openai": "^4.20.0", "pino": "^10.2.0", "pino-pretty": "^13.1.3", "playwright": "^1.56.1", "qrcode": "^1.5.4", + "sherpa-onnx": "^1.12.23", + "sherpa-onnx-node": "^1.12.23", "strip-ansi": "^7.1.2", "tiny-invariant": "^1.3.3", "uuid": "^9.0.1", diff --git a/packages/app/e2e/fixtures/recording.baseline.txt b/packages/app/e2e/fixtures/recording.baseline.txt new file mode 100644 index 000000000..4724395dc --- /dev/null +++ b/packages/app/e2e/fixtures/recording.baseline.txt @@ -0,0 +1,2 @@ +This is a voice note. + diff --git a/packages/app/src/hooks/use-audio-player.native.ts b/packages/app/src/hooks/use-audio-player.native.ts index 4729c9b9c..5e7b94ba9 100644 --- a/packages/app/src/hooks/use-audio-player.native.ts +++ b/packages/app/src/hooks/use-audio-player.native.ts @@ -14,32 +14,60 @@ interface QueuedAudio { } /** - * Resample PCM16 audio from 24kHz to 16kHz - * OpenAI returns 24kHz, Speechmatics expects 16kHz + * Resample PCM16 audio between sample rates. + * Speechmatics expects 16kHz. */ -function resamplePcm24kTo16k(pcm24k: Uint8Array): Uint8Array { - // PCM16 = 2 bytes per sample - const samples24k = pcm24k.length / 2; - const samples16k = Math.floor((samples24k * 16000) / 24000); - - const pcm16k = new Uint8Array(samples16k * 2); - const ratio = 24000 / 16000; // 1.5 - - for (let i = 0; i < samples16k; i++) { - const srcIndex = Math.floor(i * ratio) * 2; - if (srcIndex + 1 < pcm24k.length) { - pcm16k[i * 2] = pcm24k[srcIndex]; - pcm16k[i * 2 + 1] = pcm24k[srcIndex + 1]; - } +function resamplePcm16(pcm: Uint8Array, fromRate: number, toRate: number): Uint8Array { + if (fromRate === toRate) { + return pcm; } - console.log("[AudioPlayer] Resampled PCM:", { - input24k: pcm24k.length, - output16k: pcm16k.length, - durationMs: samples16k / 16, - }); + const inputSamples = Math.floor(pcm.length / 2); + const outputSamples = Math.floor((inputSamples * toRate) / fromRate); + const out = new Uint8Array(outputSamples * 2); - return pcm16k; + const ratio = fromRate / toRate; + + const readInt16 = (sampleIndex: number): number => { + const i = sampleIndex * 2; + if (i + 1 >= pcm.length) { + return 0; + } + const lo = pcm[i]!; + const hi = pcm[i + 1]!; + let value = (hi << 8) | lo; + if (value & 0x8000) { + value = value - 0x10000; + } + return value; + }; + + const writeInt16 = (sampleIndex: number, value: number): void => { + const clamped = Math.max(-32768, Math.min(32767, Math.round(value))); + const i = sampleIndex * 2; + out[i] = clamped & 0xff; + out[i + 1] = (clamped >> 8) & 0xff; + }; + + for (let i = 0; i < outputSamples; i++) { + const srcPos = i * ratio; + const i0 = Math.floor(srcPos); + const frac = srcPos - i0; + const s0 = readInt16(i0); + const s1 = readInt16(Math.min(inputSamples - 1, i0 + 1)); + writeInt16(i, s0 + (s1 - s0) * frac); + } + + return out; +} + +function parsePcmSampleRate(mimeType: string): number | null { + const match = /rate=(\d+)/i.exec(mimeType); + if (!match) { + return null; + } + const rate = Number(match[1]); + return Number.isFinite(rate) && rate > 0 ? rate : null; } export interface AudioPlayerOptions { @@ -204,12 +232,12 @@ export function useAudioPlayer(options?: AudioPlayerOptions) { console.log("[AudioPlayer] Resuming playback engine..."); resumePlayback(); - // Get PCM data from blob (server now sends PCM format) + // Get PCM data from blob (server sends PCM16) const arrayBuffer = await audioData.arrayBuffer(); - let pcm24k = new Uint8Array(arrayBuffer); + const pcm = new Uint8Array(arrayBuffer); - // Resample from 24kHz (OpenAI) to 16kHz (Speechmatics) - const pcm16k = resamplePcm24kTo16k(pcm24k); + const inputRate = parsePcmSampleRate(audioData.type || "") ?? 24000; + const pcm16k = resamplePcm16(pcm, inputRate, 16000); // Calculate total duration const samples = pcm16k.length / 2; // 16-bit = 2 bytes per sample diff --git a/packages/server/package.json b/packages/server/package.json index 10b42ac31..88293bf86 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -14,6 +14,8 @@ "start": "NODE_ENV=production node dist/server/index.js", "typecheck": "tsc -p tsconfig.server.typecheck.json --noEmit", "generate:config-schema": "tsx scripts/generate-config-schema.ts", + "speech:models": "tsx scripts/list-speech-models.ts", + "speech:download": "tsx scripts/download-speech-models.ts", "test": "vitest run", "test:watch": "vitest", "test:ui": "vitest --ui", @@ -48,6 +50,10 @@ "pino-pretty": "^13.1.3", "playwright": "^1.56.1", "qrcode": "^1.5.4", + "@sctg/sentencepiece-js": "^1.1.0", + "onnxruntime-node": "^1.23.0", + "sherpa-onnx": "^1.12.23", + "sherpa-onnx-node": "^1.12.23", "strip-ansi": "^7.1.2", "tiny-invariant": "^1.3.3", "uuid": "^9.0.1", diff --git a/packages/server/scripts/download-speech-models.ts b/packages/server/scripts/download-speech-models.ts new file mode 100644 index 000000000..978cc3e85 --- /dev/null +++ b/packages/server/scripts/download-speech-models.ts @@ -0,0 +1,42 @@ +import { resolvePaseoHome } from "../src/server/paseo-home.js"; +import { createRootLogger } from "../src/server/logger.js"; +import { ensureSherpaOnnxModels } from "../src/server/speech/sherpa/model-downloader.js"; +import type { SherpaOnnxModelId } from "../src/server/speech/sherpa/model-catalog.js"; + +function parseArgs(argv: string[]): { modelsDir: string; modelIds: SherpaOnnxModelId[] } { + const home = resolvePaseoHome(); + let modelsDir = process.env.PASEO_SHERPA_ONNX_MODELS_DIR || `${home}/models/sherpa-onnx`; + const modelIds: SherpaOnnxModelId[] = []; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--models-dir") { + modelsDir = argv[i + 1] ?? modelsDir; + i++; + continue; + } + if (arg === "--model") { + const id = argv[i + 1] as SherpaOnnxModelId | undefined; + if (!id) { + throw new Error("--model requires a value"); + } + modelIds.push(id); + i++; + continue; + } + } + + if (modelIds.length === 0) { + const stt = (process.env.PASEO_SHERPA_STT_PRESET || "zipformer-bilingual-zh-en-2023-02-20") as SherpaOnnxModelId; + const tts = (process.env.PASEO_SHERPA_TTS_PRESET || "pocket-tts-onnx-int8") as SherpaOnnxModelId; + modelIds.push(stt, tts); + } + + return { modelsDir, modelIds }; +} + +const logger = createRootLogger({ level: "info", format: "pretty" }); + +const { modelsDir, modelIds } = parseArgs(process.argv.slice(2)); +await ensureSherpaOnnxModels({ modelsDir, modelIds, autoDownload: true, logger }); +logger.info({ modelsDir, modelIds }, "Done downloading speech models"); diff --git a/packages/server/scripts/list-speech-models.ts b/packages/server/scripts/list-speech-models.ts new file mode 100644 index 000000000..d907ab4eb --- /dev/null +++ b/packages/server/scripts/list-speech-models.ts @@ -0,0 +1,11 @@ +import { listSherpaOnnxModels } from "../src/server/speech/sherpa/model-catalog.js"; + +const models = listSherpaOnnxModels() + .slice() + .sort((a, b) => a.kind.localeCompare(b.kind) || a.id.localeCompare(b.id)); + +for (const m of models) { + // eslint-disable-next-line no-console + console.log(`${m.kind}\t${m.id}\t${m.description}`); +} + diff --git a/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts b/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts index 5070c1337..799bbc95e 100644 --- a/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts +++ b/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts @@ -169,7 +169,7 @@ describe("Codex app-server provider (integration)", () => { cleanup(); rmSync(cwd, { recursive: true, force: true }); } - }, 60000); + }, 120000); test.runIf(isCodexInstalled())("listCommands includes custom prompts and executeCommand runs them", async () => { const cleanup = useTempCodexSessionDir(); @@ -730,7 +730,9 @@ describe("Codex app-server provider (integration)", () => { if (captured) { expect(sawPermissionResolved).toBe(true); } - expect(readFileSync(targetPath, "utf8").trim()).toBe("ok"); + const text = await waitForFileToContainText(targetPath, "ok", { timeoutMs: 5000 }); + expect(timelineItems.some((item) => hasApplyPatchFile(item, "approval-test.txt"))).toBe(true); + expect(text?.trim()).toBe("ok"); } finally { cleanup(); rmSync(cwd, { recursive: true, force: true }); diff --git a/packages/server/src/server/agent/stt-manager.test.ts b/packages/server/src/server/agent/stt-manager.test.ts new file mode 100644 index 000000000..df88f3bea --- /dev/null +++ b/packages/server/src/server/agent/stt-manager.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import pino from "pino"; + +import { STTManager } from "./stt-manager.js"; +import type { SpeechToTextProvider, TranscriptionResult } from "../speech/speech-provider.js"; + +class FakeStt implements SpeechToTextProvider { + constructor(private readonly result: TranscriptionResult) {} + + async transcribeAudio(): Promise { + return this.result; + } +} + +describe("STTManager", () => { + it("returns empty text for low-confidence transcriptions", async () => { + const manager = new STTManager( + "s1", + pino({ level: "silent" }), + new FakeStt({ text: "um", isLowConfidence: true, avgLogprob: -10 }) + ); + + const result = await manager.transcribe(Buffer.from("x"), "audio/wav", { label: "t" }); + expect(result.text).toBe(""); + expect(result.isLowConfidence).toBe(true); + expect(result.byteLength).toBe(1); + }); + + it("passes through normal transcriptions", async () => { + const manager = new STTManager( + "s1", + pino({ level: "silent" }), + new FakeStt({ text: "hello world", language: "en", isLowConfidence: false }) + ); + + const result = await manager.transcribe(Buffer.from("abc"), "audio/wav"); + expect(result.text).toBe("hello world"); + expect(result.language).toBe("en"); + expect(result.byteLength).toBe(3); + }); +}); + diff --git a/packages/server/src/server/agent/stt-manager.ts b/packages/server/src/server/agent/stt-manager.ts index 4abb6ea15..cb454dc3b 100644 --- a/packages/server/src/server/agent/stt-manager.ts +++ b/packages/server/src/server/agent/stt-manager.ts @@ -1,5 +1,5 @@ import type pino from "pino"; -import type { OpenAISTT, TranscriptionResult } from "./stt-openai.js"; +import type { SpeechToTextProvider, TranscriptionResult } from "../speech/speech-provider.js"; import { maybePersistDebugAudio } from "./stt-debug.js"; interface TranscriptionMetadata { @@ -21,9 +21,9 @@ export interface SessionTranscriptionResult extends TranscriptionResult { export class STTManager { private readonly sessionId: string; private readonly logger: pino.Logger; - private readonly stt: OpenAISTT | null; + private readonly stt: SpeechToTextProvider | null; - constructor(sessionId: string, logger: pino.Logger, stt: OpenAISTT | null) { + constructor(sessionId: string, logger: pino.Logger, stt: SpeechToTextProvider | null) { this.sessionId = sessionId; this.logger = logger.child({ module: "agent", component: "stt-manager", sessionId }); this.stt = stt; diff --git a/packages/server/src/server/agent/stt-openai.ts b/packages/server/src/server/agent/stt-openai.ts index 5e70c6a81..4326f6c68 100644 --- a/packages/server/src/server/agent/stt-openai.ts +++ b/packages/server/src/server/agent/stt-openai.ts @@ -5,6 +5,9 @@ import { join } from "path"; import { tmpdir } from "os"; import { v4 } from "uuid"; import { inferAudioExtension } from "./audio-utils.js"; +import type { LogprobToken, TranscriptionResult } from "../speech/speech-provider.js"; + +export type { LogprobToken, TranscriptionResult }; export interface STTConfig { apiKey: string; @@ -12,21 +15,6 @@ export interface STTConfig { confidenceThreshold?: number; // Default: -3.0 } -export interface LogprobToken { - token: string; - logprob: number; - bytes?: number[]; -} - -export interface TranscriptionResult { - text: string; - language?: string; - duration?: number; - logprobs?: LogprobToken[]; - avgLogprob?: number; - isLowConfidence?: boolean; -} - function isObject(value: unknown): value is { [key: string]: unknown } { return typeof value === "object" && value !== null; } diff --git a/packages/server/src/server/agent/tts-manager.test.ts b/packages/server/src/server/agent/tts-manager.test.ts new file mode 100644 index 000000000..5c0dea3be --- /dev/null +++ b/packages/server/src/server/agent/tts-manager.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import pino from "pino"; +import { Readable } from "node:stream"; + +import { TTSManager } from "./tts-manager.js"; +import type { TextToSpeechProvider } from "../speech/speech-provider.js"; +import type { SessionOutboundMessage } from "../messages.js"; + +class FakeTts implements TextToSpeechProvider { + async synthesizeSpeech(): Promise<{ stream: Readable; format: string }> { + return { + stream: Readable.from([Buffer.from("a"), Buffer.from("b")]), + format: "pcm;rate=24000", + }; + } +} + +describe("TTSManager", () => { + it("emits chunks and resolves once confirmed", async () => { + const manager = new TTSManager("s1", pino({ level: "silent" }), new FakeTts()); + const abort = new AbortController(); + const emitted: SessionOutboundMessage[] = []; + + const task = manager.generateAndWaitForPlayback( + "hello", + (msg) => { + emitted.push(msg); + if (msg.type === "audio_output") { + manager.confirmAudioPlayed(msg.payload.id); + } + }, + abort.signal, + true + ); + + await task; + + const audioMsgs = emitted.filter((m) => m.type === "audio_output"); + expect(audioMsgs).toHaveLength(2); + const groupId = (audioMsgs[0] as any).payload.groupId; + expect(groupId).toBeTruthy(); + expect((audioMsgs[0] as any).payload.chunkIndex).toBe(0); + expect((audioMsgs[1] as any).payload.chunkIndex).toBe(1); + expect((audioMsgs[1] as any).payload.isLastChunk).toBe(true); + }); +}); + diff --git a/packages/server/src/server/agent/tts-manager.ts b/packages/server/src/server/agent/tts-manager.ts index 2b0491a52..083c94096 100644 --- a/packages/server/src/server/agent/tts-manager.ts +++ b/packages/server/src/server/agent/tts-manager.ts @@ -1,6 +1,6 @@ import type pino from "pino"; import { v4 as uuidv4 } from "uuid"; -import type { OpenAITTS } from "./tts-openai.js"; +import type { TextToSpeechProvider } from "../speech/speech-provider.js"; import type { SessionOutboundMessage } from "../messages.js"; interface PendingPlayback { @@ -17,9 +17,9 @@ interface PendingPlayback { export class TTSManager { private pendingPlaybacks: Map = new Map(); private readonly logger: pino.Logger; - private readonly tts: OpenAITTS | null; + private readonly tts: TextToSpeechProvider | null; - constructor(sessionId: string, logger: pino.Logger, tts: OpenAITTS | null) { + constructor(sessionId: string, logger: pino.Logger, tts: TextToSpeechProvider | null) { this.logger = logger.child({ module: "agent", component: "tts-manager", sessionId }); this.tts = tts; } diff --git a/packages/server/src/server/agent/tts-openai.ts b/packages/server/src/server/agent/tts-openai.ts index efe4e4ca0..bd96a80b4 100644 --- a/packages/server/src/server/agent/tts-openai.ts +++ b/packages/server/src/server/agent/tts-openai.ts @@ -1,6 +1,9 @@ import type pino from "pino"; import OpenAI from "openai"; -import { Readable } from "stream"; +import { Readable } from "node:stream"; +import type { SpeechStreamResult } from "../speech/speech-provider.js"; + +export type { SpeechStreamResult }; export interface TTSConfig { apiKey: string; @@ -9,11 +12,6 @@ export interface TTSConfig { responseFormat?: "mp3" | "opus" | "aac" | "flac" | "wav" | "pcm"; } -export interface SpeechStreamResult { - stream: Readable; - format: string; -} - export class OpenAITTS { private readonly openaiClient: OpenAI; private readonly config: TTSConfig; diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 92f13cff7..b47deb61f 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -42,6 +42,18 @@ import { VoiceAssistantWebSocketServer } from "./websocket-server.js"; import { DownloadTokenStore } from "./file-download/token-store.js"; import { OpenAISTT, type STTConfig } from "./agent/stt-openai.js"; import { OpenAITTS, type TTSConfig } from "./agent/tts-openai.js"; +import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js"; +import type { RealtimeTranscriptionSessionFactory } from "./dictation/dictation-stream-manager.js"; +import { SherpaOnlineRecognizerEngine } from "./speech/sherpa/sherpa-online-recognizer.js"; +import { SherpaOfflineRecognizerEngine } from "./speech/sherpa/sherpa-offline-recognizer.js"; +import { SherpaOnnxSTT } from "./speech/sherpa/sherpa-stt.js"; +import { SherpaOnnxParakeetSTT } from "./speech/sherpa/sherpa-parakeet-stt.js"; +import { SherpaOnnxTTS } from "./speech/sherpa/sherpa-tts.js"; +import { SherpaRealtimeTranscriptionSession } from "./speech/sherpa/sherpa-realtime-session.js"; +import { SherpaParakeetRealtimeTranscriptionSession } from "./speech/sherpa/sherpa-parakeet-realtime-session.js"; +import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./speech/sherpa/model-downloader.js"; +import type { SherpaOnnxModelId } from "./speech/sherpa/model-catalog.js"; +import { PocketTtsOnnxTTS } from "./speech/pocket/pocket-tts-onnx.js"; import { AgentManager } from "./agent/agent-manager.js"; import { AgentStorage } from "./agent/agent-storage.js"; import { attachAgentStoragePersistence } from "./persistence-hooks.js"; @@ -72,6 +84,26 @@ export type PaseoOpenAIConfig = { tts?: Partial & { apiKey?: string }; }; +export type PaseoSherpaOnnxConfig = { + modelsDir: string; + autoDownload?: boolean; + stt?: { + preset?: string; + }; + tts?: { + preset?: string; + speakerId?: number; + speed?: number; + }; +}; + +export type PaseoSpeechConfig = { + dictationSttProvider?: "openai" | "sherpa"; + voiceSttProvider?: "openai" | "sherpa"; + voiceTtsProvider?: "openai" | "sherpa"; + sherpaOnnx?: PaseoSherpaOnnxConfig; +}; + export type PaseoDaemonConfig = { listen: string; paseoHome: string; @@ -87,6 +119,7 @@ export type PaseoDaemonConfig = { relayPublicEndpoint?: string; appBaseUrl?: string; openai?: PaseoOpenAIConfig; + speech?: PaseoSpeechConfig; openrouterApiKey?: string | null; voiceLlmModel?: string | null; dictationFinalTimeoutMs?: number; @@ -382,42 +415,282 @@ export async function createPaseoDaemon( } - let sttService: OpenAISTT | null = null; - let ttsService: OpenAITTS | null = null; + let sttService: SpeechToTextProvider | null = null; + let ttsService: TextToSpeechProvider | null = null; + let dictationSessionFactory: RealtimeTranscriptionSessionFactory | undefined; + + let sherpaOnline: SherpaOnlineRecognizerEngine | null = null; + let sherpaOffline: SherpaOfflineRecognizerEngine | null = null; + let sherpaTts: TextToSpeechProvider | null = null; const openaiApiKey = config.openai?.apiKey; - if (openaiApiKey) { + const speechConfig = config.speech ?? null; + const sherpaConfig = speechConfig?.sherpaOnnx ?? null; + + const wantsSherpaDictation = (speechConfig?.dictationSttProvider ?? "openai") === "sherpa"; + const wantsSherpaVoiceStt = (speechConfig?.voiceSttProvider ?? "openai") === "sherpa"; + const wantsSherpaVoiceTts = (speechConfig?.voiceTtsProvider ?? "openai") === "sherpa"; + + if ((wantsSherpaDictation || wantsSherpaVoiceStt || wantsSherpaVoiceTts) && sherpaConfig) { + const autoDownload = sherpaConfig.autoDownload ?? (process.env.VITEST ? false : true); + let sttPreset = (sherpaConfig.stt?.preset ?? "zipformer-bilingual-zh-en-2023-02-20").trim(); + if ( + sttPreset !== "zipformer-bilingual-zh-en-2023-02-20" && + sttPreset !== "paraformer-bilingual-zh-en" && + sttPreset !== "parakeet-tdt-0.6b-v3-int8" + ) { + logger.warn( + { sttPreset }, + "Unknown Sherpa STT preset; falling back to zipformer-bilingual-zh-en-2023-02-20" + ); + sttPreset = "zipformer-bilingual-zh-en-2023-02-20"; + } + + let ttsPreset = (sherpaConfig.tts?.preset ?? "pocket-tts-onnx-int8").trim(); + if ( + ttsPreset !== "kitten-nano-en-v0_1-fp16" && + ttsPreset !== "kokoro-en-v0_19" && + ttsPreset !== "pocket-tts-onnx-int8" + ) { + logger.warn( + { ttsPreset }, + "Unknown Sherpa TTS preset; falling back to kitten-nano-en-v0_1-fp16" + ); + ttsPreset = "kitten-nano-en-v0_1-fp16"; + } + + const modelIds: SherpaOnnxModelId[] = []; + if (wantsSherpaDictation || wantsSherpaVoiceStt) { + modelIds.push(sttPreset as SherpaOnnxModelId); + } + if (wantsSherpaVoiceTts) { + modelIds.push(ttsPreset as SherpaOnnxModelId); + } + + try { + await ensureSherpaOnnxModels({ + modelsDir: sherpaConfig.modelsDir, + modelIds, + autoDownload, + logger, + }); + } catch (err) { + logger.error( + { + err, + modelsDir: sherpaConfig.modelsDir, + autoDownload, + hint: + "Run: npm run dev --workspace=@getpaseo/server, then run: " + + "`tsx packages/server/scripts/download-speech-models.ts --models-dir --model `", + }, + "Failed to ensure local speech models" + ); + } + } + + if ((wantsSherpaDictation || wantsSherpaVoiceStt) && sherpaConfig) { + let preset = (sherpaConfig.stt?.preset ?? "zipformer-bilingual-zh-en-2023-02-20").trim(); + if ( + preset !== "zipformer-bilingual-zh-en-2023-02-20" && + preset !== "paraformer-bilingual-zh-en" && + preset !== "parakeet-tdt-0.6b-v3-int8" + ) { + logger.warn( + { preset }, + "Unknown Sherpa STT preset; falling back to zipformer-bilingual-zh-en-2023-02-20" + ); + preset = "zipformer-bilingual-zh-en-2023-02-20"; + } + const base = sherpaConfig.modelsDir; + + try { + if (preset === "parakeet-tdt-0.6b-v3-int8") { + const modelDir = getSherpaOnnxModelDir(base, "parakeet-tdt-0.6b-v3-int8"); + sherpaOffline = new SherpaOfflineRecognizerEngine( + { + model: { + kind: "nemo_transducer", + encoder: `${modelDir}/encoder.int8.onnx`, + decoder: `${modelDir}/decoder.int8.onnx`, + joiner: `${modelDir}/joiner.int8.onnx`, + tokens: `${modelDir}/tokens.txt`, + }, + numThreads: 2, + debug: 0, + }, + logger + ); + } else { + const model = + preset === "paraformer-bilingual-zh-en" + ? { + kind: "paraformer" as const, + encoder: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/encoder.int8.onnx`, + decoder: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/decoder.int8.onnx`, + tokens: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/tokens.txt`, + } + : { + kind: "transducer" as const, + encoder: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/encoder-epoch-99-avg-1.onnx`, + decoder: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/decoder-epoch-99-avg-1.onnx`, + joiner: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/joiner-epoch-99-avg-1.onnx`, + tokens: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt`, + modelType: "zipformer", + }; + + sherpaOnline = new SherpaOnlineRecognizerEngine( + { + model, + numThreads: 1, + debug: 0, + }, + logger + ); + } + } catch (err) { + logger.error( + { + err, + modelsDir: sherpaConfig.modelsDir, + preset, + hint: `Run: tsx packages/server/scripts/download-speech-models.ts --models-dir '${sherpaConfig.modelsDir}' --model '${preset}'`, + }, + "Failed to initialize Sherpa STT (models missing or invalid)" + ); + sherpaOnline = null; + sherpaOffline = null; + } + } else if (wantsSherpaDictation || wantsSherpaVoiceStt) { + logger.warn( + { configured: Boolean(sherpaConfig) }, + "Sherpa STT selected but no sherpaOnnx config found; STT will be unavailable" + ); + } + + if (wantsSherpaVoiceTts && sherpaConfig) { + let preset = (sherpaConfig.tts?.preset ?? "pocket-tts-onnx-int8").trim(); + if ( + preset !== "kitten-nano-en-v0_1-fp16" && + preset !== "kokoro-en-v0_19" && + preset !== "pocket-tts-onnx-int8" + ) { + logger.warn( + { preset }, + "Unknown Sherpa TTS preset; falling back to kitten-nano-en-v0_1-fp16" + ); + preset = "kitten-nano-en-v0_1-fp16"; + } + try { + if (preset === "pocket-tts-onnx-int8") { + const modelDir = getSherpaOnnxModelDir(sherpaConfig.modelsDir, "pocket-tts-onnx-int8"); + sherpaTts = await PocketTtsOnnxTTS.create( + { + modelDir, + precision: "int8", + targetChunkMs: 50, + }, + logger + ); + } else { + const modelDir = `${sherpaConfig.modelsDir}/${preset}`; + sherpaTts = new SherpaOnnxTTS( + { + preset: preset as any, + modelDir, + speakerId: sherpaConfig.tts?.speakerId, + speed: sherpaConfig.tts?.speed, + }, + logger + ); + } + } catch (err) { + logger.error( + { + err, + preset, + hint: `Run: tsx packages/server/scripts/download-speech-models.ts --models-dir '${sherpaConfig.modelsDir}' --model '${preset}'`, + }, + "Failed to initialize Sherpa TTS (models missing or invalid)" + ); + sherpaTts = null; + } + } else if (wantsSherpaVoiceTts) { + logger.warn( + { configured: Boolean(sherpaConfig) }, + "Sherpa TTS selected but no sherpaOnnx config found; TTS will be unavailable" + ); + } + + if (wantsSherpaVoiceStt && sherpaOffline) { + sttService = new SherpaOnnxParakeetSTT({ engine: sherpaOffline }, logger); + } else if (wantsSherpaVoiceStt && sherpaOnline) { + sttService = new SherpaOnnxSTT({ engine: sherpaOnline }, logger); + } + + if (wantsSherpaVoiceTts && sherpaTts) { + ttsService = sherpaTts; + } + + if (wantsSherpaDictation && sherpaOnline) { + dictationSessionFactory = () => new SherpaRealtimeTranscriptionSession({ engine: sherpaOnline! }); + } else if (wantsSherpaDictation && sherpaOffline) { + dictationSessionFactory = () => new SherpaParakeetRealtimeTranscriptionSession({ engine: sherpaOffline! }); + } + + const voiceSttProvider = speechConfig?.voiceSttProvider ?? "openai"; + const voiceTtsProvider = speechConfig?.voiceTtsProvider ?? "openai"; + const dictationSttProvider = speechConfig?.dictationSttProvider ?? "openai"; + + const needsOpenAiStt = !sttService && voiceSttProvider === "openai"; + const needsOpenAiTts = !ttsService && voiceTtsProvider === "openai"; + const needsOpenAiDictation = + dictationSttProvider === "openai" || (dictationSttProvider === "sherpa" && !dictationSessionFactory); + + const fallbackOpenAiStt = !sttService && voiceSttProvider === "sherpa" && Boolean(openaiApiKey); + const fallbackOpenAiTts = !ttsService && voiceTtsProvider === "sherpa" && Boolean(openaiApiKey); + + if ((needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation || fallbackOpenAiStt || fallbackOpenAiTts) && openaiApiKey) { logger.info("OpenAI client initialized"); - const sttApiKey = config.openai?.stt?.apiKey ?? openaiApiKey; - if (sttApiKey) { - const { apiKey: _sttApiKey, ...sttConfig } = config.openai?.stt ?? {}; - sttService = new OpenAISTT( - { - apiKey: sttApiKey, - ...sttConfig, - }, - logger - ); + if (fallbackOpenAiStt) { + logger.warn("Falling back to OpenAI STT because Sherpa STT is unavailable"); + } + if (needsOpenAiStt || fallbackOpenAiStt) { + const sttApiKey = config.openai?.stt?.apiKey ?? openaiApiKey; + if (sttApiKey) { + const { apiKey: _sttApiKey, ...sttConfig } = config.openai?.stt ?? {}; + sttService = new OpenAISTT( + { + apiKey: sttApiKey, + ...sttConfig, + }, + logger + ); + } } - const ttsApiKey = config.openai?.tts?.apiKey ?? openaiApiKey; - if (ttsApiKey) { - const { apiKey: _ttsApiKey, ...ttsConfig } = config.openai?.tts ?? {}; - ttsService = new OpenAITTS( - { - apiKey: ttsApiKey, - voice: "alloy", - model: "tts-1", - responseFormat: "pcm", - ...ttsConfig, - }, - logger - ); + if (fallbackOpenAiTts) { + logger.warn("Falling back to OpenAI TTS because Sherpa TTS is unavailable"); } - - } else { - logger.warn("OPENAI_API_KEY not set - LLM, STT, and TTS features will not work"); + if (needsOpenAiTts || fallbackOpenAiTts) { + const ttsApiKey = config.openai?.tts?.apiKey ?? openaiApiKey; + if (ttsApiKey) { + const { apiKey: _ttsApiKey, ...ttsConfig } = config.openai?.tts ?? {}; + ttsService = new OpenAITTS( + { + apiKey: ttsApiKey, + voice: "alloy", + model: "tts-1", + responseFormat: "pcm", + ...ttsConfig, + }, + logger + ); + } + } + } else if (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation || fallbackOpenAiStt || fallbackOpenAiTts) { + logger.warn("OPENAI_API_KEY not set - OpenAI STT/TTS/dictation fallback is unavailable"); } const wsServer = new VoiceAssistantWebSocketServer( @@ -439,6 +712,7 @@ export async function createPaseoDaemon( { openaiApiKey: config.openai?.apiKey ?? null, finalTimeoutMs: config.dictationFinalTimeoutMs, + ...(dictationSessionFactory ? { sessionFactory: dictationSessionFactory } : {}), } ); @@ -533,6 +807,11 @@ export async function createPaseoDaemon( await agentStorage.flush().catch(() => undefined); await shutdownProviders(logger); terminalManager.killAll(); + if (sherpaTts && typeof (sherpaTts as any).free === "function") { + (sherpaTts as any).free(); + } + sherpaOnline?.free(); + sherpaOffline?.free(); await relayTransport?.stop().catch(() => undefined); await wsServer.close(); await new Promise((resolve) => { diff --git a/packages/server/src/server/config.ts b/packages/server/src/server/config.ts index 2465e2b82..fca13a681 100644 --- a/packages/server/src/server/config.ts +++ b/packages/server/src/server/config.ts @@ -76,6 +76,51 @@ function parseOpenAIConfig( }; } +function parseSpeechProviderId(value: unknown): "openai" | "sherpa" | null { + if (typeof value !== "string") { + return null; + } + const normalized = value.trim().toLowerCase(); + if (!normalized) { + return null; + } + if (normalized === "openai") return "openai"; + if (normalized === "sherpa" || normalized === "sherpa-onnx" || normalized === "local") { + return "sherpa"; + } + return null; +} + +function normalizeSherpaSttPreset(value: string): string { + const raw = value.trim(); + const normalized = raw.toLowerCase(); + if (normalized === "zipformer" || normalized === "zipformer-bilingual") { + return "zipformer-bilingual-zh-en-2023-02-20"; + } + if (normalized === "paraformer") { + return "paraformer-bilingual-zh-en"; + } + if (normalized === "parakeet" || normalized === "parakeet-v3" || normalized === "parakeet-tdt") { + return "parakeet-tdt-0.6b-v3-int8"; + } + return raw; +} + +function normalizeSherpaTtsPreset(value: string): string { + const raw = value.trim(); + const normalized = raw.toLowerCase(); + if (normalized === "pocket" || normalized === "pocket-tts") { + return "pocket-tts-onnx-int8"; + } + if (normalized === "kitten") { + return "kitten-nano-en-v0_1-fp16"; + } + if (normalized === "kokoro") { + return "kokoro-en-v0_19"; + } + return raw; +} + export function loadConfig( paseoHome: string, options?: { @@ -137,6 +182,70 @@ export function loadConfig( voiceTtsVoice: persisted.features?.voiceMode?.tts?.voice, }); + const dictationSttProvider = + parseSpeechProviderId(env.PASEO_DICTATION_STT_PROVIDER) ?? + parseSpeechProviderId(persisted.features?.dictation?.stt?.provider) ?? + "sherpa"; + + const voiceSttProvider = + parseSpeechProviderId(env.PASEO_VOICE_STT_PROVIDER) ?? + parseSpeechProviderId(persisted.features?.voiceMode?.stt?.provider) ?? + "sherpa"; + + const voiceTtsProvider = + parseSpeechProviderId(env.PASEO_VOICE_TTS_PROVIDER) ?? + parseSpeechProviderId(persisted.features?.voiceMode?.tts?.provider) ?? + "sherpa"; + + const shouldConfigureSherpa = + dictationSttProvider === "sherpa" || + voiceSttProvider === "sherpa" || + voiceTtsProvider === "sherpa" || + typeof env.PASEO_SHERPA_ONNX_MODELS_DIR === "string" || + Boolean(persisted.providers?.sherpaOnnx); + + const sherpaModelsDir = + (env.PASEO_SHERPA_ONNX_MODELS_DIR ?? persisted.providers?.sherpaOnnx?.modelsDir)?.trim() || + path.join(paseoHome, "models", "sherpa-onnx"); + + const sherpaOnnx = shouldConfigureSherpa + ? { + modelsDir: sherpaModelsDir, + autoDownload: + env.PASEO_SHERPA_ONNX_AUTO_DOWNLOAD !== undefined + ? env.PASEO_SHERPA_ONNX_AUTO_DOWNLOAD === "1" + : persisted.providers?.sherpaOnnx?.autoDownload ?? + // In tests we should never hit the network unexpectedly. + Boolean(env.VITEST) === false, + stt: { + preset: normalizeSherpaSttPreset( + (env.PASEO_SHERPA_STT_PRESET ?? persisted.providers?.sherpaOnnx?.stt?.preset)?.trim() || + (persisted.features?.voiceMode?.stt?.preset ?? + persisted.features?.dictation?.stt?.preset)?.trim() || + "zipformer-bilingual-zh-en-2023-02-20" + ), + }, + tts: { + preset: normalizeSherpaTtsPreset( + (env.PASEO_SHERPA_TTS_PRESET ?? + persisted.providers?.sherpaOnnx?.tts?.preset ?? + persisted.features?.voiceMode?.tts?.preset)?.trim() || + (env.VITEST ? "kitten-nano-en-v0_1-fp16" : "pocket-tts-onnx-int8") + ), + speakerId: + env.PASEO_SHERPA_TTS_SPEAKER_ID !== undefined + ? Number.parseInt(env.PASEO_SHERPA_TTS_SPEAKER_ID, 10) + : persisted.providers?.sherpaOnnx?.tts?.speakerId ?? + persisted.features?.voiceMode?.tts?.speakerId, + speed: + env.PASEO_SHERPA_TTS_SPEED !== undefined + ? Number.parseFloat(env.PASEO_SHERPA_TTS_SPEED) + : persisted.providers?.sherpaOnnx?.tts?.speed ?? + persisted.features?.voiceMode?.tts?.speed, + }, + } + : undefined; + const openrouterApiKey = env.OPENROUTER_API_KEY ?? persisted.providers?.openrouter?.apiKey ?? null; const voiceLlmModel = persisted.features?.voiceMode?.llm?.model ?? null; @@ -158,6 +267,12 @@ export function loadConfig( relayPublicEndpoint, appBaseUrl, openai, + speech: { + dictationSttProvider, + voiceSttProvider, + voiceTtsProvider, + ...(sherpaOnnx ? { sherpaOnnx } : {}), + }, openrouterApiKey, voiceLlmModel, }; diff --git a/packages/server/src/server/daemon-client.e2e.test.ts b/packages/server/src/server/daemon-client.e2e.test.ts index d612d10fa..5a253f513 100644 --- a/packages/server/src/server/daemon-client.e2e.test.ts +++ b/packages/server/src/server/daemon-client.e2e.test.ts @@ -1,6 +1,6 @@ import { describe, test, expect, beforeAll, afterAll, beforeEach } from "vitest"; import { mkdtempSync, writeFileSync, rmSync, existsSync } from "node:fs"; -import { tmpdir } from "node:os"; +import { tmpdir, homedir } from "node:os"; import path from "node:path"; import { execSync } from "node:child_process"; @@ -13,12 +13,45 @@ import { getFullAccessConfig, getAskModeConfig } from "./daemon-e2e/agent-config import { chunkPcm16, parsePcm16MonoWav, - requireEnv, - transcribeBaselineOpenAI, wordSimilarity, } from "./test-utils/dictation-e2e.js"; -const hasOpenAICredentials = !!process.env.OPENAI_API_KEY; +const openaiApiKey = process.env.OPENAI_API_KEY ?? null; + +const sherpaModelsDir = + process.env.PASEO_SHERPA_ONNX_MODELS_DIR ?? + path.join(homedir(), ".paseo", "models", "sherpa-onnx"); + +function hasSherpaZipformerModels(modelsDir: string): boolean { + return ( + existsSync( + path.join( + modelsDir, + "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", + "encoder-epoch-99-avg-1.onnx" + ) + ) && + existsSync( + path.join( + modelsDir, + "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", + "tokens.txt" + ) + ) + ); +} + +function hasSherpaKittenModels(modelsDir: string): boolean { + return ( + existsSync(path.join(modelsDir, "kitten-nano-en-v0_1-fp16", "model.fp16.onnx")) && + existsSync(path.join(modelsDir, "kitten-nano-en-v0_1-fp16", "voices.bin")) && + existsSync(path.join(modelsDir, "kitten-nano-en-v0_1-fp16", "tokens.txt")) + ); +} + +const hasLocalSpeech = hasSherpaZipformerModels(sherpaModelsDir) && hasSherpaKittenModels(sherpaModelsDir); +const hasAnySpeech = hasLocalSpeech || Boolean(openaiApiKey); +const speechTest = hasAnySpeech ? test : test.skip; function tmpCwd(): string { return mkdtempSync(path.join(tmpdir(), "daemon-client-")); @@ -55,14 +88,27 @@ function waitForSignal( }); } -(hasOpenAICredentials ? describe : describe.skip)("daemon client E2E", () => { +describe("daemon client E2E", () => { let ctx: DaemonTestContext; beforeAll(async () => { - const openaiApiKey = process.env.OPENAI_API_KEY ?? ""; ctx = await createDaemonTestContext({ dictationFinalTimeoutMs: 5000, - openai: { apiKey: openaiApiKey }, + ...(openaiApiKey ? { openai: { apiKey: openaiApiKey } } : {}), + speech: { + dictationSttProvider: "sherpa", + voiceSttProvider: "sherpa", + voiceTtsProvider: "sherpa", + sherpaOnnx: { + modelsDir: sherpaModelsDir, + stt: { + preset: process.env.PASEO_SHERPA_STT_PRESET ?? "zipformer-bilingual-zh-en-2023-02-20", + }, + tts: { + preset: process.env.PASEO_SHERPA_TTS_PRESET ?? "kitten-nano-en-v0_1-fp16", + }, + }, + }, }); }, 60000); @@ -569,11 +615,9 @@ function waitForSignal( 120000 ); - test( + speechTest( "voice mode buffers audio until isLast and emits transcription_result", async () => { - requireEnv("OPENAI_API_KEY"); - await ctx.client.setVoiceConversation(true, `voice-${Date.now()}`); const transcription = waitForSignal(30_000, (resolve) => { @@ -675,11 +719,9 @@ function waitForSignal( 90_000 ); - test( - "streams dictation PCM and returns final transcript via OpenAI Realtime transcription", + speechTest( + "streams dictation PCM and returns final transcript", async () => { - requireEnv("OPENAI_API_KEY"); - const fixturePath = path.resolve( process.cwd(), "..", @@ -713,11 +755,9 @@ function waitForSignal( 30_000 ); - test( - "realtime dictation transcript is similar to baseline (OpenAI transcriptions API)", + speechTest( + "realtime dictation transcript is similar to baseline fixture", async () => { - const apiKey = requireEnv("OPENAI_API_KEY"); - const fixturePath = path.resolve( process.cwd(), "..", @@ -732,14 +772,17 @@ function waitForSignal( const dictationId = `dict-baseline-${Date.now()}`; const format = "audio/pcm;rate=16000;bits=16"; - const baseline = await transcribeBaselineOpenAI({ - apiKey, - wav, - model: process.env.STT_MODEL ?? "whisper-1", - prompt: - process.env.OPENAI_REALTIME_DICTATION_TRANSCRIPTION_PROMPT ?? - "Transcribe only what the speaker says. Do not add words. Preserve punctuation and casing. If the audio is silence or non-speech noise, return an empty transcript.", - }); + const baselinePath = path.resolve( + process.cwd(), + "..", + "app", + "e2e", + "fixtures", + "recording.baseline.txt" + ); + const baseline = await import("node:fs/promises") + .then((fs) => fs.readFile(baselinePath, "utf-8")) + .then((text) => text.trim()); await ctx.client.startDictationStream(dictationId, format); @@ -755,16 +798,14 @@ function waitForSignal( const result = await ctx.client.finishDictationStream(dictationId, finalSeq); expect(result.dictationId).toBe(dictationId); - expect(wordSimilarity(result.text, baseline)).toBeGreaterThan(0.8); + expect(wordSimilarity(result.text, baseline)).toBeGreaterThan(0.6); }, 30_000 ); - test( + speechTest( "fails fast if dictation finishes without sending required chunks", async () => { - requireEnv("OPENAI_API_KEY"); - const dictationId = `dict-missing-chunks-${Date.now()}`; const format = "audio/pcm;rate=16000;bits=16"; diff --git a/packages/server/src/server/dictation/dictation-stream-manager.test.ts b/packages/server/src/server/dictation/dictation-stream-manager.test.ts index c0581942c..830739d74 100644 --- a/packages/server/src/server/dictation/dictation-stream-manager.test.ts +++ b/packages/server/src/server/dictation/dictation-stream-manager.test.ts @@ -151,3 +151,34 @@ describe("DictationStreamManager (semantic VAD grace fallback)", () => { expect(final?.payload.text).toBe("hi there"); }); }); + +describe("DictationStreamManager (provider-agnostic session factory)", () => { + it("can start with a custom session factory even without OPENAI_API_KEY", async () => { + const original = process.env.OPENAI_API_KEY; + delete process.env.OPENAI_API_KEY; + + try { + const session = new FakeRealtimeSession(); + const factory: RealtimeTranscriptionSessionFactory = () => session; + const emitted: Array<{ type: string; payload: any }> = []; + const manager = new DictationStreamManager({ + logger: pino({ level: "silent" }), + emit: (msg) => emitted.push(msg), + sessionId: "s1", + openaiApiKey: null, + sessionFactory: factory, + }); + + await manager.handleStart("d-local", "audio/pcm;rate=16000;bits=16"); + + expect(session.connected).toBe(true); + expect(emitted.find((msg) => msg.type === "dictation_stream_error")).toBeUndefined(); + } finally { + if (original !== undefined) { + process.env.OPENAI_API_KEY = original; + } else { + delete process.env.OPENAI_API_KEY; + } + } + }); +}); diff --git a/packages/server/src/server/dictation/dictation-stream-manager.ts b/packages/server/src/server/dictation/dictation-stream-manager.ts index 20a9581cf..ebad8281b 100644 --- a/packages/server/src/server/dictation/dictation-stream-manager.ts +++ b/packages/server/src/server/dictation/dictation-stream-manager.ts @@ -11,7 +11,7 @@ import { OpenAIRealtimeTranscriptionSession } from "../agent/openai-realtime-tra const PCM_CHANNELS = 1; const PCM_BITS_PER_SAMPLE = 16; -const DICTATION_PCM_OUTPUT_RATE = 24000; +const OPENAI_DICTATION_PCM_OUTPUT_RATE = 24000; const DEFAULT_DICTATION_FINAL_TIMEOUT_MS = 10000; const DICTATION_VAD_GRACE_TIMEOUT_MS = Number.parseInt( process.env.OPENAI_REALTIME_DICTATION_VAD_GRACE_TIMEOUT_MS ?? "2000", @@ -106,7 +106,7 @@ export type RealtimeTranscriptionSession = { }; export type RealtimeTranscriptionSessionFactory = (params: { - apiKey: string; + apiKey?: string | null; logger: pino.Logger; transcriptionModel: string; language?: string; @@ -149,6 +149,7 @@ type DictationStreamState = { inputFormat: string; openai: RealtimeTranscriptionSession; inputRate: number; + outputRate: number; resampler: Pcm16MonoResampler | null; debugAudioChunks: Buffer[]; debugRecordingPath: string | null; @@ -194,6 +195,7 @@ export class DictationStreamManager { private readonly openaiApiKey: string | null; private readonly finalTimeoutMs: number; private readonly createSession: RealtimeTranscriptionSessionFactory; + private readonly requiresApiKey: boolean; private readonly streams = new Map(); constructor(params: { @@ -209,10 +211,18 @@ export class DictationStreamManager { this.sessionId = params.sessionId; this.openaiApiKey = params.openaiApiKey ?? null; this.finalTimeoutMs = params.finalTimeoutMs ?? DEFAULT_DICTATION_FINAL_TIMEOUT_MS; + this.requiresApiKey = !params.sessionFactory; this.createSession = params.sessionFactory ?? - ((factoryParams) => - new OpenAIRealtimeTranscriptionSession(factoryParams)); + ((factoryParams) => { + if (!factoryParams.apiKey) { + throw new Error("OPENAI_API_KEY not set"); + } + return new OpenAIRealtimeTranscriptionSession({ + ...factoryParams, + apiKey: factoryParams.apiKey, + }); + }); } public cleanupAll(): void { @@ -225,7 +235,7 @@ export class DictationStreamManager { this.cleanupDictationStream(dictationId); const apiKey = this.openaiApiKey ?? process.env.OPENAI_API_KEY; - if (!apiKey) { + if (this.requiresApiKey && !apiKey) { this.failDictationStream(dictationId, "OPENAI_API_KEY not set", false); return; } @@ -238,7 +248,7 @@ export class DictationStreamManager { const turnDetection = parseDictationTurnDetection(); const openai = this.createSession({ - apiKey, + apiKey: apiKey ?? null, logger: this.logger.child({ dictationId }), transcriptionModel, language: "en", @@ -338,18 +348,21 @@ export class DictationStreamManager { this.logger ); + const outputRate = this.requiresApiKey ? OPENAI_DICTATION_PCM_OUTPUT_RATE : inputRate; + this.streams.set(dictationId, { dictationId, sessionId: this.sessionId, inputFormat: format, openai, inputRate, + outputRate, resampler: - inputRate === DICTATION_PCM_OUTPUT_RATE + inputRate === outputRate ? null : new Pcm16MonoResampler({ inputRate, - outputRate: DICTATION_PCM_OUTPUT_RATE, + outputRate, }), debugAudioChunks: [], debugRecordingPath: null, @@ -510,7 +523,7 @@ export class DictationStreamManager { const pcmBuffer = Buffer.concat(state.debugAudioChunks); const wavBuffer = convertPCMToWavBuffer( pcmBuffer, - DICTATION_PCM_OUTPUT_RATE, + state.outputRate, PCM_CHANNELS, PCM_BITS_PER_SAMPLE ); @@ -610,7 +623,7 @@ export class DictationStreamManager { } else { const silenceBytes = Math.max( 0, - Math.round((DICTATION_PCM_OUTPUT_RATE * 2 * DICTATION_FLUSH_SILENCE_MS) / 1000) + Math.round((state.outputRate * 2 * DICTATION_FLUSH_SILENCE_MS) / 1000) ); if (silenceBytes > 0) { this.logger.debug( diff --git a/packages/server/src/server/persisted-config.ts b/packages/server/src/server/persisted-config.ts index f463fcdb1..0c3ad2792 100644 --- a/packages/server/src/server/persisted-config.ts +++ b/packages/server/src/server/persisted-config.ts @@ -17,10 +17,32 @@ const ProviderCredentialsSchema = z }) .strict(); +const SherpaOnnxProviderSchema = z + .object({ + modelsDir: z.string().min(1).optional(), + autoDownload: z.boolean().optional(), + stt: z + .object({ + preset: z.string().min(1).optional(), + }) + .strict() + .optional(), + tts: z + .object({ + preset: z.string().min(1).optional(), + speakerId: z.number().int().optional(), + speed: z.number().optional(), + }) + .strict() + .optional(), + }) + .strict(); + const ProvidersSchema = z .object({ openai: ProviderCredentialsSchema.optional(), openrouter: ProviderCredentialsSchema.optional(), + sherpaOnnx: SherpaOnnxProviderSchema.optional(), }) .strict(); @@ -28,8 +50,9 @@ const FeatureDictationSchema = z .object({ stt: z .object({ - provider: z.enum(["openai"]).optional(), + provider: z.enum(["openai", "sherpa"]).optional(), model: z.string().min(1).optional(), + preset: z.string().min(1).optional(), confidenceThreshold: z.number().optional(), }) .strict() @@ -48,16 +71,20 @@ const FeatureVoiceModeSchema = z .optional(), stt: z .object({ - provider: z.enum(["openai"]).optional(), + provider: z.enum(["openai", "sherpa"]).optional(), model: z.string().min(1).optional(), + preset: z.string().min(1).optional(), }) .strict() .optional(), tts: z .object({ - provider: z.enum(["openai"]).optional(), + provider: z.enum(["openai", "sherpa"]).optional(), model: z.enum(["tts-1", "tts-1-hd"]).optional(), voice: z.enum(["alloy", "echo", "fable", "onyx", "nova", "shimmer"]).optional(), + preset: z.string().min(1).optional(), + speakerId: z.number().int().optional(), + speed: z.number().optional(), }) .strict() .optional(), diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 8c72c2ffe..d6a6885de 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -33,12 +33,12 @@ import { getSystemPrompt } from "./agent/system-prompt.js"; import { getAllTools } from "./agent/llm-openai.js"; import { TTSManager } from "./agent/tts-manager.js"; import { STTManager } from "./agent/stt-manager.js"; -import type { OpenAISTT } from "./agent/stt-openai.js"; -import type { OpenAITTS } from "./agent/tts-openai.js"; +import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js"; import { maybePersistTtsDebugAudio } from "./agent/tts-debug.js"; import { isPaseoDictationDebugEnabled } from "./agent/recordings-debug.js"; import { DictationStreamManager, + type RealtimeTranscriptionSessionFactory, } from "./dictation/dictation-stream-manager.js"; import type { VoiceConversationStore } from "./voice-conversation-store.js"; import { @@ -325,8 +325,8 @@ export class Session { agentManager: AgentManager, agentStorage: AgentStorage, createAgentMcpTransport: AgentMcpTransportFactory, - stt: OpenAISTT | null, - tts: OpenAITTS | null, + stt: SpeechToTextProvider | null, + tts: TextToSpeechProvider | null, terminalManager: TerminalManager | null, voiceConversationStore: VoiceConversationStore, voice?: { @@ -336,6 +336,7 @@ export class Session { dictation?: { openaiApiKey?: string | null; finalTimeoutMs?: number; + sessionFactory?: RealtimeTranscriptionSessionFactory; } ) { this.clientId = clientId; @@ -368,6 +369,7 @@ export class Session { emit: (msg) => this.emit(msg as unknown as SessionOutboundMessage), openaiApiKey: dictation?.openaiApiKey ?? null, finalTimeoutMs: dictation?.finalTimeoutMs, + ...(dictation?.sessionFactory ? { sessionFactory: dictation.sessionFactory } : {}), }); // Initialize agent MCP client asynchronously diff --git a/packages/server/src/server/speech/audio.ts b/packages/server/src/server/speech/audio.ts new file mode 100644 index 000000000..6d1907ce0 --- /dev/null +++ b/packages/server/src/server/speech/audio.ts @@ -0,0 +1,113 @@ +export function parsePcm16MonoWav(buffer: Buffer): { sampleRate: number; pcm16: Buffer } { + if (buffer.toString("ascii", 0, 4) !== "RIFF" || buffer.toString("ascii", 8, 12) !== "WAVE") { + throw new Error("Invalid WAV header"); + } + + let offset = 12; + let fmt: + | { audioFormat: number; channels: number; sampleRate: number; bitsPerSample: number } + | null = null; + let dataChunk: Buffer | null = null; + + while (offset + 8 <= buffer.length) { + const id = buffer.toString("ascii", offset, offset + 4); + const size = buffer.readUInt32LE(offset + 4); + const payloadStart = offset + 8; + const payloadEnd = payloadStart + size; + if (payloadEnd > buffer.length) { + break; + } + + if (id === "fmt ") { + const audioFormat = buffer.readUInt16LE(payloadStart); + const channels = buffer.readUInt16LE(payloadStart + 2); + const sampleRate = buffer.readUInt32LE(payloadStart + 4); + const bitsPerSample = buffer.readUInt16LE(payloadStart + 14); + fmt = { audioFormat, channels, sampleRate, bitsPerSample }; + } else if (id === "data") { + dataChunk = buffer.subarray(payloadStart, payloadEnd); + } + + offset = payloadEnd + (size % 2); + } + + if (!fmt || !dataChunk) { + throw new Error("Missing WAV fmt/data chunks"); + } + if (fmt.audioFormat !== 1) { + throw new Error(`Unsupported WAV encoding (audioFormat=${fmt.audioFormat})`); + } + if (fmt.channels !== 1 || fmt.bitsPerSample !== 16) { + throw new Error( + `Unexpected WAV format: channels=${fmt.channels} rate=${fmt.sampleRate} bits=${fmt.bitsPerSample}` + ); + } + if (dataChunk.length % 2 !== 0) { + throw new Error("WAV PCM16 data length must be even"); + } + return { sampleRate: fmt.sampleRate, pcm16: dataChunk }; +} + +export function parsePcmRateFromFormat(format: string, fallback: number | null = null): number | null { + const match = /(?:^|[;,\s])rate\s*=\s*(\d+)(?:$|[;,\s])/i.exec(format); + if (!match) { + return fallback; + } + const rate = Number.parseInt(match[1]!, 10); + return Number.isFinite(rate) && rate > 0 ? rate : fallback; +} + +export function pcm16lePeakAbs(pcm16le: Buffer): number { + if (pcm16le.length === 0) { + return 0; + } + if (pcm16le.length % 2 !== 0) { + throw new Error(`PCM16 chunk byteLength must be even, got ${pcm16le.length}`); + } + const samples = new Int16Array(pcm16le.buffer, pcm16le.byteOffset, pcm16le.byteLength / 2); + let peak = 0; + for (let i = 0; i < samples.length; i += 1) { + const v = samples[i]!; + const abs = v < 0 ? -v : v; + if (abs > peak) { + peak = abs; + if (peak >= 32767) { + break; + } + } + } + return peak; +} + +export function pcm16leToFloat32(pcm16le: Buffer, gain: number = 1): Float32Array { + if (pcm16le.length % 2 !== 0) { + throw new Error(`PCM16 chunk byteLength must be even, got ${pcm16le.length}`); + } + const int16 = new Int16Array(pcm16le.buffer, pcm16le.byteOffset, pcm16le.byteLength / 2); + const out = new Float32Array(int16.length); + for (let i = 0; i < int16.length; i += 1) { + const v = (int16[i]! / 32768.0) * gain; + out[i] = v > 1 ? 1 : v < -1 ? -1 : v; + } + return out; +} + +export function float32ToPcm16le(samples: Float32Array): Buffer { + const out = new Int16Array(samples.length); + for (let i = 0; i < samples.length; i += 1) { + const clamped = Math.max(-1, Math.min(1, samples[i]!)); + out[i] = Math.round(clamped * 32767); + } + return Buffer.from(out.buffer, out.byteOffset, out.byteLength); +} + +export function chunkBuffer(buffer: Buffer, chunkBytes: number): Buffer[] { + if (chunkBytes <= 0) { + return [buffer]; + } + const out: Buffer[] = []; + for (let offset = 0; offset < buffer.length; offset += chunkBytes) { + out.push(buffer.subarray(offset, Math.min(buffer.length, offset + chunkBytes))); + } + return out; +} diff --git a/packages/server/src/server/speech/pocket/pocket-tts-onnx.ts b/packages/server/src/server/speech/pocket/pocket-tts-onnx.ts new file mode 100644 index 000000000..58fd16e96 --- /dev/null +++ b/packages/server/src/server/speech/pocket/pocket-tts-onnx.ts @@ -0,0 +1,559 @@ +import { existsSync } from "node:fs"; +import { readFile } from "node:fs/promises"; +import { Readable } from "node:stream"; +import type pino from "pino"; + +import type { SpeechStreamResult, TextToSpeechProvider } from "../speech-provider.js"; +import { chunkBuffer, float32ToPcm16le, parsePcm16MonoWav, pcm16leToFloat32 } from "../audio.js"; +import { Pcm16MonoResampler } from "../../agent/pcm16-resampler.js"; + +type OrtModule = typeof import("onnxruntime-node"); +type OrtSession = import("onnxruntime-node").InferenceSession; +type OrtTensor = import("onnxruntime-node").Tensor; + +type SentencePieceProcessor = { + encodeIds: (text: string) => number[]; + load?: (modelPath: string) => unknown; + Load?: (modelPath: string) => unknown; +}; + +function assertFileExists(filePath: string, label: string): void { + if (!existsSync(filePath)) { + throw new Error(`Missing ${label}: ${filePath}`); + } +} + +function product(dims: number[]): number { + let out = 1; + for (const d of dims) out *= d; + return out; +} + +function normalizeDims(dims: Array): number[] { + // ONNX metadata can contain dynamic dimensions as strings (e.g. "batch") or -1. + // For state tensors we want a valid minimal shape, so coerce unknown/invalid dims to 1. + // Preserve explicit 0 dims (some models use empty initial state buffers with shape [0]). + return dims.map((d) => { + if (typeof d === "number" && Number.isFinite(d)) { + if (d === 0) return 0; + if (d > 0) return d; + return 1; + } + return 1; + }); +} + +function getSessionInputMeta( + session: OrtSession, + inputName: string +): { type?: string; dims?: Array } | undefined { + const metaAny = (session as any).inputMetadata as unknown; + if (Array.isArray(metaAny)) { + const entry = metaAny.find((m) => m && typeof m === "object" && (m as any).name === inputName) as any; + if (!entry) return undefined; + return { type: entry.type, dims: entry.shape }; + } + + if (metaAny && typeof metaAny === "object" && inputName in (metaAny as any)) { + const entry = (metaAny as any)[inputName] as any; + return { type: entry?.type, dims: entry?.dimensions ?? entry?.shape }; + } + + return undefined; +} + +function toBigInt64(values: number[]): BigInt64Array { + const out = new BigInt64Array(values.length); + for (let i = 0; i < values.length; i += 1) { + out[i] = BigInt(values[i]!); + } + return out; +} + +function randn(): number { + // Box–Muller + let u = 0; + let v = 0; + while (u === 0) u = Math.random(); + while (v === 0) v = Math.random(); + return Math.sqrt(-2.0 * Math.log(u)) * Math.cos(2.0 * Math.PI * v); +} + +function normalizeTextForPocket(text: string): string { + const trimmed = text.trim(); + if (!trimmed) { + throw new Error("Cannot synthesize empty text"); + } + let out = trimmed; + if (out.length > 0 && /[A-Za-z0-9]$/.test(out)) { + out = `${out}.`; + } + if (out.length > 0 && /[a-z]/.test(out[0]!)) { + out = out[0]!.toUpperCase() + out.slice(1); + } + return out; +} + +async function loadOrt(): Promise { + return (await import("onnxruntime-node")) as OrtModule; +} + +async function loadSentencePiece(tokenizerModelPath: string): Promise { + const mod = await import("@sctg/sentencepiece-js"); + + const Processor = + (mod as any).SentencePieceProcessor ?? + (mod as any).default?.SentencePieceProcessor ?? + (mod as any).default; + + if (!Processor) { + throw new Error("Failed to load SentencePiece processor from @sctg/sentencepiece-js"); + } + + const sp: SentencePieceProcessor = new Processor(); + + if (typeof sp.load === "function") { + await sp.load(tokenizerModelPath); + } else if (typeof sp.Load === "function") { + sp.Load(tokenizerModelPath); + } else { + throw new Error("SentencePiece processor does not expose load()/Load()"); + } + + return sp; +} + +function getOrtProviders(ort: OrtModule, device: "auto" | "cpu" | "cuda"): string[] { + // NOTE: onnxruntime-node uses backend names like "cpu"/"coreml"/"webgpu" (not "CPUExecutionProvider"). + if (device === "cpu") return ["cpu"]; + if (device === "cuda") return ["cuda", "cpu"]; + // auto + // CoreML EP does not support some dynamic/zero-length shapes used by Pocket TTS (e.g. [1, 0, 32]). + // Default to CPU to keep behavior predictable across platforms. + void ort; + return ["cpu"]; +} + +function createZeroTensorForInput(ort: OrtModule, session: OrtSession, inputName: string): OrtTensor { + const meta = getSessionInputMeta(session, inputName); + const dims = normalizeDims(meta?.dims ?? []); + if (dims.length === 0) { + throw new Error(`Missing input metadata shape for ${inputName}`); + } + + const type = (meta?.type ?? "float32").toLowerCase(); + const size = product(dims); + + if (type.includes("int64")) { + return new ort.Tensor("int64", new BigInt64Array(size), dims); + } + if (type.includes("bool")) { + return new ort.Tensor("bool", new Uint8Array(size), dims); + } + return new ort.Tensor("float32", new Float32Array(size), dims); +} + +function initState(session: OrtSession, ort: OrtModule): Record { + const out: Record = {}; + for (const name of (session as any).inputNames as string[]) { + if (name.startsWith("state_")) { + out[name] = createZeroTensorForInput(ort, session, name); + } + } + return out; +} + +function updateStateFromOutputs( + state: Record, + outputs: Record +): void { + for (const [name, tensor] of Object.entries(outputs)) { + if (!name.startsWith("out_state_")) continue; + const idx = Number.parseInt(name.replace("out_state_", ""), 10); + if (Number.isFinite(idx)) { + state[`state_${idx}`] = tensor; + } + } +} + +function tensorDataFloat32(t: OrtTensor): Float32Array { + const data = (t as any).data; + if (data instanceof Float32Array) return data; + if (Array.isArray(data)) return Float32Array.from(data as number[]); + throw new Error("Unexpected tensor data type (expected Float32Array)"); +} + +export type PocketTtsOnnxConfig = { + modelDir: string; + precision?: "int8" | "fp32"; + device?: "auto" | "cpu" | "cuda"; + temperature?: number; + lsdSteps?: number; + maxFrames?: number; + framesAfterEos?: number; + firstChunkFrames?: number; + maxChunkFrames?: number; + targetChunkMs?: number; + referenceAudioFile?: string; +}; + +class PocketTtsOnnxEngine { + static readonly SAMPLE_RATE = 24000; + static readonly SAMPLES_PER_FRAME = 1920; + + private readonly ort: OrtModule; + + private readonly temperature: number; + private readonly lsdSteps: number; + private readonly maxFrames: number; + private readonly framesAfterEos: number; + + private readonly firstChunkFrames: number; + private readonly maxChunkFrames: number; + + private readonly tokenizer: SentencePieceProcessor; + private readonly textConditioner: OrtSession; + private readonly flowLmMain: OrtSession; + private readonly flowLmFlow: OrtSession; + private readonly mimiDecoder: OrtSession; + + private readonly stBuffers: Array<{ s: OrtTensor; t: OrtTensor }>; + private readonly voiceEmbeddings: OrtTensor; + + private constructor(args: { + ort: OrtModule; + temperature: number; + lsdSteps: number; + maxFrames: number; + framesAfterEos: number; + firstChunkFrames: number; + maxChunkFrames: number; + tokenizer: SentencePieceProcessor; + textConditioner: OrtSession; + flowLmMain: OrtSession; + flowLmFlow: OrtSession; + mimiDecoder: OrtSession; + stBuffers: Array<{ s: OrtTensor; t: OrtTensor }>; + voiceEmbeddings: OrtTensor; + }) { + this.ort = args.ort; + this.temperature = args.temperature; + this.lsdSteps = args.lsdSteps; + this.maxFrames = args.maxFrames; + this.framesAfterEos = args.framesAfterEos; + this.firstChunkFrames = args.firstChunkFrames; + this.maxChunkFrames = args.maxChunkFrames; + this.tokenizer = args.tokenizer; + this.textConditioner = args.textConditioner; + this.flowLmMain = args.flowLmMain; + this.flowLmFlow = args.flowLmFlow; + this.mimiDecoder = args.mimiDecoder; + this.stBuffers = args.stBuffers; + this.voiceEmbeddings = args.voiceEmbeddings; + } + + static async create(config: PocketTtsOnnxConfig, logger: pino.Logger): Promise { + const log = logger.child({ module: "speech", provider: "pocket-tts", component: "onnx-engine" }); + + const modelDir = config.modelDir; + const onnxDir = `${modelDir}/onnx`; + const precision = config.precision ?? "int8"; + const device = config.device ?? "auto"; + const temperature = config.temperature ?? 0.7; + const lsdSteps = config.lsdSteps ?? 10; + const maxFrames = config.maxFrames ?? 500; + const framesAfterEos = config.framesAfterEos ?? 3; + const firstChunkFrames = config.firstChunkFrames ?? 2; + const maxChunkFrames = config.maxChunkFrames ?? 15; + + const tokenizerPath = `${modelDir}/tokenizer.model`; + const referenceAudioFile = config.referenceAudioFile ?? `${modelDir}/reference_sample.wav`; + + const flowMainFile = precision === "int8" ? "flow_lm_main_int8.onnx" : "flow_lm_main.onnx"; + const flowFlowFile = precision === "int8" ? "flow_lm_flow_int8.onnx" : "flow_lm_flow.onnx"; + const decoderFile = precision === "int8" ? "mimi_decoder_int8.onnx" : "mimi_decoder.onnx"; + + assertFileExists(`${onnxDir}/mimi_encoder.onnx`, "PocketTTS mimi_encoder"); + assertFileExists(`${onnxDir}/text_conditioner.onnx`, "PocketTTS text_conditioner"); + assertFileExists(`${onnxDir}/${flowMainFile}`, "PocketTTS flow_lm_main"); + assertFileExists(`${onnxDir}/${flowFlowFile}`, "PocketTTS flow_lm_flow"); + assertFileExists(`${onnxDir}/${decoderFile}`, "PocketTTS mimi_decoder"); + assertFileExists(tokenizerPath, "PocketTTS tokenizer.model"); + assertFileExists(referenceAudioFile, "PocketTTS reference_sample.wav"); + + const ort = await loadOrt(); + const providers = getOrtProviders(ort, device); + + const [tokenizer, mimiEncoder, textConditioner, flowLmMain, flowLmFlow, mimiDecoder] = await Promise.all([ + loadSentencePiece(tokenizerPath), + ort.InferenceSession.create(`${onnxDir}/mimi_encoder.onnx`, { executionProviders: providers }), + ort.InferenceSession.create(`${onnxDir}/text_conditioner.onnx`, { executionProviders: providers }), + ort.InferenceSession.create(`${onnxDir}/${flowMainFile}`, { executionProviders: providers }), + ort.InferenceSession.create(`${onnxDir}/${flowFlowFile}`, { executionProviders: providers }), + ort.InferenceSession.create(`${onnxDir}/${decoderFile}`, { executionProviders: providers }), + ]); + + // Precompute flow matching time-step buffers. + const stBuffers: Array<{ s: OrtTensor; t: OrtTensor }> = []; + for (let j = 0; j < lsdSteps; j += 1) { + const s = j / lsdSteps; + const t = s + 1.0 / lsdSteps; + stBuffers.push({ + s: new ort.Tensor("float32", new Float32Array([s]), [1, 1]), + t: new ort.Tensor("float32", new Float32Array([t]), [1, 1]), + }); + } + + // Precompute reference voice embeddings once. + const refWav = await readFile(referenceAudioFile); + const parsed = parsePcm16MonoWav(refWav); + let pcm16 = parsed.pcm16; + if (parsed.sampleRate !== PocketTtsOnnxEngine.SAMPLE_RATE) { + const resampler = new Pcm16MonoResampler({ + inputRate: parsed.sampleRate, + outputRate: PocketTtsOnnxEngine.SAMPLE_RATE, + }); + pcm16 = resampler.processChunk(pcm16); + } + const floatAudio = pcm16leToFloat32(pcm16); + const audioTensor = new ort.Tensor( + "float32", + floatAudio, + [1, 1, floatAudio.length] + ); + + const encoded = await mimiEncoder.run({ audio: audioTensor }); + const firstOutName = (mimiEncoder as any).outputNames?.[0] as string | undefined; + const voiceEmb = firstOutName ? (encoded as any)[firstOutName] : (Object.values(encoded)[0] as any); + if (!voiceEmb) { + throw new Error("PocketTTS mimi_encoder: missing output"); + } + + log.info( + { precision, device, providers, lsdSteps, temperature }, + "PocketTTS ONNX initialized" + ); + + return new PocketTtsOnnxEngine({ + ort, + temperature, + lsdSteps, + maxFrames, + framesAfterEos, + firstChunkFrames, + maxChunkFrames, + tokenizer, + textConditioner, + flowLmMain, + flowLmFlow, + mimiDecoder, + stBuffers, + voiceEmbeddings: voiceEmb, + }); + } + + private tokenize(text: string): OrtTensor { + const normalized = normalizeTextForPocket(text); + const ids = this.tokenizer.encodeIds(normalized); + const data = toBigInt64(ids ?? []); + return new this.ort.Tensor("int64", data, [1, data.length]); + } + + private async runTextConditioner(tokenIds: OrtTensor): Promise { + const out = await this.textConditioner.run({ token_ids: tokenIds } as any); + const firstOutName = (this.textConditioner as any).outputNames?.[0] as string | undefined; + const t = firstOutName ? (out as any)[firstOutName] : (Object.values(out)[0] as any); + if (!t) throw new Error("PocketTTS text_conditioner: missing output"); + return t; + } + + private async *runFlowLm(textEmbeddings: OrtTensor): AsyncGenerator { + const ort = this.ort; + const state = initState(this.flowLmMain, ort); + + const emptySeq = new ort.Tensor("float32", new Float32Array(0), [1, 0, 32]); + const emptyText = new ort.Tensor("float32", new Float32Array(0), [1, 0, 1024]); + + // Voice conditioning pass + const resVoice = await this.flowLmMain.run({ + sequence: emptySeq, + text_embeddings: this.voiceEmbeddings, + ...state, + } as any); + updateStateFromOutputs(state, resVoice as any); + + // Text conditioning pass + const resText = await this.flowLmMain.run({ + sequence: emptySeq, + text_embeddings: textEmbeddings, + ...state, + } as any); + updateStateFromOutputs(state, resText as any); + + // Autoregressive generation + const curr = new Float32Array(32); + curr.fill(Number.NaN); + let currTensor = new ort.Tensor("float32", curr, [1, 1, 32]); + + const dt = 1.0 / this.lsdSteps; + let eosStep: number | null = null; + + for (let step = 0; step < this.maxFrames; step += 1) { + const resStep = await this.flowLmMain.run({ + sequence: currTensor, + text_embeddings: emptyText, + ...state, + } as any); + + const outputNames = (this.flowLmMain as any).outputNames as string[] | undefined; + const conditioningName = outputNames?.[0] ?? Object.keys(resStep)[0]!; + const eosName = outputNames?.[1] ?? Object.keys(resStep)[1]!; + + const conditioning = (resStep as any)[conditioningName] as OrtTensor; + const eos = (resStep as any)[eosName] as OrtTensor; + if (!conditioning || !eos) { + throw new Error("PocketTTS flow_lm_main: missing conditioning/EOS outputs"); + } + updateStateFromOutputs(state, resStep as any); + + const eosData = tensorDataFloat32(eos); + if (eosData[0]! > -4.0 && eosStep === null) { + eosStep = step; + } + if (eosStep !== null && step >= eosStep + this.framesAfterEos) { + break; + } + + // Flow matching with external Euler loop. + const std = this.temperature > 0 ? Math.sqrt(this.temperature) : 0; + const x = new Float32Array(32); + if (std > 0) { + for (let i = 0; i < x.length; i += 1) { + x[i] = randn() * std; + } + } + + for (const st of this.stBuffers) { + const xTensor = new ort.Tensor("float32", x, [1, 32]); + const flowOut = await this.flowLmFlow.run({ + c: conditioning, + s: st.s, + t: st.t, + x: xTensor, + } as any); + const first = (this.flowLmFlow as any).outputNames?.[0] as string | undefined; + const flowTensor = first ? (flowOut as any)[first] : (Object.values(flowOut)[0] as any); + if (!flowTensor) throw new Error("PocketTTS flow_lm_flow: missing output"); + const delta = tensorDataFloat32(flowTensor); + for (let i = 0; i < x.length; i += 1) { + x[i] = x[i]! + delta[i]! * dt; + } + } + + yield x; + currTensor = new ort.Tensor("float32", x, [1, 1, 32]); + } + } + + private async decodeLatentsChunk( + frames: Float32Array[], + state: Record + ): Promise { + const ort = this.ort; + const frameCount = frames.length; + const flattened = new Float32Array(frameCount * 32); + for (let i = 0; i < frameCount; i += 1) { + flattened.set(frames[i]!, i * 32); + } + const latent = new ort.Tensor("float32", flattened, [1, frameCount, 32]); + + const out = await this.mimiDecoder.run({ latent, ...state } as any); + updateStateFromOutputs(state, out as any); + + const firstOutName = (this.mimiDecoder as any).outputNames?.[0] as string | undefined; + const audioTensor = firstOutName ? (out as any)[firstOutName] : (Object.values(out)[0] as any); + if (!audioTensor) { + throw new Error("PocketTTS mimi_decoder: missing audio output"); + } + return tensorDataFloat32(audioTensor); + } + + async *streamAudio(text: string): AsyncGenerator { + const tokenIds = this.tokenize(text); + const textEmb = await this.runTextConditioner(tokenIds); + + const decoderState = initState(this.mimiDecoder, this.ort); + + const generated: Float32Array[] = []; + let decodedFrames = 0; + + for await (const latent of this.runFlowLm(textEmb)) { + generated.push(latent); + const pending = generated.length - decodedFrames; + + let chunkSize = 0; + if (decodedFrames === 0) { + if (pending >= this.firstChunkFrames) { + chunkSize = this.firstChunkFrames; + } + } else if (pending >= this.maxChunkFrames) { + chunkSize = this.maxChunkFrames; + } + + if (chunkSize > 0) { + const audio = await this.decodeLatentsChunk( + generated.slice(decodedFrames, decodedFrames + chunkSize), + decoderState + ); + decodedFrames += chunkSize; + yield audio; + } + } + + if (decodedFrames < generated.length) { + const audio = await this.decodeLatentsChunk(generated.slice(decodedFrames), decoderState); + yield audio; + } + } +} + +export class PocketTtsOnnxTTS implements TextToSpeechProvider { + private readonly engine: PocketTtsOnnxEngine; + private readonly chunkMs: number; + private readonly logger: pino.Logger; + + private constructor(engine: PocketTtsOnnxEngine, logger: pino.Logger, chunkMs: number) { + this.engine = engine; + this.chunkMs = chunkMs; + this.logger = logger.child({ module: "speech", provider: "pocket-tts", component: "tts" }); + } + + static async create(config: PocketTtsOnnxConfig, logger: pino.Logger): Promise { + const engine = await PocketTtsOnnxEngine.create(config, logger); + const chunkMs = config.targetChunkMs ?? 50; + return new PocketTtsOnnxTTS(engine, logger, chunkMs); + } + + async synthesizeSpeech(text: string): Promise { + const start = Date.now(); + const sampleRate = PocketTtsOnnxEngine.SAMPLE_RATE; + + const pcmChunkBytes = Math.max(2, Math.round((sampleRate * this.chunkMs) / 1000) * 2); + + const iterable = (async function* (engine: PocketTtsOnnxEngine) { + for await (const floatChunk of engine.streamAudio(text)) { + const pcm = float32ToPcm16le(floatChunk); + for (const chunk of chunkBuffer(pcm, pcmChunkBytes)) { + yield chunk; + } + } + })(this.engine); + + this.logger.debug({ ms: Date.now() - start, textLength: text.length }, "PocketTTS stream ready"); + + return { + stream: Readable.from(iterable), + format: `pcm;rate=${sampleRate}`, + }; + } +} diff --git a/packages/server/src/server/speech/sherpa/model-catalog.ts b/packages/server/src/server/speech/sherpa/model-catalog.ts new file mode 100644 index 000000000..94c6d62bd --- /dev/null +++ b/packages/server/src/server/speech/sherpa/model-catalog.ts @@ -0,0 +1,128 @@ +export type SherpaOnnxModelKind = "stt-online" | "stt-offline" | "tts"; + +export type SherpaOnnxModelId = + | "zipformer-bilingual-zh-en-2023-02-20" + | "paraformer-bilingual-zh-en" + | "parakeet-tdt-0.6b-v3-int8" + | "kitten-nano-en-v0_1-fp16" + | "kokoro-en-v0_19" + | "pocket-tts-onnx-int8"; + +export type SherpaOnnxModelSpec = { + id: SherpaOnnxModelId; + kind: SherpaOnnxModelKind; + archiveUrl?: string; + downloadFiles?: Array<{ url: string; relPath: string }>; + extractedDir: string; + requiredFiles: string[]; + description: string; +}; + +export const SHERPA_ONNX_MODEL_CATALOG: Record = { + "zipformer-bilingual-zh-en-2023-02-20": { + id: "zipformer-bilingual-zh-en-2023-02-20", + kind: "stt-online", + archiveUrl: + "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2", + extractedDir: "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", + requiredFiles: [ + "encoder-epoch-99-avg-1.onnx", + "decoder-epoch-99-avg-1.onnx", + "joiner-epoch-99-avg-1.onnx", + "tokens.txt", + ], + description: "Streaming Zipformer transducer (fast, good accuracy).", + }, + "paraformer-bilingual-zh-en": { + id: "paraformer-bilingual-zh-en", + kind: "stt-online", + archiveUrl: + "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2", + extractedDir: "sherpa-onnx-streaming-paraformer-bilingual-zh-en", + requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "tokens.txt"], + description: "Streaming Paraformer (often strong accuracy; heavier).", + }, + "parakeet-tdt-0.6b-v3-int8": { + id: "parakeet-tdt-0.6b-v3-int8", + kind: "stt-offline", + archiveUrl: + "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2", + extractedDir: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8", + requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", "tokens.txt"], + description: "NVIDIA Parakeet TDT v3 (offline NeMo transducer, multilingual).", + }, + "kitten-nano-en-v0_1-fp16": { + id: "kitten-nano-en-v0_1-fp16", + kind: "tts", + archiveUrl: + "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2", + extractedDir: "kitten-nano-en-v0_1-fp16", + requiredFiles: ["model.fp16.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"], + description: "KittenTTS (small, fast English TTS).", + }, + "kokoro-en-v0_19": { + id: "kokoro-en-v0_19", + kind: "tts", + archiveUrl: "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2", + extractedDir: "kokoro-en-v0_19", + requiredFiles: ["model.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"], + description: "Kokoro TTS (higher quality; larger).", + }, + "pocket-tts-onnx-int8": { + id: "pocket-tts-onnx-int8", + kind: "tts", + extractedDir: "pocket-tts-onnx-int8", + downloadFiles: [ + { + url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/mimi_encoder.onnx", + relPath: "onnx/mimi_encoder.onnx", + }, + { + url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/text_conditioner.onnx", + relPath: "onnx/text_conditioner.onnx", + }, + { + url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/flow_lm_main_int8.onnx", + relPath: "onnx/flow_lm_main_int8.onnx", + }, + { + url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/flow_lm_flow_int8.onnx", + relPath: "onnx/flow_lm_flow_int8.onnx", + }, + { + url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/mimi_decoder_int8.onnx", + relPath: "onnx/mimi_decoder_int8.onnx", + }, + { + url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/tokenizer.model", + relPath: "tokenizer.model", + }, + { + url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/reference_sample.wav", + relPath: "reference_sample.wav", + }, + ], + requiredFiles: [ + "onnx/mimi_encoder.onnx", + "onnx/text_conditioner.onnx", + "onnx/flow_lm_main_int8.onnx", + "onnx/flow_lm_flow_int8.onnx", + "onnx/mimi_decoder_int8.onnx", + "tokenizer.model", + "reference_sample.wav", + ], + description: "Pocket TTS ONNX (INT8) with streaming decode support (via onnxruntime).", + }, +}; + +export function listSherpaOnnxModels(): SherpaOnnxModelSpec[] { + return Object.values(SHERPA_ONNX_MODEL_CATALOG); +} + +export function getSherpaOnnxModelSpec(id: SherpaOnnxModelId): SherpaOnnxModelSpec { + const spec = SHERPA_ONNX_MODEL_CATALOG[id]; + if (!spec) { + throw new Error(`Unknown sherpa-onnx model id: ${id}`); + } + return spec; +} diff --git a/packages/server/src/server/speech/sherpa/model-downloader.test.ts b/packages/server/src/server/speech/sherpa/model-downloader.test.ts new file mode 100644 index 000000000..ab898bfae --- /dev/null +++ b/packages/server/src/server/speech/sherpa/model-downloader.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, test } from "vitest"; +import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import pino from "pino"; + +import { ensureSherpaOnnxModel, getSherpaOnnxModelDir } from "./model-downloader.js"; + +function makeTmpDir(): string { + return mkdtempSync(path.join(tmpdir(), "paseo-speech-models-")); +} + +const logger = pino({ level: "silent" }); + +describe("sherpa model downloader", () => { + test("getSherpaOnnxModelDir maps modelId to extractedDir", () => { + const modelsDir = "/tmp/models"; + expect(getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v3-int8")).toContain( + "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8" + ); + expect(getSherpaOnnxModelDir(modelsDir, "pocket-tts-onnx-int8")).toContain( + "pocket-tts-onnx-int8" + ); + }); + + test("ensureSherpaOnnxModel succeeds without downloading when files exist", async () => { + const modelsDir = makeTmpDir(); + const modelDir = getSherpaOnnxModelDir(modelsDir, "kitten-nano-en-v0_1-fp16"); + + mkdirSync(path.join(modelDir, "espeak-ng-data"), { recursive: true }); + writeFileSync(path.join(modelDir, "model.fp16.onnx"), "x"); + writeFileSync(path.join(modelDir, "voices.bin"), "x"); + writeFileSync(path.join(modelDir, "tokens.txt"), "x"); + + const out = await ensureSherpaOnnxModel({ + modelsDir, + modelId: "kitten-nano-en-v0_1-fp16", + autoDownload: false, + logger, + }); + + expect(out).toBe(modelDir); + }); + + test("ensureSherpaOnnxModel throws when missing and autoDownload is false", async () => { + const modelsDir = makeTmpDir(); + await expect( + ensureSherpaOnnxModel({ + modelsDir, + modelId: "zipformer-bilingual-zh-en-2023-02-20", + autoDownload: false, + logger, + }) + ).rejects.toThrow(/auto-download/i); + }); +}); diff --git a/packages/server/src/server/speech/sherpa/model-downloader.ts b/packages/server/src/server/speech/sherpa/model-downloader.ts new file mode 100644 index 000000000..9c434c4be --- /dev/null +++ b/packages/server/src/server/speech/sherpa/model-downloader.ts @@ -0,0 +1,190 @@ +import { createWriteStream } from "node:fs"; +import { mkdir, rename, rm, stat } from "node:fs/promises"; +import path from "node:path"; +import { Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { spawn } from "node:child_process"; +import type pino from "pino"; + +import { getSherpaOnnxModelSpec, type SherpaOnnxModelId } from "./model-catalog.js"; + +export type EnsureSherpaOnnxModelOptions = { + modelsDir: string; + modelId: SherpaOnnxModelId; + autoDownload: boolean; + logger: pino.Logger; +}; + +export function getSherpaOnnxModelDir(modelsDir: string, modelId: SherpaOnnxModelId): string { + const spec = getSherpaOnnxModelSpec(modelId); + return path.join(modelsDir, spec.extractedDir); +} + +async function hasRequiredFiles(modelDir: string, requiredFiles: string[]): Promise { + for (const rel of requiredFiles) { + const abs = path.join(modelDir, rel); + try { + const s = await stat(abs); + if (s.isDirectory()) { + continue; + } + if (s.isFile() && s.size > 0) { + continue; + } + return false; + } catch { + return false; + } + } + return true; +} + +async function downloadToFile(url: string, outputPath: string, logger: pino.Logger): Promise { + const res = await fetch(url); + if (!res.ok) { + throw new Error(`Failed to download ${url}: ${res.status} ${res.statusText}`); + } + if (!res.body) { + throw new Error(`Failed to download ${url}: missing response body`); + } + + const tmpPath = `${outputPath}.tmp-${Date.now()}`; + await mkdir(path.dirname(outputPath), { recursive: true }); + + const total = Number(res.headers.get("content-length") ?? "0"); + let downloaded = 0; + let lastLoggedBucket = -1; + + const nodeStream = Readable.fromWeb(res.body as any).on("data", (chunk: Buffer) => { + downloaded += chunk.length; + if (total > 0) { + const pct = Math.floor((downloaded / total) * 100); + const bucket = Math.min(100, Math.floor(pct / 10) * 10); + if (bucket >= 0 && bucket <= 100 && bucket !== lastLoggedBucket) { + lastLoggedBucket = bucket; + logger.info({ pct: bucket, downloaded, total }, "Downloading model artifact"); + } + } + }); + + await pipeline(nodeStream, createWriteStream(tmpPath)); + await rename(tmpPath, outputPath); +} + +async function extractTarArchive(archivePath: string, destDir: string): Promise { + await mkdir(destDir, { recursive: true }); + + await new Promise((resolve, reject) => { + const child = spawn("tar", ["xf", archivePath, "-C", destDir], { stdio: "inherit" }); + child.on("error", reject); + child.on("exit", (code) => { + if (code === 0) resolve(); + else reject(new Error(`tar exited with code ${code}`)); + }); + }); +} + +async function isNonEmptyFile(filePath: string): Promise { + try { + const s = await stat(filePath); + return s.isFile() && s.size > 0; + } catch { + return false; + } +} + +export async function ensureSherpaOnnxModel(options: EnsureSherpaOnnxModelOptions): Promise { + const logger = options.logger.child({ + module: "speech", + provider: "sherpa-onnx", + component: "model-downloader", + modelId: options.modelId, + }); + + const spec = getSherpaOnnxModelSpec(options.modelId); + const modelDir = path.join(options.modelsDir, spec.extractedDir); + if (await hasRequiredFiles(modelDir, spec.requiredFiles)) { + return modelDir; + } + + if (!options.autoDownload) { + throw new Error( + `Missing sherpa-onnx model files for ${options.modelId} in ${modelDir}. ` + + `Set PASEO_SHERPA_ONNX_AUTO_DOWNLOAD=1 to auto-download.` + ); + } + + if (spec.archiveUrl) { + logger.info({ modelsDir: options.modelsDir, url: spec.archiveUrl }, "Model files missing; downloading"); + + const downloadsDir = path.join(options.modelsDir, ".downloads"); + const archiveFilename = path.basename(new URL(spec.archiveUrl).pathname); + const archivePath = path.join(downloadsDir, archiveFilename); + + if (!(await isNonEmptyFile(archivePath))) { + await downloadToFile(spec.archiveUrl, archivePath, logger); + } else { + logger.info({ archivePath }, "Using cached archive"); + } + + await extractTarArchive(archivePath, options.modelsDir); + + if (!(await hasRequiredFiles(modelDir, spec.requiredFiles))) { + throw new Error( + `Downloaded and extracted ${archiveFilename}, but required files are still missing in ${modelDir}.` + ); + } + + try { + await rm(archivePath, { force: true }); + } catch { + // ignore + } + + logger.info({ modelDir }, "Model ready"); + return modelDir; + } + + if (spec.downloadFiles && spec.downloadFiles.length > 0) { + logger.info({ modelsDir: options.modelsDir, fileCount: spec.downloadFiles.length }, "Model files missing; downloading"); + await mkdir(modelDir, { recursive: true }); + + for (const file of spec.downloadFiles) { + const dst = path.join(modelDir, file.relPath); + if (await isNonEmptyFile(dst)) { + continue; + } + await downloadToFile(file.url, dst, logger); + } + + if (!(await hasRequiredFiles(modelDir, spec.requiredFiles))) { + throw new Error( + `Downloaded files for ${options.modelId}, but required files are still missing in ${modelDir}.` + ); + } + + logger.info({ modelDir }, "Model ready"); + return modelDir; + } + + throw new Error(`Model spec for ${options.modelId} has no archiveUrl or downloadFiles`); +} + +export async function ensureSherpaOnnxModels(options: { + modelsDir: string; + modelIds: SherpaOnnxModelId[]; + autoDownload: boolean; + logger: pino.Logger; +}): Promise> { + const uniq = Array.from(new Set(options.modelIds)); + const out: Partial> = {}; + for (const id of uniq) { + out[id] = await ensureSherpaOnnxModel({ + modelsDir: options.modelsDir, + modelId: id, + autoDownload: options.autoDownload, + logger: options.logger, + }); + } + return out as Record; +} diff --git a/packages/server/src/server/speech/sherpa/sherpa-offline-recognizer.ts b/packages/server/src/server/speech/sherpa/sherpa-offline-recognizer.ts new file mode 100644 index 000000000..810b8d165 --- /dev/null +++ b/packages/server/src/server/speech/sherpa/sherpa-offline-recognizer.ts @@ -0,0 +1,104 @@ +import { existsSync } from "node:fs"; +import type pino from "pino"; + +import { loadSherpaOnnxNode } from "./sherpa-onnx-node-loader.js"; + +function assertFileExists(filePath: string, label: string): void { + if (!existsSync(filePath)) { + throw new Error(`Missing ${label}: ${filePath}`); + } +} + +export type SherpaOfflineRecognizerModel = { + kind: "nemo_transducer"; + encoder: string; + decoder: string; + joiner: string; + tokens: string; +}; + +export type SherpaOfflineRecognizerConfig = { + model: SherpaOfflineRecognizerModel; + numThreads?: number; + provider?: "cpu"; + debug?: 0 | 1; + sampleRate?: number; + featureDim?: number; + decodingMethod?: "greedy_search"; + maxActivePaths?: number; +}; + +export class SherpaOfflineRecognizerEngine { + public readonly recognizer: any; + public readonly sampleRate: number; + private readonly logger: pino.Logger; + + constructor(config: SherpaOfflineRecognizerConfig, logger: pino.Logger) { + this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "offline-recognizer" }); + + assertFileExists(config.model.encoder, "offline encoder"); + assertFileExists(config.model.decoder, "offline decoder"); + assertFileExists(config.model.joiner, "offline joiner"); + assertFileExists(config.model.tokens, "tokens"); + + const sherpa = loadSherpaOnnxNode(); + + const recognizerConfig = { + featConfig: { + sampleRate: config.sampleRate ?? 16000, + featureDim: config.featureDim ?? 80, + }, + modelConfig: { + transducer: { + encoder: config.model.encoder, + decoder: config.model.decoder, + joiner: config.model.joiner, + }, + tokens: config.model.tokens, + modelType: "nemo_transducer", + numThreads: config.numThreads ?? 1, + provider: config.provider ?? "cpu", + debug: config.debug ?? 0, + }, + decodingMethod: config.decodingMethod ?? "greedy_search", + maxActivePaths: config.maxActivePaths ?? 4, + }; + + this.recognizer = new sherpa.OfflineRecognizer(recognizerConfig); + const sr = this.recognizer?.config?.featConfig?.sampleRate; + this.sampleRate = typeof sr === "number" && Number.isFinite(sr) && sr > 0 + ? sr + : recognizerConfig.featConfig.sampleRate; + + this.logger.info( + { sampleRate: this.sampleRate, numThreads: recognizerConfig.modelConfig.numThreads }, + "Sherpa offline recognizer initialized" + ); + } + + createStream(): any { + return this.recognizer.createStream(); + } + + acceptWaveform(stream: any, sampleRate: number, samples: Float32Array): void { + if (!stream || typeof stream.acceptWaveform !== "function") { + throw new Error("Unexpected sherpa offline stream: missing acceptWaveform()"); + } + + // sherpa-onnx-node expects: acceptWaveform({ samples, sampleRate }) + // sherpa-onnx (WASM) expects: acceptWaveform(sampleRate, samples) + if (stream.acceptWaveform.length <= 1) { + stream.acceptWaveform({ samples, sampleRate }); + } else { + stream.acceptWaveform(sampleRate, samples); + } + } + + free(): void { + try { + this.recognizer?.free?.(); + } catch (err) { + this.logger.warn({ err }, "Failed to free sherpa offline recognizer"); + } + } +} diff --git a/packages/server/src/server/speech/sherpa/sherpa-online-recognizer.ts b/packages/server/src/server/speech/sherpa/sherpa-online-recognizer.ts new file mode 100644 index 000000000..c7c8c3b1b --- /dev/null +++ b/packages/server/src/server/speech/sherpa/sherpa-online-recognizer.ts @@ -0,0 +1,127 @@ +import { existsSync } from "node:fs"; +import type pino from "pino"; + +import { loadSherpaOnnx } from "./sherpa-onnx-loader.js"; + +export type SherpaOnlineRecognizerModel = + | { + kind: "transducer"; + encoder: string; + decoder: string; + joiner: string; + tokens: string; + modelType?: string; + } + | { + kind: "paraformer"; + encoder: string; + decoder: string; + tokens: string; + }; + +export type SherpaOnlineRecognizerConfig = { + model: SherpaOnlineRecognizerModel; + numThreads?: number; + provider?: "cpu"; + debug?: 0 | 1; + sampleRate?: number; + featureDim?: number; + decodingMethod?: "greedy_search"; + maxActivePaths?: number; + enableEndpoint?: 0 | 1; + rule1MinTrailingSilence?: number; + rule2MinTrailingSilence?: number; + rule3MinUtteranceLength?: number; +}; + +function assertFileExists(filePath: string, label: string): void { + if (!existsSync(filePath)) { + throw new Error(`Missing ${label}: ${filePath}`); + } +} + +export class SherpaOnlineRecognizerEngine { + public readonly recognizer: any; + public readonly sampleRate: number; + private readonly logger: pino.Logger; + + constructor(config: SherpaOnlineRecognizerConfig, logger: pino.Logger) { + this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "online-recognizer" }); + + const { model } = config; + if (model.kind === "transducer") { + assertFileExists(model.encoder, "transducer encoder"); + assertFileExists(model.decoder, "transducer decoder"); + assertFileExists(model.joiner, "transducer joiner"); + assertFileExists(model.tokens, "tokens"); + } else { + assertFileExists(model.encoder, "paraformer encoder"); + assertFileExists(model.decoder, "paraformer decoder"); + assertFileExists(model.tokens, "tokens"); + } + + const sherpa = loadSherpaOnnx(); + const modelConfig = + model.kind === "transducer" + ? { + transducer: { + encoder: model.encoder, + decoder: model.decoder, + joiner: model.joiner, + }, + tokens: model.tokens, + modelType: model.modelType ?? "zipformer", + } + : { + paraformer: { + encoder: model.encoder, + decoder: model.decoder, + }, + tokens: model.tokens, + }; + + const featConfig = { + sampleRate: config.sampleRate ?? 16000, + featureDim: config.featureDim ?? 80, + }; + + const recognizerConfig = { + featConfig, + modelConfig: { + ...modelConfig, + // NOTE: In the WASM-backed `sherpa-onnx` npm package, online recognizers + // error when `numThreads > 1`. Keep the default conservative. + numThreads: config.numThreads ?? 1, + provider: config.provider ?? "cpu", + debug: config.debug ?? 0, + }, + decodingMethod: config.decodingMethod ?? "greedy_search", + maxActivePaths: config.maxActivePaths ?? 4, + enableEndpoint: config.enableEndpoint ?? 0, + rule1MinTrailingSilence: config.rule1MinTrailingSilence ?? 2.4, + rule2MinTrailingSilence: config.rule2MinTrailingSilence ?? 1.2, + rule3MinUtteranceLength: config.rule3MinUtteranceLength ?? 20, + }; + + this.recognizer = sherpa.createOnlineRecognizer(recognizerConfig); + const sr = this.recognizer?.config?.featConfig?.sampleRate; + this.sampleRate = typeof sr === "number" && Number.isFinite(sr) && sr > 0 ? sr : featConfig.sampleRate; + + this.logger.info( + { sampleRate: this.sampleRate, modelKind: model.kind, numThreads: config.numThreads ?? 2 }, + "Sherpa online recognizer initialized" + ); + } + + createStream(): any { + return this.recognizer.createStream(); + } + + free(): void { + try { + this.recognizer?.free?.(); + } catch (err) { + this.logger.warn({ err }, "Failed to free sherpa recognizer"); + } + } +} diff --git a/packages/server/src/server/speech/sherpa/sherpa-onnx-loader.ts b/packages/server/src/server/speech/sherpa/sherpa-onnx-loader.ts new file mode 100644 index 000000000..80bf1abe2 --- /dev/null +++ b/packages/server/src/server/speech/sherpa/sherpa-onnx-loader.ts @@ -0,0 +1,18 @@ +import { createRequire } from "node:module"; + +export type SherpaOnnxModule = { + createOnlineRecognizer: (config: any) => any; + createOfflineRecognizer: (config: any) => any; + createOfflineTts: (config: any) => any; +}; + +let cached: SherpaOnnxModule | null = null; + +export function loadSherpaOnnx(): SherpaOnnxModule { + if (cached) { + return cached; + } + const require = createRequire(import.meta.url); + cached = require("sherpa-onnx") as SherpaOnnxModule; + return cached; +} diff --git a/packages/server/src/server/speech/sherpa/sherpa-onnx-node-loader.ts b/packages/server/src/server/speech/sherpa/sherpa-onnx-node-loader.ts new file mode 100644 index 000000000..8318b8673 --- /dev/null +++ b/packages/server/src/server/speech/sherpa/sherpa-onnx-node-loader.ts @@ -0,0 +1,53 @@ +import { createRequire } from "node:module"; +import path from "node:path"; + +export type SherpaOnnxNodeModule = { + OfflineRecognizer: new (config: any) => any; + OnlineRecognizer?: new (config: any) => any; + OfflineTts?: new (config: any) => any; +}; + +let cached: SherpaOnnxNodeModule | null = null; + +function platformArch(): string { + const platform = process.platform === "win32" ? "win" : process.platform; + return `${platform}-${process.arch}`; +} + +function prependLibraryPath(envKey: "DYLD_LIBRARY_PATH" | "LD_LIBRARY_PATH", dir: string): void { + const current = process.env[envKey] ?? ""; + const parts = current.split(":").filter(Boolean); + if (parts.includes(dir)) { + return; + } + process.env[envKey] = [dir, ...parts].join(":"); +} + +export function loadSherpaOnnxNode(): SherpaOnnxNodeModule { + if (cached) { + return cached; + } + + const require = createRequire(import.meta.url); + + // sherpa-onnx-node depends on a platform-specific package (e.g. sherpa-onnx-darwin-arm64) + // that contains the native addon + its shared libraries. On macOS/Linux we need to ensure + // the appropriate library path env var includes that directory before requiring the addon. + const arch = platformArch(); + const pkgName = `sherpa-onnx-${arch}`; + + try { + const pkgJson = require.resolve(`${pkgName}/package.json`); + const pkgDir = path.dirname(pkgJson); + if (process.platform === "darwin") { + prependLibraryPath("DYLD_LIBRARY_PATH", pkgDir); + } else if (process.platform === "linux") { + prependLibraryPath("LD_LIBRARY_PATH", pkgDir); + } + } catch { + // Best effort - if the platform package isn't present, require() below will throw a useful error. + } + + cached = require("sherpa-onnx-node") as SherpaOnnxNodeModule; + return cached; +} diff --git a/packages/server/src/server/speech/sherpa/sherpa-parakeet-realtime-session.ts b/packages/server/src/server/speech/sherpa/sherpa-parakeet-realtime-session.ts new file mode 100644 index 000000000..6a912bd69 --- /dev/null +++ b/packages/server/src/server/speech/sherpa/sherpa-parakeet-realtime-session.ts @@ -0,0 +1,153 @@ +import { EventEmitter } from "node:events"; +import { v4 as uuidv4 } from "uuid"; + +import type { RealtimeTranscriptionSession } from "../../dictation/dictation-stream-manager.js"; +import { pcm16lePeakAbs, pcm16leToFloat32 } from "../audio.js"; +import { SherpaOfflineRecognizerEngine } from "./sherpa-offline-recognizer.js"; + +export class SherpaParakeetRealtimeTranscriptionSession + extends EventEmitter + implements RealtimeTranscriptionSession +{ + private readonly engine: SherpaOfflineRecognizerEngine; + private connected = false; + + private currentItemId: string | null = null; + private previousItemId: string | null = null; + private lastPartialText = ""; + + private pcm16 = Buffer.alloc(0); + private lastDecodeAt = 0; + private decoding = false; + private pendingDecode = false; + private readonly minDecodeIntervalMs: number; + + constructor(params: { engine: SherpaOfflineRecognizerEngine; minDecodeIntervalMs?: number }) { + super(); + this.engine = params.engine; + this.minDecodeIntervalMs = params.minDecodeIntervalMs ?? 350; + } + + async connect(): Promise { + if (this.connected) { + return; + } + this.currentItemId = uuidv4(); + this.connected = true; + } + + appendPcm16Base64(base64Audio: string): void { + if (!this.connected || !this.currentItemId) { + this.emit("error", new Error("Parakeet realtime session not connected")); + return; + } + + try { + const chunk = Buffer.from(base64Audio, "base64"); + this.pcm16 = this.pcm16.length === 0 ? chunk : Buffer.concat([this.pcm16, chunk]); + void this.maybeDecode(false); + } catch (err) { + this.emit("error", err instanceof Error ? err : new Error(String(err))); + } + } + + commit(): void { + if (!this.connected || !this.currentItemId) { + this.emit("error", new Error("Parakeet realtime session not connected")); + return; + } + + void (async () => { + try { + await this.maybeDecode(true); + const finalText = this.lastPartialText; + const itemId = this.currentItemId!; + const previousItemId = this.previousItemId; + + this.emit("committed", { itemId, previousItemId }); + this.emit("transcript", { itemId, transcript: finalText, isFinal: true }); + + this.previousItemId = itemId; + this.currentItemId = uuidv4(); + this.lastPartialText = ""; + this.pcm16 = Buffer.alloc(0); + } catch (err) { + this.emit("error", err instanceof Error ? err : new Error(String(err))); + } + })(); + } + + clear(): void { + if (!this.connected) { + return; + } + this.pcm16 = Buffer.alloc(0); + this.currentItemId = uuidv4(); + this.lastPartialText = ""; + } + + close(): void { + this.connected = false; + this.currentItemId = null; + this.pcm16 = Buffer.alloc(0); + } + + private async maybeDecode(force: boolean): Promise { + if (!this.connected || !this.currentItemId) { + return; + } + + const now = Date.now(); + if (!force && now - this.lastDecodeAt < this.minDecodeIntervalMs) { + return; + } + + if (this.decoding) { + this.pendingDecode = true; + return; + } + + this.decoding = true; + try { + const text = await this.decodeNow(); + this.lastDecodeAt = Date.now(); + if (text !== this.lastPartialText) { + this.lastPartialText = text; + this.emit("transcript", { itemId: this.currentItemId, transcript: text, isFinal: false }); + } + } finally { + this.decoding = false; + if (this.pendingDecode) { + this.pendingDecode = false; + await this.maybeDecode(true); + } + } + } + + private async decodeNow(): Promise { + if (this.pcm16.length === 0) { + return ""; + } + + const peak = pcm16lePeakAbs(this.pcm16); + const peakFloat = peak / 32768.0; + const targetPeak = 0.6; + const maxGain = 50; + const gain = peakFloat > 0 && peakFloat < targetPeak ? Math.min(maxGain, targetPeak / peakFloat) : 1; + + const stream = this.engine.createStream(); + try { + const floatSamples = pcm16leToFloat32(this.pcm16, gain); + this.engine.acceptWaveform(stream, this.engine.sampleRate, floatSamples); + this.engine.recognizer.decode(stream); + const result = this.engine.recognizer.getResult(stream); + return String(result?.text ?? result ?? "").trim(); + } finally { + try { + stream.free?.(); + } catch { + // ignore + } + } + } +} diff --git a/packages/server/src/server/speech/sherpa/sherpa-parakeet-stt.ts b/packages/server/src/server/speech/sherpa/sherpa-parakeet-stt.ts new file mode 100644 index 000000000..d3b0c151f --- /dev/null +++ b/packages/server/src/server/speech/sherpa/sherpa-parakeet-stt.ts @@ -0,0 +1,80 @@ +import type pino from "pino"; + +import type { SpeechToTextProvider, TranscriptionResult } from "../speech-provider.js"; +import { Pcm16MonoResampler } from "../../agent/pcm16-resampler.js"; +import { parsePcm16MonoWav, parsePcmRateFromFormat, pcm16lePeakAbs, pcm16leToFloat32 } from "../audio.js"; +import { SherpaOfflineRecognizerEngine } from "./sherpa-offline-recognizer.js"; + +export type SherpaParakeetSttConfig = { + engine: SherpaOfflineRecognizerEngine; + silencePeakThreshold?: number; +}; + +export class SherpaOnnxParakeetSTT implements SpeechToTextProvider { + private readonly engine: SherpaOfflineRecognizerEngine; + private readonly silencePeakThreshold: number; + private readonly logger: pino.Logger; + + constructor(config: SherpaParakeetSttConfig, logger: pino.Logger) { + this.engine = config.engine; + this.silencePeakThreshold = config.silencePeakThreshold ?? 300; + this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "parakeet-stt" }); + } + + async transcribeAudio(audioBuffer: Buffer, format: string): Promise { + const start = Date.now(); + + let inputRate: number; + let pcm16: Buffer; + + if (format.toLowerCase().includes("audio/wav")) { + const parsed = parsePcm16MonoWav(audioBuffer); + inputRate = parsed.sampleRate; + pcm16 = parsed.pcm16; + } else if (format.toLowerCase().includes("audio/pcm")) { + inputRate = parsePcmRateFromFormat(format, this.engine.sampleRate) ?? this.engine.sampleRate; + pcm16 = audioBuffer; + } else { + throw new Error(`Unsupported audio format for sherpa Parakeet STT: ${format}`); + } + + const peak = pcm16lePeakAbs(pcm16); + if (peak < this.silencePeakThreshold) { + return { text: "", duration: Date.now() - start, isLowConfidence: true }; + } + + let pcmForModel = pcm16; + if (inputRate !== this.engine.sampleRate) { + const resampler = new Pcm16MonoResampler({ inputRate, outputRate: this.engine.sampleRate }); + pcmForModel = resampler.processChunk(pcm16); + inputRate = this.engine.sampleRate; + } + + const peakForModel = pcm16lePeakAbs(pcmForModel); + const peakFloat = peakForModel / 32768.0; + const targetPeak = 0.6; + const maxGain = 50; + const gain = + peakFloat > 0 && peakFloat < targetPeak + ? Math.min(maxGain, targetPeak / peakFloat) + : 1; + + const stream = this.engine.createStream(); + try { + const floatSamples = pcm16leToFloat32(pcmForModel, gain); + this.engine.acceptWaveform(stream, inputRate, floatSamples); + this.engine.recognizer.decode(stream); + const result = this.engine.recognizer.getResult(stream); + const text = String(result?.text ?? result ?? "").trim(); + const duration = Date.now() - start; + this.logger.debug({ duration, textLength: text.length }, "Parakeet transcription complete"); + return { text, duration, ...(text.length === 0 ? { isLowConfidence: true } : {}) }; + } finally { + try { + stream.free?.(); + } catch { + // ignore + } + } + } +} diff --git a/packages/server/src/server/speech/sherpa/sherpa-realtime-session.ts b/packages/server/src/server/speech/sherpa/sherpa-realtime-session.ts new file mode 100644 index 000000000..dec4fc297 --- /dev/null +++ b/packages/server/src/server/speech/sherpa/sherpa-realtime-session.ts @@ -0,0 +1,127 @@ +import { EventEmitter } from "node:events"; +import { v4 as uuidv4 } from "uuid"; + +import type { RealtimeTranscriptionSession } from "../../dictation/dictation-stream-manager.js"; +import { pcm16lePeakAbs, pcm16leToFloat32 } from "../audio.js"; +import { SherpaOnlineRecognizerEngine } from "./sherpa-online-recognizer.js"; + +export class SherpaRealtimeTranscriptionSession + extends EventEmitter + implements RealtimeTranscriptionSession +{ + private readonly engine: SherpaOnlineRecognizerEngine; + private stream: any | null = null; + private connected = false; + + private currentItemId: string | null = null; + private previousItemId: string | null = null; + private lastPartialText = ""; + private readonly tailPaddingMs: number; + + constructor(params: { engine: SherpaOnlineRecognizerEngine; tailPaddingMs?: number }) { + super(); + this.engine = params.engine; + this.tailPaddingMs = params.tailPaddingMs ?? 500; + } + + async connect(): Promise { + if (this.connected) { + return; + } + this.stream = this.engine.createStream(); + this.currentItemId = uuidv4(); + this.connected = true; + } + + appendPcm16Base64(base64Audio: string): void { + if (!this.connected || !this.stream || !this.currentItemId) { + this.emit("error", new Error("Sherpa realtime session not connected")); + return; + } + + try { + const pcm16 = Buffer.from(base64Audio, "base64"); + const peak = pcm16lePeakAbs(pcm16); + const peakFloat = peak / 32768.0; + const targetPeak = 0.6; + const maxGain = 50; + const gain = + peakFloat > 0 && peakFloat < targetPeak + ? Math.min(maxGain, targetPeak / peakFloat) + : 1; + const floatSamples = pcm16leToFloat32(pcm16, gain); + this.stream.acceptWaveform(this.engine.sampleRate, floatSamples); + + while (this.engine.recognizer.isReady(this.stream)) { + this.engine.recognizer.decode(this.stream); + } + + const text = String(this.engine.recognizer.getResult(this.stream)?.text ?? "").trim(); + if (text !== this.lastPartialText) { + this.lastPartialText = text; + this.emit("transcript", { itemId: this.currentItemId, transcript: text, isFinal: false }); + } + } catch (err) { + this.emit("error", err instanceof Error ? err : new Error(String(err))); + } + } + + commit(): void { + if (!this.connected || !this.stream || !this.currentItemId) { + this.emit("error", new Error("Sherpa realtime session not connected")); + return; + } + + try { + const padSamples = Math.max(0, Math.round((this.engine.sampleRate * this.tailPaddingMs) / 1000)); + if (padSamples > 0) { + this.stream.acceptWaveform(this.engine.sampleRate, new Float32Array(padSamples)); + } + + while (this.engine.recognizer.isReady(this.stream)) { + this.engine.recognizer.decode(this.stream); + } + + const finalText = String(this.engine.recognizer.getResult(this.stream)?.text ?? "").trim(); + const itemId = this.currentItemId; + const previousItemId = this.previousItemId; + + this.emit("committed", { itemId, previousItemId }); + this.emit("transcript", { itemId, transcript: finalText, isFinal: true }); + + this.previousItemId = itemId; + this.currentItemId = uuidv4(); + this.lastPartialText = ""; + this.engine.recognizer.reset(this.stream); + } catch (err) { + this.emit("error", err instanceof Error ? err : new Error(String(err))); + } + } + + clear(): void { + if (!this.connected || !this.stream) { + return; + } + try { + this.engine.recognizer.reset(this.stream); + this.currentItemId = uuidv4(); + this.lastPartialText = ""; + } catch (err) { + this.emit("error", err instanceof Error ? err : new Error(String(err))); + } + } + + close(): void { + if (!this.stream) { + return; + } + try { + this.stream.free?.(); + } catch { + // ignore + } finally { + this.stream = null; + this.connected = false; + } + } +} diff --git a/packages/server/src/server/speech/sherpa/sherpa-stt.ts b/packages/server/src/server/speech/sherpa/sherpa-stt.ts new file mode 100644 index 000000000..863b4270d --- /dev/null +++ b/packages/server/src/server/speech/sherpa/sherpa-stt.ts @@ -0,0 +1,95 @@ +import type pino from "pino"; + +import type { SpeechToTextProvider, TranscriptionResult } from "../speech-provider.js"; +import { Pcm16MonoResampler } from "../../agent/pcm16-resampler.js"; +import { parsePcm16MonoWav, parsePcmRateFromFormat, pcm16lePeakAbs, pcm16leToFloat32 } from "../audio.js"; +import { SherpaOnlineRecognizerEngine } from "./sherpa-online-recognizer.js"; + +export type SherpaSttConfig = { + engine: SherpaOnlineRecognizerEngine; + silencePeakThreshold?: number; + tailPaddingMs?: number; +}; + +export class SherpaOnnxSTT implements SpeechToTextProvider { + private readonly engine: SherpaOnlineRecognizerEngine; + private readonly silencePeakThreshold: number; + private readonly tailPaddingMs: number; + private readonly logger: pino.Logger; + + constructor(config: SherpaSttConfig, logger: pino.Logger) { + this.engine = config.engine; + this.silencePeakThreshold = config.silencePeakThreshold ?? 300; + this.tailPaddingMs = config.tailPaddingMs ?? 500; + this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "stt" }); + } + + async transcribeAudio(audioBuffer: Buffer, format: string): Promise { + const start = Date.now(); + + let inputRate: number; + let pcm16: Buffer; + + if (format.toLowerCase().includes("audio/wav")) { + const parsed = parsePcm16MonoWav(audioBuffer); + inputRate = parsed.sampleRate; + pcm16 = parsed.pcm16; + } else if (format.toLowerCase().includes("audio/pcm")) { + inputRate = parsePcmRateFromFormat(format, this.engine.sampleRate) ?? this.engine.sampleRate; + pcm16 = audioBuffer; + } else { + throw new Error(`Unsupported audio format for sherpa STT: ${format}`); + } + + const peak = pcm16lePeakAbs(pcm16); + if (peak < this.silencePeakThreshold) { + return { text: "", duration: Date.now() - start, isLowConfidence: true }; + } + + let pcmForModel = pcm16; + if (inputRate !== this.engine.sampleRate) { + const resampler = new Pcm16MonoResampler({ inputRate, outputRate: this.engine.sampleRate }); + pcmForModel = resampler.processChunk(pcm16); + inputRate = this.engine.sampleRate; + } + + const peakForModel = pcm16lePeakAbs(pcmForModel); + const peakFloat = peakForModel / 32768.0; + const targetPeak = 0.6; + const maxGain = 50; + const gain = + peakFloat > 0 && peakFloat < targetPeak + ? Math.min(maxGain, targetPeak / peakFloat) + : 1; + + const stream = this.engine.createStream(); + try { + const floatSamples = pcm16leToFloat32(pcmForModel, gain); + stream.acceptWaveform(inputRate, floatSamples); + + while (this.engine.recognizer.isReady(stream)) { + this.engine.recognizer.decode(stream); + } + + const padSamples = Math.max(0, Math.round((this.engine.sampleRate * this.tailPaddingMs) / 1000)); + if (padSamples > 0) { + stream.acceptWaveform(this.engine.sampleRate, new Float32Array(padSamples)); + } + + while (this.engine.recognizer.isReady(stream)) { + this.engine.recognizer.decode(stream); + } + + const text = String(this.engine.recognizer.getResult(stream)?.text ?? "").trim(); + const duration = Date.now() - start; + this.logger.debug({ duration, textLength: text.length }, "Sherpa transcription complete"); + return { text, duration, ...(text.length === 0 ? { isLowConfidence: true } : {}) }; + } finally { + try { + stream.free?.(); + } catch { + // ignore + } + } + } +} diff --git a/packages/server/src/server/speech/sherpa/sherpa-tts.ts b/packages/server/src/server/speech/sherpa/sherpa-tts.ts new file mode 100644 index 000000000..2b5482248 --- /dev/null +++ b/packages/server/src/server/speech/sherpa/sherpa-tts.ts @@ -0,0 +1,121 @@ +import type pino from "pino"; +import { Readable } from "node:stream"; +import { existsSync } from "node:fs"; + +import type { SpeechStreamResult, TextToSpeechProvider } from "../speech-provider.js"; +import { chunkBuffer, float32ToPcm16le } from "../audio.js"; +import { loadSherpaOnnx } from "./sherpa-onnx-loader.js"; + +export type SherpaTtsPreset = "kokoro-en-v0_19" | "kitten-nano-en-v0_1-fp16"; + +export type SherpaTtsConfig = { + preset: SherpaTtsPreset; + modelDir: string; + speakerId?: number; + speed?: number; + lengthScale?: number; + numThreads?: number; +}; + +function assertFileExists(filePath: string, label: string): void { + if (!existsSync(filePath)) { + throw new Error(`Missing ${label}: ${filePath}`); + } +} + +export class SherpaOnnxTTS implements TextToSpeechProvider { + private readonly tts: any; + private readonly speakerId: number; + private readonly speed: number; + private readonly logger: pino.Logger; + + constructor(config: SherpaTtsConfig, logger: pino.Logger) { + if (config.preset !== "kokoro-en-v0_19" && config.preset !== "kitten-nano-en-v0_1-fp16") { + throw new Error(`Unsupported Sherpa TTS preset: ${config.preset}`); + } + this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "tts" }); + this.speakerId = config.speakerId ?? 0; + this.speed = config.speed ?? 1.0; + + const sherpa = loadSherpaOnnx(); + + const modelFile = config.preset === "kokoro-en-v0_19" ? "model.onnx" : "model.fp16.onnx"; + const modelPath = `${config.modelDir}/${modelFile}`; + const voicesPath = `${config.modelDir}/voices.bin`; + const tokensPath = `${config.modelDir}/tokens.txt`; + const dataDir = `${config.modelDir}/espeak-ng-data`; + + assertFileExists(modelPath, "TTS model"); + assertFileExists(voicesPath, "TTS voices"); + assertFileExists(tokensPath, "TTS tokens"); + assertFileExists(dataDir, "TTS espeak-ng dataDir"); + + const modelConfigKey = + config.preset === "kokoro-en-v0_19" + ? "offlineTtsKokoroModelConfig" + : "offlineTtsKittenModelConfig"; + + const modelConfig = { + [modelConfigKey]: { + model: modelPath, + voices: voicesPath, + tokens: tokensPath, + dataDir, + lengthScale: config.lengthScale ?? 1.0, + }, + numThreads: config.numThreads ?? 2, + debug: 0, + provider: "cpu", + }; + + const offlineTtsConfig = { + offlineTtsModelConfig: modelConfig, + maxNumSentences: 1, + }; + + this.tts = sherpa.createOfflineTts(offlineTtsConfig); + this.logger.info({ preset: config.preset, modelDir: config.modelDir }, "Sherpa offline TTS initialized"); + } + + async synthesizeSpeech(text: string): Promise { + const trimmed = text.trim(); + if (!trimmed) { + throw new Error("Cannot synthesize empty text"); + } + + const audio = this.tts.generate({ text: trimmed, sid: this.speakerId, speed: this.speed }); + const samples: Float32Array | null = + audio && audio.samples instanceof Float32Array + ? audio.samples + : audio && Array.isArray(audio.samples) + ? Float32Array.from(audio.samples as number[]) + : null; + const sampleRate: number = + audio && typeof audio.sampleRate === "number" && Number.isFinite(audio.sampleRate) && audio.sampleRate > 0 + ? audio.sampleRate + : typeof this.tts.sampleRate === "number" + ? this.tts.sampleRate + : 24000; + + if (!samples) { + throw new Error("Unexpected sherpa TTS output: missing Float32 samples"); + } + + const pcm16 = float32ToPcm16le(samples); + const chunkBytes = Math.max(2, Math.round(sampleRate * 0.05) * 2); // ~50ms + const chunks = chunkBuffer(pcm16, chunkBytes); + + return { + stream: Readable.from(chunks), + format: `pcm;rate=${sampleRate}`, + }; + } + + free(): void { + try { + this.tts?.free?.(); + } catch { + // ignore + } + } +} diff --git a/packages/server/src/server/speech/sherpa/speech-download.e2e.test.ts b/packages/server/src/server/speech/sherpa/speech-download.e2e.test.ts new file mode 100644 index 000000000..da828a29d --- /dev/null +++ b/packages/server/src/server/speech/sherpa/speech-download.e2e.test.ts @@ -0,0 +1,264 @@ +import { describe, expect, test } from "vitest"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import pino from "pino"; + +import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./model-downloader.js"; +import type { SherpaOnnxModelId } from "./model-catalog.js"; +import { createDaemonTestContext } from "../../test-utils/index.js"; +import { parsePcm16MonoWav, wordSimilarity } from "../../test-utils/dictation-e2e.js"; +import { SherpaOnnxTTS } from "./sherpa-tts.js"; +import { PocketTtsOnnxTTS } from "../pocket/pocket-tts-onnx.js"; +import { SherpaOnlineRecognizerEngine } from "./sherpa-online-recognizer.js"; +import { SherpaOnnxSTT } from "./sherpa-stt.js"; +import { SherpaOfflineRecognizerEngine } from "./sherpa-offline-recognizer.js"; +import { SherpaOnnxParakeetSTT } from "./sherpa-parakeet-stt.js"; + +const RUN = process.env.PASEO_SPEECH_E2E_DOWNLOAD === "1"; +const downloadTest = RUN ? test : test.skip; + +type ModelSet = "zipformer-kitten" | "parakeet-pocket"; + +function getModelSet(): ModelSet { + const raw = (process.env.PASEO_SPEECH_E2E_MODEL_SET ?? "parakeet-pocket").trim().toLowerCase(); + if (raw === "zipformer-kitten" || raw === "zipformer") return "zipformer-kitten"; + if (raw === "parakeet-pocket" || raw === "parakeet") return "parakeet-pocket"; + throw new Error(`Unknown PASEO_SPEECH_E2E_MODEL_SET: ${raw}`); +} + +async function readFixtureWav(): Promise { + const fixturePath = path.resolve(process.cwd(), "..", "app", "e2e", "fixtures", "recording.wav"); + return import("node:fs/promises").then((fs) => fs.readFile(fixturePath)); +} + +async function readBaseline(): Promise { + const baselinePath = path.resolve(process.cwd(), "..", "app", "e2e", "fixtures", "recording.baseline.txt"); + return import("node:fs/promises").then((fs) => fs.readFile(baselinePath, "utf-8")).then((t) => t.trim()); +} + +async function readAllChunks(stream: NodeJS.ReadableStream): Promise<{ chunks: Buffer[]; combined: Buffer }> { + const chunks: Buffer[] = []; + for await (const chunk of stream as any) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return { chunks, combined: Buffer.concat(chunks) }; +} + +function waitForSignal( + timeoutMs: number, + setup: ( + resolve: (value: T) => void, + reject: (error: Error) => void + ) => () => void +): Promise { + return new Promise((resolve, reject) => { + let cleanup: (() => void) | null = null; + const timeout = setTimeout(() => { + cleanup?.(); + reject(new Error(`Timeout waiting for event after ${timeoutMs}ms`)); + }, timeoutMs); + + cleanup = setup( + (value) => { + clearTimeout(timeout); + cleanup?.(); + resolve(value); + }, + (error) => { + clearTimeout(timeout); + cleanup?.(); + reject(error); + } + ); + }); +} + +function toAudioPcmFormat(format: string): string { + const trimmed = format.trim(); + const lower = trimmed.toLowerCase(); + if (lower.startsWith("audio/pcm")) { + return trimmed; + } + if (lower.startsWith("pcm")) { + const rest = trimmed.replace(/^pcm;?/i, ""); + return rest ? `audio/pcm;${rest}` : "audio/pcm"; + } + // Fall back to treating it as a suffix (e.g. "rate=24000") + return `audio/pcm;${trimmed}`; +} + +describe("speech models (download E2E)", () => { + downloadTest( + "downloads models and runs streaming STT + streaming TTS", + async () => { + const logger = pino({ level: "silent" }); + const set = getModelSet(); + + const paseoHomeRoot = mkdtempSync(path.join(tmpdir(), "paseo-speech-download-")); + const modelsDir = path.join(paseoHomeRoot, ".paseo", "models", "sherpa-onnx"); + + const modelIds: SherpaOnnxModelId[] = + set === "parakeet-pocket" + ? ["parakeet-tdt-0.6b-v3-int8", "pocket-tts-onnx-int8"] + : ["zipformer-bilingual-zh-en-2023-02-20", "kitten-nano-en-v0_1-fp16"]; + + await ensureSherpaOnnxModels({ + modelsDir, + modelIds, + autoDownload: true, + logger, + }); + + const ctx = await createDaemonTestContext({ + paseoHomeRoot, + dictationFinalTimeoutMs: 8000, + speech: { + dictationSttProvider: "sherpa", + voiceSttProvider: "sherpa", + voiceTtsProvider: "sherpa", + sherpaOnnx: { + modelsDir, + autoDownload: false, + stt: { preset: set === "parakeet-pocket" ? "parakeet-tdt-0.6b-v3-int8" : "zipformer-bilingual-zh-en-2023-02-20" }, + tts: { preset: set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16" }, + }, + }, + }); + + try { + const wav = await readFixtureWav(); + const { sampleRate, pcm16 } = parsePcm16MonoWav(wav); + expect(sampleRate).toBe(16000); + const format = "audio/pcm;rate=16000;bits=16"; + + // Streaming STT: dictation path (verifies websocket streaming + partials + final) + const dictationId = `dict-download-${Date.now()}`; + let partialCount = 0; + const unsubscribe = ctx.client.on("dictation_stream_partial", (message) => { + if (message.type !== "dictation_stream_partial") return; + if (message.payload.dictationId !== dictationId) return; + partialCount += 1; + }); + + await ctx.client.startDictationStream(dictationId, format); + const chunkBytes = 3200; // 100ms @ 16kHz mono PCM16 + let seq = 0; + for (let offset = 0; offset < pcm16.length; offset += chunkBytes) { + const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes)); + ctx.client.sendDictationStreamChunk(dictationId, seq, chunk.toString("base64"), format); + seq += 1; + } + const finalSeq = seq - 1; + const dictationFinal = await ctx.client.finishDictationStream(dictationId, finalSeq); + unsubscribe(); + + expect(dictationFinal.text.toLowerCase()).toContain("voice note"); + const baseline = await readBaseline(); + expect(wordSimilarity(dictationFinal.text, baseline)).toBeGreaterThan(0.45); + expect(partialCount).toBeGreaterThan(0); + + // Voice-mode STT: chunked upload until isLast=true + const transcriptionPromise = waitForSignal(30000, (resolve, reject) => { + const offResult = ctx.client.on("transcription_result", (message) => { + if (message.type !== "transcription_result") return; + resolve(message.payload.text); + }); + const offError = ctx.client.on("activity_log", (message) => { + if (message.type !== "activity_log") return; + const payload = message.payload as { type?: unknown; content?: unknown }; + if (payload.type !== "error") return; + const content = typeof payload.content === "string" ? payload.content : null; + if (!content) return; + reject(new Error(content)); + }); + return () => { + offResult(); + offError(); + }; + }); + + await ctx.client.setVoiceConversation(true, `voice-download-${Date.now()}`); + for (let offset = 0; offset < pcm16.length; offset += chunkBytes) { + const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes)); + const isLast = offset + chunkBytes >= pcm16.length; + await ctx.client.sendVoiceAudioChunk(chunk.toString("base64"), format, isLast); + } + const voiceText = (await transcriptionPromise).trim().toLowerCase(); + if (voiceText.length > 0) { + expect(voiceText).toContain("voice note"); + } + await ctx.client.setVoiceConversation(false); + + // Streaming TTS: generate locally from downloaded model and validate chunking. + const ttsText = "This is a voice note."; + if (set === "parakeet-pocket") { + const modelDir = getSherpaOnnxModelDir(modelsDir, "pocket-tts-onnx-int8"); + const tts = await PocketTtsOnnxTTS.create({ modelDir, precision: "int8", targetChunkMs: 50 }, logger); + const { stream, format: ttsFormat } = await tts.synthesizeSpeech(ttsText); + const { chunks, combined } = await readAllChunks(stream); + + expect(ttsFormat).toMatch(/rate=\d+/); + expect(chunks.length).toBeGreaterThan(3); + expect(combined.byteLength).toBeGreaterThan(2000); + + // Round trip: TTS -> STT (offline parakeet) + const sttModelDir = getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v3-int8"); + const engine = new SherpaOfflineRecognizerEngine( + { + model: { + kind: "nemo_transducer", + encoder: `${sttModelDir}/encoder.int8.onnx`, + decoder: `${sttModelDir}/decoder.int8.onnx`, + joiner: `${sttModelDir}/joiner.int8.onnx`, + tokens: `${sttModelDir}/tokens.txt`, + }, + numThreads: 2, + debug: 0, + }, + logger + ); + const stt = new SherpaOnnxParakeetSTT({ engine }, logger); + const rt = await stt.transcribeAudio(combined, toAudioPcmFormat(ttsFormat)); + engine.free(); + expect(wordSimilarity(rt.text, ttsText)).toBeGreaterThan(0.25); + } else { + const ttsModelDir = path.join(modelsDir, "kitten-nano-en-v0_1-fp16"); + const tts = new SherpaOnnxTTS({ preset: "kitten-nano-en-v0_1-fp16", modelDir: ttsModelDir }, logger); + const { stream, format: ttsFormat } = await tts.synthesizeSpeech(ttsText); + const { chunks, combined } = await readAllChunks(stream); + tts.free(); + + expect(ttsFormat).toMatch(/rate=\d+/); + expect(chunks.length).toBeGreaterThan(3); + expect(combined.byteLength).toBeGreaterThan(2000); + + // Round trip: TTS -> STT (online zipformer, offline segment) + const sttModelDir = path.join(modelsDir, "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20"); + const engine = new SherpaOnlineRecognizerEngine( + { + model: { + kind: "transducer", + encoder: `${sttModelDir}/encoder-epoch-99-avg-1.onnx`, + decoder: `${sttModelDir}/decoder-epoch-99-avg-1.onnx`, + joiner: `${sttModelDir}/joiner-epoch-99-avg-1.onnx`, + tokens: `${sttModelDir}/tokens.txt`, + modelType: "zipformer", + }, + numThreads: 1, + debug: 0, + }, + logger + ); + const stt = new SherpaOnnxSTT({ engine }, logger); + const rt = await stt.transcribeAudio(combined, toAudioPcmFormat(ttsFormat)); + engine.free(); + expect(wordSimilarity(rt.text, ttsText)).toBeGreaterThan(0.25); + } + } finally { + await ctx.cleanup(); + } + }, + 15 * 60_000 + ); +}); diff --git a/packages/server/src/server/speech/speech-provider.ts b/packages/server/src/server/speech/speech-provider.ts new file mode 100644 index 000000000..3ed103666 --- /dev/null +++ b/packages/server/src/server/speech/speech-provider.ts @@ -0,0 +1,30 @@ +import type { Readable } from "node:stream"; + +export interface LogprobToken { + token: string; + logprob: number; + bytes?: number[]; +} + +export interface TranscriptionResult { + text: string; + language?: string; + duration?: number; + logprobs?: LogprobToken[]; + avgLogprob?: number; + isLowConfidence?: boolean; +} + +export interface SpeechToTextProvider { + transcribeAudio(audioBuffer: Buffer, format: string): Promise; +} + +export interface SpeechStreamResult { + stream: Readable; + format: string; +} + +export interface TextToSpeechProvider { + synthesizeSpeech(text: string): Promise; +} + diff --git a/packages/server/src/server/test-utils/paseo-daemon.ts b/packages/server/src/server/test-utils/paseo-daemon.ts index 89028cde6..7024991b8 100644 --- a/packages/server/src/server/test-utils/paseo-daemon.ts +++ b/packages/server/src/server/test-utils/paseo-daemon.ts @@ -4,7 +4,7 @@ import path from "node:path"; import { mkdir, mkdtemp, rm } from "node:fs/promises"; import pino from "pino"; -import { createPaseoDaemon, type PaseoDaemonConfig, type PaseoOpenAIConfig } from "../bootstrap.js"; +import { createPaseoDaemon, type PaseoDaemonConfig, type PaseoOpenAIConfig, type PaseoSpeechConfig } from "../bootstrap.js"; import type { AgentClient, AgentProvider } from "../agent/agent-sdk-types.js"; import { createTestAgentClients } from "./fake-agent-client.js"; @@ -20,6 +20,7 @@ type TestPaseoDaemonOptions = { staticDir?: string; cleanup?: boolean; openai?: PaseoOpenAIConfig; + speech?: PaseoSpeechConfig; dictationFinalTimeoutMs?: number; }; @@ -76,6 +77,7 @@ export async function createTestPaseoDaemon( relayEndpoint: options.relayEndpoint ?? "relay.paseo.sh:443", appBaseUrl: "https://app.paseo.sh", openai: options.openai, + speech: options.speech, openrouterApiKey: null, dictationFinalTimeoutMs: options.dictationFinalTimeoutMs, downloadTokenTtlMs: options.downloadTokenTtlMs, diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index f3691c1fd..b7e84f4bf 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -6,8 +6,6 @@ import { hostname as getHostname } from "node:os"; import type { AgentManager } from "./agent/agent-manager.js"; import type { AgentStorage } from "./agent/agent-storage.js"; import type { DownloadTokenStore } from "./file-download/token-store.js"; -import type { OpenAISTT } from "./agent/stt-openai.js"; -import type { OpenAITTS } from "./agent/tts-openai.js"; import type { TerminalManager } from "../terminal/terminal-manager.js"; import type pino from "pino"; import { @@ -22,6 +20,8 @@ import type { AgentProvider } from "./agent/agent-sdk-types.js"; import { PushTokenStore } from "./push/token-store.js"; import { PushService } from "./push/push-service.js"; import { VoiceConversationStore } from "./voice-conversation-store.js"; +import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js"; +import type { RealtimeTranscriptionSessionFactory } from "./dictation/dictation-stream-manager.js"; export type AgentMcpTransportFactory = () => Promise; @@ -67,13 +67,14 @@ export class VoiceAssistantWebSocketServer { private readonly pushTokenStore: PushTokenStore; private readonly pushService: PushService; private readonly createAgentMcpTransport: AgentMcpTransportFactory; - private readonly stt: OpenAISTT | null; - private readonly tts: OpenAITTS | null; + private readonly stt: SpeechToTextProvider | null; + private readonly tts: TextToSpeechProvider | null; private readonly terminalManager: TerminalManager | null; private readonly voiceConversationStore: VoiceConversationStore; private readonly dictation: { openaiApiKey?: string | null; finalTimeoutMs?: number; + sessionFactory?: RealtimeTranscriptionSessionFactory; } | null; private readonly voice: { openrouterApiKey?: string | null; @@ -90,7 +91,7 @@ export class VoiceAssistantWebSocketServer { paseoHome: string, createAgentMcpTransport: AgentMcpTransportFactory, wsConfig: WebSocketServerConfig, - speech?: { stt: OpenAISTT | null; tts: OpenAITTS | null }, + speech?: { stt: SpeechToTextProvider | null; tts: TextToSpeechProvider | null }, terminalManager?: TerminalManager | null, voice?: { openrouterApiKey?: string | null; @@ -99,6 +100,7 @@ export class VoiceAssistantWebSocketServer { dictation?: { openaiApiKey?: string | null; finalTimeoutMs?: number; + sessionFactory?: RealtimeTranscriptionSessionFactory; } ) { this.logger = logger.child({ module: "websocket-server" }); diff --git a/scripts/speech/download-sherpa-models.sh b/scripts/speech/download-sherpa-models.sh new file mode 100755 index 000000000..5a3ccdc5c --- /dev/null +++ b/scripts/speech/download-sherpa-models.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Download local speech models for Paseo (sherpa-onnx). + +Defaults: + - STT: sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 + - TTS: kitten-nano-en-v0_1-fp16 + +Usage: + scripts/speech/download-sherpa-models.sh [--models-dir DIR] [--with-kokoro] [--with-paraformer] + +Preferred: + npm run speech:download --workspace=@getpaseo/server + +Notes: + - Models are downloaded from the sherpa-onnx GitHub releases. + - Pocket TTS is downloaded by the Node script (`npm run speech:download --workspace=@getpaseo/server`) + because it is a file-based HuggingFace model (not a single tarball). + - Set PASEO_SHERPA_ONNX_MODELS_DIR to override where the daemon looks. +EOF +} + +MODELS_DIR="" +WITH_KOKORO=0 +WITH_PARAFORMER=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --models-dir) + MODELS_DIR="${2:-}" + shift 2 + ;; + --with-kokoro) + WITH_KOKORO=1 + shift 1 + ;; + --with-paraformer) + WITH_PARAFORMER=1 + shift 1 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown arg: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ -z "${MODELS_DIR}" ]]; then + if [[ -n "${PASEO_SHERPA_ONNX_MODELS_DIR:-}" ]]; then + MODELS_DIR="${PASEO_SHERPA_ONNX_MODELS_DIR}" + elif [[ -n "${PASEO_HOME:-}" ]]; then + MODELS_DIR="${PASEO_HOME}/models/sherpa-onnx" + else + MODELS_DIR="${HOME}/.paseo/models/sherpa-onnx" + fi +fi + +mkdir -p "${MODELS_DIR}" +cd "${MODELS_DIR}" + +download_and_extract() { + local url="$1" + local filename + filename="$(basename "$url")" + + echo "Downloading ${filename}..." + curl -fsSL -O "${url}" + echo "Extracting ${filename}..." + tar xf "${filename}" + rm -f "${filename}" +} + +echo "NOTE: This script is deprecated. Prefer: npm run speech:download --workspace=@getpaseo/server" >&2 + +download_and_extract "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2" +download_and_extract "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2" + +if [[ "${WITH_PARAFORMER}" -eq 1 ]]; then + download_and_extract "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2" +fi + +if [[ "${WITH_KOKORO}" -eq 1 ]]; then + download_and_extract "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2" +fi + +echo "Done." +echo "Models dir: ${MODELS_DIR}"