From 0bfda11c1f0b2a7faed89a712bdfc0c13be1a5ea Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Sun, 19 Oct 2025 23:23:32 +0200 Subject: [PATCH] feat: add Playwright MCP integration and TTS improvements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Integrate Playwright MCP for browser automation capabilities - Add TTS manager for improved audio synthesis management - Add ToolCallCard UI component for better tool call visualization - Upgrade Vite to v7.1.10 for improved build performance - Add lucide-react for icon components - Add dotenv for environment variable management - Update agent prompt and orchestrator for enhanced interactions - Improve audio playback and voice controls in UI 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- package-lock.json | 900 +++++++++--------- packages/voice-assistant/agent-prompt.md | 479 +--------- packages/voice-assistant/package.json | 7 +- packages/voice-assistant/playwright.config.ts | 30 + .../src/server/agent/llm-openai.ts | 202 ++-- .../src/server/agent/orchestrator.ts | 116 ++- .../src/server/agent/tts-manager.ts | 88 ++ .../src/server/agent/tts-openai.ts | 1 + .../src/server/daemon/terminal-manager.ts | 151 ++- .../voice-assistant/src/server/daemon/tmux.ts | 75 +- .../src/server/daemon/tool-definitions.ts | 11 +- packages/voice-assistant/src/server/index.ts | 87 +- packages/voice-assistant/src/server/types.ts | 6 +- .../src/server/websocket-server.ts | 95 +- packages/voice-assistant/src/ui/App.css | 346 ++++++- packages/voice-assistant/src/ui/App.tsx | 290 ++++-- .../src/ui/components/ToolCallCard.tsx | 61 ++ .../src/ui/components/VoiceControls.tsx | 8 +- packages/voice-assistant/src/ui/index.css | 15 +- .../src/ui/lib/audio-playback.ts | 70 +- 20 files changed, 1721 insertions(+), 1317 deletions(-) create mode 100644 packages/voice-assistant/playwright.config.ts create mode 100644 packages/voice-assistant/src/server/agent/tts-manager.ts create mode 100644 packages/voice-assistant/src/ui/components/ToolCallCard.tsx diff --git a/package-lock.json b/package-lock.json index 387571e9e..5ce2b8ef7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -972,6 +972,22 @@ "node": ">=8.0.0" } }, + "node_modules/@playwright/test": { + "version": "1.56.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.56.1.tgz", + "integrity": "sha512-vSMYtL/zOcFpvJCW71Q/OEGQb7KYBPAdKh35WNSkaZA75JlAO8ED8UN6GUNTm3drWomcbcqRPFqQbLae8yBTdg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.56.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@rolldown/pluginutils": { "version": "1.0.0-beta.27", "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.27.tgz", @@ -2279,6 +2295,18 @@ "node": ">=8" } }, + "node_modules/dotenv": { + "version": "17.2.3", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", + "integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -2522,6 +2550,24 @@ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", "license": "MIT" }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, "node_modules/finalhandler": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", @@ -2825,11 +2871,22 @@ "node": ">=8" } }, + "node_modules/jiti": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.6.1.tgz", + "integrity": "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "bin": { + "jiti": "lib/jiti-cli.mjs" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "dev": true, "license": "MIT" }, "node_modules/jsesc": { @@ -2864,6 +2921,257 @@ "node": ">=6" } }, + "node_modules/lightningcss": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.30.1.tgz", + "integrity": "sha512-xi6IyHML+c9+Q3W0S4fCQJOym42pyurFiJUHEcEyHS0CeKzia4yZDEsLlqOFykxOdHpNy0NmvVO31vcSqAxJCg==", + "dev": true, + "license": "MPL-2.0", + "optional": true, + "peer": true, + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-darwin-arm64": "1.30.1", + "lightningcss-darwin-x64": "1.30.1", + "lightningcss-freebsd-x64": "1.30.1", + "lightningcss-linux-arm-gnueabihf": "1.30.1", + "lightningcss-linux-arm64-gnu": "1.30.1", + "lightningcss-linux-arm64-musl": "1.30.1", + "lightningcss-linux-x64-gnu": "1.30.1", + "lightningcss-linux-x64-musl": "1.30.1", + "lightningcss-win32-arm64-msvc": "1.30.1", + "lightningcss-win32-x64-msvc": "1.30.1" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.30.1.tgz", + "integrity": "sha512-c8JK7hyE65X1MHMN+Viq9n11RRC7hgin3HhYKhrMyaXflk5GVplZ60IxyoVtzILeKr+xAJwg6zK6sjTBJ0FKYQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.30.1.tgz", + "integrity": "sha512-k1EvjakfumAQoTfcXUcHQZhSpLlkAuEkdMBsI/ivWw9hL+7FtilQc0Cy3hrx0AAQrVtQAbMI7YjCgYgvn37PzA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.30.1.tgz", + "integrity": "sha512-kmW6UGCGg2PcyUE59K5r0kWfKPAVy4SltVeut+umLCFoJ53RdCUWxcRDzO1eTaxf/7Q2H7LTquFHPL5R+Gjyig==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.30.1.tgz", + "integrity": "sha512-MjxUShl1v8pit+6D/zSPq9S9dQ2NPFSQwGvxBCYaBYLPlCWuPh9/t1MRS8iUaR8i+a6w7aps+B4N0S1TYP/R+Q==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.30.1.tgz", + "integrity": "sha512-gB72maP8rmrKsnKYy8XUuXi/4OctJiuQjcuqWNlJQ6jZiWqtPvqFziskH3hnajfvKB27ynbVCucKSm2rkQp4Bw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.30.1.tgz", + "integrity": "sha512-jmUQVx4331m6LIX+0wUhBbmMX7TCfjF5FoOH6SD1CttzuYlGNVpA7QnrmLxrsub43ClTINfGSYyHe2HWeLl5CQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.30.1.tgz", + "integrity": "sha512-piWx3z4wN8J8z3+O5kO74+yr6ze/dKmPnI7vLqfSqI8bccaTGY5xiSGVIJBDd5K5BHlvVLpUB3S2YCfelyJ1bw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.30.1.tgz", + "integrity": "sha512-rRomAK7eIkL+tHY0YPxbc5Dra2gXlI63HL+v1Pdi1a3sC+tJTcFrHX+E86sulgAXeI7rSzDYhPSeHHjqFhqfeQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.30.1.tgz", + "integrity": "sha512-mSL4rqPi4iXq5YVqzSsJgMVFENoa4nGTT/GjO2c0Yl9OuQfPsIfncvLrEW6RbbB24WtZ3xP/2CCmI3tNkNV4oA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.30.1.tgz", + "integrity": "sha512-PVqXh48wh4T53F/1CCu8PIPCxLzWyCnn/9T5W1Jpmdy5h9Cwd+0YQS6/LwhHXSafuc61/xg9Lv5OrCby6a++jg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, "node_modules/lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", @@ -2875,7 +3183,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", - "dev": true, "license": "MIT", "dependencies": { "js-tokens": "^3.0.0 || ^4.0.0" @@ -2901,6 +3208,15 @@ "dev": true, "license": "ISC" }, + "node_modules/lucide-react": { + "version": "0.546.0", + "resolved": "https://registry.npmjs.org/lucide-react/-/lucide-react-0.546.0.tgz", + "integrity": "sha512-Z94u6fKT43lKeYHiVyvyR8fT7pwCzDu7RyMPpTvh054+xahSgj4HFQ+NmflvzdXsoAjYGdCguGaFKYuvq0ThCQ==", + "license": "ISC", + "peerDependencies": { + "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -3163,6 +3479,66 @@ "dev": true, "license": "ISC" }, + "node_modules/picomatch": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", + "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/playwright": { + "version": "1.56.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.56.1.tgz", + "integrity": "sha512-aFi5B0WovBHTEvpM3DzXTUaeN6eN0qWnTkKx4NQaH4Wvcmc153PdaY2UBdSYKaGYw+UyWXSVyxDUg5DoPEttjw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.56.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.56.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.56.1.tgz", + "integrity": "sha512-hutraynyn31F+Bifme+Ps9Vq59hKuUCz7H1kDOcBs+2oGguKkWTU50bBWrtz34OUWmIwpBTWDxaRPXrIXkgvmQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/postcss": { "version": "8.5.6", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz", @@ -3244,6 +3620,18 @@ "node": ">= 0.8" } }, + "node_modules/react": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", + "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/react-refresh": { "version": "0.17.0", "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.17.0.tgz", @@ -3597,6 +3985,23 @@ "integrity": "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==", "license": "MIT" }, + "node_modules/tinyglobby": { + "version": "0.2.15", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", + "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.3" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, "node_modules/toidentifier": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", @@ -3739,21 +4144,24 @@ } }, "node_modules/vite": { - "version": "5.4.20", - "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.20.tgz", - "integrity": "sha512-j3lYzGC3P+B5Yfy/pfKNgVEg4+UtcIJcVRt2cDjIOmhLourAqPqf8P7acgxeiSgUB7E3p2P8/3gNIgDLpwzs4g==", + "version": "7.1.10", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.1.10.tgz", + "integrity": "sha512-CmuvUBzVJ/e3HGxhg6cYk88NGgTnBoOo7ogtfJJ0fefUWAxN/WDSUa50o+oVBxuIhO8FoEZW0j2eW7sfjs5EtA==", "dev": true, "license": "MIT", "dependencies": { - "esbuild": "^0.21.3", - "postcss": "^8.4.43", - "rollup": "^4.20.0" + "esbuild": "^0.25.0", + "fdir": "^6.5.0", + "picomatch": "^4.0.3", + "postcss": "^8.5.6", + "rollup": "^4.43.0", + "tinyglobby": "^0.2.15" }, "bin": { "vite": "bin/vite.js" }, "engines": { - "node": "^18.0.0 || >=20.0.0" + "node": "^20.19.0 || >=22.12.0" }, "funding": { "url": "https://github.com/vitejs/vite?sponsor=1" @@ -3762,19 +4170,25 @@ "fsevents": "~2.3.3" }, "peerDependencies": { - "@types/node": "^18.0.0 || >=20.0.0", - "less": "*", + "@types/node": "^20.19.0 || >=22.12.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", "lightningcss": "^1.21.0", - "sass": "*", - "sass-embedded": "*", - "stylus": "*", - "sugarss": "*", - "terser": "^5.4.0" + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" }, "peerDependenciesMeta": { "@types/node": { "optional": true }, + "jiti": { + "optional": true + }, "less": { "optional": true }, @@ -3795,439 +4209,15 @@ }, "terser": { "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true } } }, - "node_modules/vite/node_modules/@esbuild/aix-ppc64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", - "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/android-arm": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz", - "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/android-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz", - "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/android-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz", - "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/darwin-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz", - "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/darwin-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz", - "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/freebsd-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz", - "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/freebsd-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz", - "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-arm": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz", - "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz", - "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-ia32": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz", - "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-loong64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz", - "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-mips64el": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz", - "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-ppc64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz", - "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-riscv64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz", - "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-s390x": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz", - "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz", - "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/netbsd-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", - "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/openbsd-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", - "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/sunos-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", - "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/win32-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz", - "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/win32-ia32": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz", - "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/win32-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz", - "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/esbuild": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", - "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=12" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.21.5", - "@esbuild/android-arm": "0.21.5", - "@esbuild/android-arm64": "0.21.5", - "@esbuild/android-x64": "0.21.5", - "@esbuild/darwin-arm64": "0.21.5", - "@esbuild/darwin-x64": "0.21.5", - "@esbuild/freebsd-arm64": "0.21.5", - "@esbuild/freebsd-x64": "0.21.5", - "@esbuild/linux-arm": "0.21.5", - "@esbuild/linux-arm64": "0.21.5", - "@esbuild/linux-ia32": "0.21.5", - "@esbuild/linux-loong64": "0.21.5", - "@esbuild/linux-mips64el": "0.21.5", - "@esbuild/linux-ppc64": "0.21.5", - "@esbuild/linux-riscv64": "0.21.5", - "@esbuild/linux-s390x": "0.21.5", - "@esbuild/linux-x64": "0.21.5", - "@esbuild/netbsd-x64": "0.21.5", - "@esbuild/openbsd-x64": "0.21.5", - "@esbuild/sunos-x64": "0.21.5", - "@esbuild/win32-arm64": "0.21.5", - "@esbuild/win32-ia32": "0.21.5", - "@esbuild/win32-x64": "0.21.5" - } - }, "node_modules/web-streams-polyfill": { "version": "4.0.0-beta.3", "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", @@ -4397,7 +4387,9 @@ "@deepgram/sdk": "^3.4.0", "@openrouter/ai-sdk-provider": "^1.2.0", "ai": "^5.0.76", + "dotenv": "^17.2.3", "express": "^4.18.2", + "lucide-react": "^0.546.0", "openai": "^4.20.0", "tiny-invariant": "^1.3.3", "uuid": "^9.0.1", @@ -4405,6 +4397,7 @@ "zod": "^3.23.8" }, "devDependencies": { + "@playwright/test": "^1.56.1", "@types/express": "^4.17.20", "@types/node": "^20.9.0", "@types/react": "^18.2.37", @@ -4417,7 +4410,7 @@ "react-dom": "^18.2.0", "tsx": "^4.6.0", "typescript": "^5.2.2", - "vite": "^5.0.0" + "vite": "^7.1.10" } }, "packages/voice-assistant/node_modules/@types/node": { @@ -4430,19 +4423,6 @@ "undici-types": "~6.21.0" } }, - "packages/voice-assistant/node_modules/react": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", - "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "loose-envify": "^1.1.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, "packages/voice-assistant/node_modules/react-dom": { "version": "18.3.1", "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", diff --git a/packages/voice-assistant/agent-prompt.md b/packages/voice-assistant/agent-prompt.md index c85881ad3..33ca329a9 100644 --- a/packages/voice-assistant/agent-prompt.md +++ b/packages/voice-assistant/agent-prompt.md @@ -244,88 +244,6 @@ You: "Done." - Fast, efficient communication - User may be on their phone away from laptop - verbal feedback is essential -### Multi-Tool Workflows - -**CRITICAL**: When executing multiple tools in sequence, provide brief progress updates so users know the system is working. - -**Pattern for Multi-Step Operations:** - -1. **Announce the overall goal** (1 sentence max) -2. **Execute tools in sequence** -3. **Provide brief progress markers** (only for 3+ tools or slow operations) -4. **Report final result** - -**When to Provide Progress Updates:** - -- **2 tool calls**: Announce upfront, execute both, report result (no intermediate updates needed) -- **3+ tool calls**: Provide brief markers between steps -- **Fast operations** (<1s each): Skip intermediate updates, just report final result -- **Slow operations** (>2s): Provide brief marker so user knows it's working - -**Examples:** - -**2-step workflow (no intermediate updates):** - -``` -User: "What's in the web terminal?" -You: [Execute list-terminals] -You: [Execute capture-terminal on web terminal] -You: "Next.js dev server running on port 3000." -``` - -**3-step workflow (with brief markers):** - -``` -User: "Run the tests and tell me if they passed" -You: "Running tests." -[Execute send-text with npm test] -You: "Tests running..." -[Execute capture-terminal after wait] -You: "All 47 tests passed." -``` - -**5-step worktree workflow (with progress markers):** - -``` -User: "Launch Claude in a worktree called fix-auth" -You: "Creating worktree and launching Claude." - -[Execute create-terminal] -You: "Creating worktree..." - -[Execute send-text with create-worktree command] -[Parse WORKTREE_PATH from output] -You: "Worktree created." - -[Execute send-text with cd command] -[Execute send-text with claude command] -You: "Claude launched in fix-auth worktree." -``` - -**Fast multi-step (no intermediate updates):** - -``` -User: "Check all terminals" -You: [Execute list-terminals] -You: [Execute capture-terminal on each] -You: "Web terminal running dev server. Agent terminal idle. MCP terminal running on port 6767." -``` - -**Key Guidelines:** - -- **Be ultra-brief with progress markers**: "Creating...", "Done", "Running...", "Launching..." -- **Don't repeat the full plan**: User already heard the initial announcement -- **Group related operations**: Multiple fast operations can be reported together -- **Balance informativeness vs verbosity**: Voice users need to know it's working, not a narration -- **Final result is most important**: Progress markers are just reassurance - -**Why This Matters:** - -- Multi-step operations can take 5-10 seconds -- Voice users can't see terminal activity -- Without updates, users wonder if the system froze -- Brief markers provide reassurance without excess verbosity - ### Communication Style **Remember: This is VOICE interaction. Keep it natural and efficient.** @@ -339,72 +257,6 @@ You: "Web terminal running dev server. Agent terminal idle. MCP terminal running - **Assume intelligence** - user knows what they want, STT is the issue - **One clarifying question max** - if you need to ask, make it count -#### Conversational Voice Responses - -**CRITICAL: Speak naturally, not like reading a document.** - -**Rules:** - -- **NEVER use bullet points** in spoken responses - they're impossible to parse in voice -- **Use narrative, flowing language** - "First... then... when you... after that..." -- **Describe things as a journey or flow** - not as structured lists -- **Natural speech patterns** - like you're explaining to someone face-to-face -- **Group related items in sentences** - not as separate bullet points - -**Examples:** - -❌ **BAD (Robotic, Document Style):** - -``` -The homepage structure is: -**Navigation:** -- Link to Dashboard -- Link to Settings -- Link to Profile - -**Main Content:** -- Welcome message -- Recent activity -- Action buttons -``` - -✅ **GOOD (Conversational, Natural):** - -``` -"The homepage starts with navigation at the top, where you can go to the dashboard, settings, or your profile. Below that, you'll see a welcome message, your recent activity, and some action buttons to get started." -``` - -❌ **BAD (Structured List):** - -``` -Organization settings page has: -**Organization Details Card:** -- Organization logo/avatar -- Organization name -- Organization slug -- Created date - -**Pending Invitations Card:** -- Header showing count of pending invitations -- List of invitations, each showing: - - Email address - - Role - - Invited date -``` - -✅ **GOOD (Natural Flow):** - -``` -"The organization settings page shows your organization details at the top—like your logo, name, and slug, along with when it was created. Below that, there's a pending invitations section that displays how many invitations are outstanding and lists each one with the email, role, and invite date." -``` - -**Why This Matters:** - -- User is listening, not reading -- Bullet points sound choppy and robotic when spoken -- Natural speech is easier to follow and understand -- Creates better voice UX - ## Terminal Management You interact with the user's machine through **terminals** (isolated shell environments). Each terminal has its own working directory and command history. @@ -485,332 +337,30 @@ Claude Code cycles through **4 permission modes** with **shift+tab** (BTab): 3. **⏸ plan mode on** - Shows plan before executing 4. **⏵⏵ bypass permissions on** - Auto-executes ALL actions -**CRITICAL: Mode switching with repeat parameter:** +**Efficient mode switching with repeat parameter:** -**ALWAYS use the `repeat` parameter when pressing the same key multiple times.** Never make multiple separate send-keys calls. +- To plan mode from default: send-keys(terminalId, "BTab", repeat=2, return_output={lines: 50}) +- To plan mode from bypass permissions: send-keys(terminalId, "BTab", repeat=3, return_output={lines: 50}) +- To bypass from default: send-keys(terminalId, "BTab", repeat=3, return_output={lines: 50}) -- To plan mode from default: `send-keys(terminalName, "BTab", repeat=2, return_output={lines: 50})` -- To bypass from default: `send-keys(terminalName, "BTab", repeat=3, return_output={lines: 50})` -- To cycle back to default: `send-keys(terminalName, "BTab", repeat=4, return_output={lines: 50})` +### Claude Code Workflow -### Plan Mode Trigger Rules - -**CRITICAL**: When the user says **"ask Claude to plan X"** or **"have Claude plan X"**, you MUST: - -1. Switch Claude to plan mode FIRST (if not already in plan mode) -2. Then submit the request - -**Example:** - -``` -User: "Ask Claude to plan adding dark mode" - -Step 1: Check current mode from terminal output -Step 2: If not in plan mode, switch to it: - send-keys(terminalName, "BTab", repeat=2, return_output={lines: 50}) -Step 3: Submit the request: - send-text(terminalName, "add dark mode", pressEnter=true) -``` - -**When to use plan mode:** - -- User explicitly asks for planning -- Complex, multi-step tasks where seeing the plan first is valuable -- When user wants to review approach before execution - -### Understanding Claude's Response Types - -**CRITICAL**: Always tell the user what type of prompt Claude is currently showing. - -**Response Types:** - -1. **Working State** - Claude is executing: - - ``` - ✻ Catapulting… (esc to interrupt) - ``` - - Other verbs: "Thinking", "Pondering", "Analyzing", etc. - Key indicator: "(esc to interrupt)" means Claude is busy - - - Tell user: "Claude is working..." - -2. **Plan Approval Menu** (CRITICAL): - - ``` - Would you like to proceed? - - ❯ 1. Yes, and bypass permissions - 2. Yes, and manually approve edits - 3. No, keep planning - ``` - - **CRITICAL RULES:** - - - **NEVER approve or reject without explicit user instruction** - - **ALWAYS capture and read the plan to the user first** - - **WAIT for user to say "approve", "reject", "yes", or "no"** - - **Understanding the Options:** - - - **Option 1**: Approve and execute everything automatically - - **Option 2**: Approve but ask for confirmation on each edit - - **Option 3**: REJECT - Go back to planning (this is the "NO" option) - - **Correct Workflow:** - - ``` - Step 1: Capture terminal showing the plan - Step 2: Read plan summary to user - Step 3: Tell user: "Claude has a plan. Approve or reject?" - Step 4: WAIT for user response - Step 5: If user says "approve" or "yes": Press 1 or 2 based on their preference - Step 6: If user says "reject" or "no": Press 3 - ``` - - **Examples:** - - - User: "approve" → Press 1 (unless they want manual approval, then ask) - - User: "yes" → Press 1 - - User: "reject" → Press 3 (No, keep planning) - - User: "no" → Press 3 (No, keep planning) - - User: "approve but let me review edits" → Press 2 - - **NEVER assume approval or rejection. Always ask the user.** - -3. **Other Menu Prompts**: - - ``` - [1] Option A [2] Option B [3] Option C - ``` - - Key indicators: Numbers or options in brackets - - - Tell user: "Claude is asking a question with options..." - -4. **Question Menu**: - - ``` - Which approach? - 1. Use OAuth - 2. Use JWT - 3. Other - ``` - - - Tell user: "Claude is asking a question with options..." - -5. **Regular Text Response**: - - ``` - I've completed the task. The changes are in... - ``` - - - Tell user: "Claude responded: [summary]" - -6. **Input Prompt** (showing `-- INSERT --`): - ``` - > [cursor here] - -- INSERT -- - ``` - - Tell user: "Claude is ready for input" - -**Always capture terminal output and identify which type before reporting to user.** - -### Claude Code Commands - -**Available Commands:** - -- **`/clear`** - Clear context and start a new task - - - Use when: Starting a different task in the same Claude instance - - Common operation - user will request this frequently - - Claude's context is cleared but the session stays open - -- **`/exit`** - Exit Claude Code entirely - - Use when: User explicitly asks to "close Claude" or "exit Claude" - - RARE operation - only do this when explicitly requested - - Ends the Claude session completely - -**CRITICAL Distinction:** - -- **Clearing** (`/clear`) = Start fresh on a new task, keep Claude running -- **Closing** (`/exit`) = Shut down Claude entirely - -**Examples:** - -``` -send-text(terminalName, "/clear", pressEnter=true, return_output={lines: 20}) -send-text(terminalName, "/exit", pressEnter=true, return_output={lines: 20}) -``` - -### Turn-Taking and Interruption - -**When Claude is Working:** - -Claude shows a working indicator like: - -``` -✻ Catapulting… (esc to interrupt) -✻ Pondering… (esc to interrupt) -✻ Working… (esc to interrupt) -``` - -Key indicator: **"(esc to interrupt)"** means Claude is busy executing. - -**Two Options:** - -1. **Interrupt** - Press ESC to stop Claude immediately - - - Stops current task execution - - Use for: Changing direction, stopping unwanted action - -2. **Steer** - Submit a message to be processed after current task - - Message gets queued, Claude reads it after finishing current step - - Use for: Live guidance, additional requirements - -**CRITICAL: Ask for Clarification Unless Explicit** - -**Explicit Interrupt:** - -- "interrupt and tell it to do X instead" -- "stop Claude and..." -- "cancel that and..." - → You know to interrupt: `send-keys(terminalName, "Escape", return_output={lines: 50})` - -**Explicit Steering:** - -- "make sure it also deals with X" -- "tell it to also handle Y" -- "add requirement Z" - → You know to steer: Wait for working to finish or submit message directly - -**Ambiguous:** - -- "tell Claude to do X" (while Claude is working) -- "change this to Y" - → ASK: "Claude is working. Do you want to interrupt and change direction, or add this as guidance after the current step?" - -**Workflow for Steering:** - -``` -User: "Make sure Claude also adds tests" -Context: Claude is working (saw "✻ Working… (esc to interrupt)") - -You: [Infer steering intent] -You: "Adding steering guidance." -Step 1: Ensure INSERT mode (if needed) -Step 2: send-text(terminalName, "also add tests for this", pressEnter=true) -You: "Guidance queued. Claude will see this after the current step." -``` - -**Workflow for Interruption:** - -``` -User: "Interrupt and tell it to use OAuth instead" -Context: Claude is working - -You: "Interrupting Claude." -Step 1: send-keys(terminalName, "Escape", return_output={lines: 50}) -Step 2: Wait and check terminal shows input prompt -Step 3: send-text(terminalName, "use OAuth instead", pressEnter=true) -You: "Interrupted. Told Claude to use OAuth instead." -``` - -### Starting Claude Code - -**Workflow:** +**Starting Claude Code:** 1. create-terminal or use existing terminal -2. send-text(terminalName, "claude --dangerously-skip-permissions", pressEnter=true, return_output={lines: 50}) +2. send-text(terminalId, "claude --dangerously-skip-permissions", pressEnter=true, return_output={lines: 50}) 3. Wait for Claude Code interface to appear -### Asking Claude Code a Question - -**Workflow:** +**Asking Claude Code a question:** 1. Check for "-- INSERT --" in terminal output -2. If not in insert mode: send-keys(terminalName, "i", return_output={lines: 20}) -3. send-text(terminalName, "your question", pressEnter=true, return_output={lines: 50, wait: 1000}) -4. Capture and identify response type, then report to user +2. If not in insert mode: send-keys(terminalId, "i", return_output={lines: 20}) +3. send-text(terminalId, "your question", pressEnter=true, return_output={lines: 50, wait: 1000}) -### Working Through Claude Code +**Closing Claude Code:** -**CRITICAL: When Claude Code is running in a terminal, delegate ALL tasks to Claude via natural language.** - -**The Rule:** - -If Claude Code is active in a terminal, that terminal is in **"Claude mode"**. All development tasks should be delegated to Claude by typing natural language instructions, NOT by running raw commands. - -**Examples:** - -✅ **CORRECT - Delegate to Claude:** - -``` -User: "commit the changes" -Context: Claude Code is running in terminal - -You: "Asking Claude to commit." -Step 1: Ensure INSERT mode -Step 2: send-text(terminalName, "commit the changes", pressEnter=true) -You: "Asked Claude to commit the changes." -``` - -``` -User: "run the tests" -Context: Claude Code is running - -You: "Asking Claude to run tests." -Step 1: send-text(terminalName, "run the tests", pressEnter=true) -``` - -``` -User: "add dark mode toggle" -Context: Claude Code is running - -You: "Asking Claude to add dark mode." -Step 1: send-text(terminalName, "add dark mode toggle", pressEnter=true) -``` - -``` -User: "fix the bug in auth.ts line 45" -Context: Claude Code is running - -You: "Asking Claude to fix the bug." -Step 1: send-text(terminalName, "fix the bug in auth.ts line 45", pressEnter=true) -``` - -❌ **WRONG - Running raw commands:** - -``` -User: "commit the changes" -Context: Claude Code is running - -You: send-text(terminalName, "git commit -m 'message'", pressEnter=true) # WRONG! -``` - -**Exception - Raw Commands:** - -Only run raw commands when: - -1. **User explicitly says**: "run X in the terminal" or "execute X directly" -2. **Claude is NOT running** in the terminal -3. **Quick info gathering** where Claude isn't needed (git status, ls, etc.) - -**Common Delegated Tasks:** - -- Committing: "commit the changes" or "commit with message X" -- Testing: "run the tests" or "run tests for X" -- Building: "build the project" -- Code changes: "add feature X", "fix bug in Y", "refactor Z" -- Git operations: "create a PR", "push the changes" -- Any coding task - -**Why This Matters:** - -- Claude Code understands the codebase context -- Claude can handle complex multi-step operations -- Natural language is more powerful than raw commands -- Maintains consistent workflow through Claude +- Method 1: send-text(terminalId, "/exit", pressEnter=true, return_output={lines: 20}) +- Method 2: send-keys(terminalId, "C-c", repeat=2, return_output={lines: 20}) ### Launching Claude Code - Workflow Patterns @@ -1042,7 +592,8 @@ send-keys(terminalId="@123", keys="C-c", return_output={lines: 20}) # Ctrl+C to ### Faro - Autonomous Competitive Intelligence Tool -- Location: ~/dev/faro/main +- Bare repo: ~/dev/faro +- Main checkout: ~/dev/faro/main ### Blank.page - A minimal text editor in your browser diff --git a/packages/voice-assistant/package.json b/packages/voice-assistant/package.json index 8b77ce5b1..1bfe0249b 100644 --- a/packages/voice-assistant/package.json +++ b/packages/voice-assistant/package.json @@ -9,7 +9,10 @@ "build:server": "tsc -p tsconfig.server.json", "build": "npm run build:ui && npm run build:server", "start": "NODE_ENV=production node dist/server/index.js", - "typecheck": "tsc --noEmit && tsc -p tsconfig.server.json --noEmit" + "typecheck": "tsc --noEmit && tsc -p tsconfig.server.json --noEmit", + "test:e2e": "playwright test", + "test:e2e:ui": "playwright test --ui", + "test:e2e:mobile": "playwright test --project='Mobile Chrome'" }, "dependencies": { "@ai-sdk/openai": "^2.0.52", @@ -18,6 +21,7 @@ "ai": "^5.0.76", "dotenv": "^17.2.3", "express": "^4.18.2", + "lucide-react": "^0.546.0", "openai": "^4.20.0", "tiny-invariant": "^1.3.3", "uuid": "^9.0.1", @@ -25,6 +29,7 @@ "zod": "^3.23.8" }, "devDependencies": { + "@playwright/test": "^1.56.1", "@types/express": "^4.17.20", "@types/node": "^20.9.0", "@types/react": "^18.2.37", diff --git a/packages/voice-assistant/playwright.config.ts b/packages/voice-assistant/playwright.config.ts new file mode 100644 index 000000000..5da77e844 --- /dev/null +++ b/packages/voice-assistant/playwright.config.ts @@ -0,0 +1,30 @@ +import { defineConfig, devices } from '@playwright/test'; + +export default defineConfig({ + testDir: './tests/e2e', + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + workers: process.env.CI ? 1 : undefined, + reporter: 'html', + use: { + baseURL: 'http://localhost:3000', + trace: 'on-first-retry', + }, + projects: [ + { + name: 'Mobile Chrome', + use: { ...devices['iPhone 12'] }, + }, + { + name: 'Desktop Chrome', + use: { ...devices['Desktop Chrome'] }, + }, + ], + webServer: { + command: 'npm run dev', + url: 'http://localhost:3000', + reuseExistingServer: !process.env.CI, + timeout: 120000, + }, +}); diff --git a/packages/voice-assistant/src/server/agent/llm-openai.ts b/packages/voice-assistant/src/server/agent/llm-openai.ts index 5591dc49b..12e0a5c31 100644 --- a/packages/voice-assistant/src/server/agent/llm-openai.ts +++ b/packages/voice-assistant/src/server/agent/llm-openai.ts @@ -6,6 +6,7 @@ import { createTerminal, captureTerminal, sendText, + sendKeys, renameTerminal, killTerminal, } from "../daemon/terminal-manager.js"; @@ -17,7 +18,8 @@ import invariant from "tiny-invariant"; */ export const terminalTools = { list_terminals: tool({ - description: "List all available terminals in the voice-dev session", + description: + "List all terminals (isolated shell environments). Returns terminal name, active status, current working directory, currently running command, and the last 5 lines of output for each terminal.", inputSchema: z.object({}), execute: async () => { const terminals = await listTerminals(); @@ -27,14 +29,24 @@ export const terminalTools = { create_terminal: tool({ description: - "Create a new terminal with specified name and working directory", + "Create a new terminal (isolated shell environment) at a specific working directory. Optionally execute an initial command after creation. Terminal names must be unique. Always specify workingDirectory based on context - use project paths when working on projects, or the same directory as current terminal when user says 'another terminal here'. Defaults to ~ only if no context.", inputSchema: z.object({ - name: z.string().describe("Unique name for the terminal"), - workingDirectory: z.string().describe("Working directory path"), + name: z + .string() + .describe( + "Unique name for the terminal. Should be descriptive of what the terminal is used for (e.g., 'web-dev', 'api-server', 'tests')." + ), + workingDirectory: z + .string() + .describe( + "Absolute path to the working directory for this terminal. Can use ~ for home directory. Required parameter - set contextually based on what the user is working on. Use project paths when working on projects. Defaults to home directory (~) only if no context." + ), initialCommand: z .string() .optional() - .describe("Optional command to run on creation"), + .describe( + "Optional command to execute after creating the terminal (e.g., 'npm run dev', 'python -m venv venv'). The command runs after changing to the working directory." + ), }), execute: async ({ name, workingDirectory, initialCommand }) => { const terminal = await createTerminal({ @@ -47,9 +59,10 @@ export const terminalTools = { }), capture_terminal: tool({ - description: "Capture and return the output from a terminal", + description: + "Capture and return the output from a terminal. Returns the last N lines of terminal content. Useful for checking command results, monitoring running processes, or debugging issues.", inputSchema: z.object({ - terminalId: z.string().describe("Terminal ID (e.g., @123)"), + terminalName: z.string().describe("Name of the terminal"), lines: z .number() .optional() @@ -57,34 +70,51 @@ export const terminalTools = { wait: z .number() .optional() - .describe("Milliseconds to wait before capture"), + .describe( + "Milliseconds to wait before capturing output. Useful for slow commands." + ), }), - execute: async ({ terminalId, lines, wait }) => { - const output = await captureTerminal(terminalId, lines, wait); + execute: async ({ terminalName, lines, wait }) => { + const output = await captureTerminal(terminalName, lines, wait); return { output }; }, }), send_text: tool({ - description: "Send text or commands to a terminal", + description: + "Type text into a terminal. This is the PRIMARY way to execute shell commands with bash operators (&&, ||, |, ;, etc.) - set pressEnter=true to run the command. Also use for interactive applications, REPLs, forms, and text entry. For special keys or control sequences, use send_keys instead.", inputSchema: z.object({ - terminalId: z.string().describe("Terminal ID (e.g., @123)"), - text: z.string().describe("Text to send to the terminal"), + terminalName: z.string().describe("Name of the terminal"), + text: z + .string() + .describe( + "Text to type into the terminal. For shell commands, can use any bash operators: && (chain), || (or), | (pipe), ; (sequential), etc." + ), pressEnter: z .boolean() .optional() - .describe("Whether to press Enter after text"), + .describe( + "Press Enter after typing the text (default: false). Set to true to execute shell commands or submit text input." + ), return_output: z .object({ - lines: z.number().optional(), - wait: z.number().optional(), + lines: z + .number() + .optional() + .describe("Number of lines to capture (default: 200)"), + wait: z + .number() + .optional() + .describe("Milliseconds to wait before capturing output"), }) .optional() - .describe("Capture output after sending"), + .describe( + "Capture terminal output after sending text. Specify 'wait' for slow commands." + ), }), - execute: async ({ terminalId, text, pressEnter, return_output }) => { + execute: async ({ terminalName, text, pressEnter, return_output }) => { const output = await sendText( - terminalId, + terminalName, text, pressEnter, return_output @@ -93,25 +123,72 @@ export const terminalTools = { }, }), - rename_terminal: tool({ - description: "Rename an existing terminal", + send_keys: tool({ + description: + "Send special keys or key combinations to a terminal. Use for TUI navigation and control sequences. Examples: 'Up', 'Down', 'Enter', 'Escape', 'C-c' (Ctrl+C), 'M-x' (Alt+X). For typing regular text, use send_text instead. Supports repeating key presses and optionally capturing output after sending keys.", inputSchema: z.object({ - terminalId: z.string().describe("Terminal ID to rename"), - newName: z.string().describe("New unique name for the terminal"), + terminalName: z.string().describe("Name of the terminal"), + keys: z + .string() + .describe( + "Special key name or key combination: 'Up', 'Down', 'Left', 'Right', 'Enter', 'Escape', 'Tab', 'Space', 'C-c', 'M-x', etc." + ), + repeat: z + .number() + .min(1) + .optional() + .describe("Number of times to repeat the key press (default: 1)"), + return_output: z + .object({ + lines: z + .number() + .optional() + .describe("Number of lines to capture (default: 200)"), + wait: z + .number() + .optional() + .describe("Milliseconds to wait before capturing output"), + }) + .optional() + .describe( + "Capture terminal output after sending keys. Specify 'wait' for slow commands." + ), }), - execute: async ({ terminalId, newName }) => { - await renameTerminal(terminalId, newName); + execute: async ({ terminalName, keys, repeat, return_output }) => { + const output = await sendKeys(terminalName, keys, repeat, return_output); + return { output }; + }, + }), + + rename_terminal: tool({ + description: + "Rename a terminal to a more descriptive name. The new name must be unique among all terminals.", + inputSchema: z.object({ + terminalName: z.string().describe("Current name of the terminal"), + newName: z + .string() + .describe( + "New unique name for the terminal. Should be descriptive of the terminal's purpose." + ), + }), + execute: async ({ terminalName, newName }) => { + await renameTerminal(terminalName, newName); return { success: true }; }, }), kill_terminal: tool({ - description: "Close and destroy a terminal", + description: + "Close and destroy a terminal. This will terminate any running processes in the terminal. Use with caution.", inputSchema: z.object({ - terminalId: z.string().describe("Terminal ID to kill"), + terminalName: z + .string() + .describe( + "Name of the terminal to kill. Get this from list_terminals." + ), }), - execute: async ({ terminalId }) => { - await killTerminal(terminalId); + execute: async ({ terminalName }) => { + await killTerminal(terminalName); return { success: true }; }, }), @@ -129,12 +206,14 @@ export interface Message { * Streaming LLM parameters */ export interface StreamLLMParams { + systemPrompt: string; messages: Message[]; - onChunk?: (chunk: string) => void; + abortSignal?: AbortSignal; + onChunk?: (chunk: string) => void | Promise; onTextSegment?: (segment: string) => void; - onToolCall?: (toolName: string, args: any) => void; - onToolResult?: (toolName: string, result: any) => void; - onFinish?: (fullText: string) => void; + onToolCall?: (toolCallId: string, toolName: string, args: any) => Promise; + onToolResult?: (toolCallId: string, toolName: string, result: any) => void; + onFinish?: (fullText: string) => void | Promise; } /** @@ -149,12 +228,32 @@ export async function streamLLM(params: StreamLLMParams): Promise { const result = await streamText({ model: openrouter("anthropic/claude-haiku-4.5"), + system: params.systemPrompt, messages: params.messages, tools: terminalTools, - onChunk: (chuk) => { - // if (params.onChunk) { - // params.onChunk(chunk); - // } + abortSignal: params.abortSignal, + onChunk: async ({ chunk }) => { + // console.log("onChunk", chunk); + if (chunk.type === "text-delta") { + // Accumulate text in buffer + textBuffer += chunk.text; + fullText += chunk.text; + + params.onChunk?.(chunk.text); + } else if (chunk.type === "tool-call") { + // Flush accumulated text as a segment before tool call + flushTextBuffer(); + + // Emit tool call event + if (params.onToolCall) { + await params.onToolCall(chunk.toolCallId, chunk.toolName, chunk.input); + } + } else if (chunk.type === "tool-result") { + // Emit tool result event + if (params.onToolResult) { + params.onToolResult(chunk.toolCallId, chunk.toolName, chunk.output); + } + } }, stopWhen: stepCountIs(10), }); @@ -169,38 +268,15 @@ export async function streamLLM(params: StreamLLMParams): Promise { textBuffer = ""; } - // Stream all events from fullStream - for await (const part of result.fullStream) { - if (part.type === "text-delta") { - // Accumulate text in buffer - textBuffer += part.text; - fullText += part.text; - - // Stream individual deltas to caller - if (params.onChunk) { - params.onChunk(part.text); - } - } else if (part.type === "tool-call") { - // Flush accumulated text as a segment before tool call - flushTextBuffer(); - - // Emit tool call event - if (params.onToolCall) { - params.onToolCall(part.toolName, part.input); - } - } else if (part.type === "tool-result") { - // Emit tool result event - if (params.onToolResult) { - params.onToolResult(part.toolName, part.output); - } - } + for await (const _part of result.fullStream) { + // console.log("part", _part); } // Flush any remaining text at the end flushTextBuffer(); if (params.onFinish) { - params.onFinish(fullText); + await params.onFinish(fullText); } return fullText; diff --git a/packages/voice-assistant/src/server/agent/orchestrator.ts b/packages/voice-assistant/src/server/agent/orchestrator.ts index 09991ffd6..f0e2f4c3e 100644 --- a/packages/voice-assistant/src/server/agent/orchestrator.ts +++ b/packages/voice-assistant/src/server/agent/orchestrator.ts @@ -1,6 +1,7 @@ import { v4 as uuidv4 } from "uuid"; import { getSystemPrompt } from "./system-prompt.js"; import { streamLLM, type Message } from "./llm-openai.js"; +import { generateTTSAndWaitForPlayback } from "./tts-manager.js"; import type { VoiceAssistantWebSocketServer } from "../websocket-server.js"; interface ConversationContext { @@ -17,13 +18,13 @@ interface ConversationContext { const conversations = new Map(); /** - * Create a new conversation with system prompt + * Create a new conversation */ export function createConversation(): string { const id = uuidv4(); conversations.set(id, { id, - messages: [{ role: "system", content: getSystemPrompt() }], + messages: [], createdAt: new Date(), lastActivity: new Date(), }); @@ -37,6 +38,13 @@ export function getConversation(id: string): ConversationContext | null { return conversations.get(id) || null; } +/** + * Delete a conversation by ID + */ +export function deleteConversation(id: string): void { + conversations.delete(id); +} + /** * Process user message through the LLM orchestrator * Handles streaming, tool calls, and WebSocket broadcasting @@ -46,6 +54,7 @@ export async function processUserMessage(params: { message: string; wsServer?: VoiceAssistantWebSocketServer; enableTTS?: boolean; + abortSignal?: AbortSignal; }): Promise { const conversation = conversations.get(params.conversationId); if (!conversation) { @@ -65,65 +74,72 @@ export async function processUserMessage(params: { let assistantResponse = ""; try { + // Track pending TTS playback promise + let pendingTTS: Promise | null = null; + // Stream LLM response with tool execution assistantResponse = await streamLLM({ + systemPrompt: getSystemPrompt(), messages: conversation.messages, - onChunk: (chunk) => { - // Broadcast streaming chunks to WebSocket - if (params.wsServer) { - params.wsServer.broadcast({ - type: "assistant_chunk", - payload: { chunk }, - }); - } - }, + abortSignal: params.abortSignal, onTextSegment: (segment) => { - // Broadcast complete text segments (for future TTS integration) - if (params.wsServer) { - params.wsServer.broadcastActivityLog({ - id: uuidv4(), - timestamp: new Date(), - type: "system", - content: `Text segment: ${segment}`, - metadata: { segment, readyForTTS: true }, - }); + // Create TTS promise (don't await it yet) + if (params.wsServer && params.enableTTS) { + pendingTTS = generateTTSAndWaitForPlayback(segment, params.wsServer); } - }, - onToolCall: (toolName, args) => { - // Broadcast tool call to WebSocket - if (params.wsServer) { - params.wsServer.broadcastActivityLog({ - id: uuidv4(), - timestamp: new Date(), - type: "tool_call", - content: `Calling ${toolName}`, - metadata: { toolName, arguments: args }, - }); - } - }, - onToolResult: (toolName, result) => { - // Broadcast tool result to WebSocket - if (params.wsServer) { - params.wsServer.broadcastActivityLog({ - id: uuidv4(), - timestamp: new Date(), - type: "tool_result", - content: `Tool ${toolName} completed`, - metadata: { toolName, result }, - }); - } - }, - onFinish: (fullText) => { - // Broadcast complete assistant response + + // Broadcast complete text segments if (params.wsServer) { params.wsServer.broadcastActivityLog({ id: uuidv4(), timestamp: new Date(), type: "assistant", - content: fullText, + content: segment, }); } }, + onChunk: async (chunk) => { + params.wsServer?.broadcast({ + type: "assistant_chunk", + payload: { chunk }, + }); + }, + onToolCall: async (toolCallId, toolName, args) => { + if (pendingTTS) { + console.log("Waiting for pending TTS to finish to execute", toolName); + await pendingTTS; + pendingTTS = null; + } + // Broadcast tool call to WebSocket + if (params.wsServer) { + params.wsServer.broadcastActivityLog({ + id: toolCallId, + timestamp: new Date(), + type: "tool_call", + content: `Calling ${toolName}`, + metadata: { toolCallId, toolName, arguments: args }, + }); + } + }, + onToolResult: (toolCallId, toolName, result) => { + // Broadcast tool result to WebSocket + if (params.wsServer) { + params.wsServer.broadcastActivityLog({ + id: toolCallId, + timestamp: new Date(), + type: "tool_result", + content: `Tool ${toolName} completed`, + metadata: { toolCallId, toolName, result }, + }); + } + }, + onFinish: async () => { + // Wait for any final pending TTS to complete + if (pendingTTS) { + await pendingTTS; + pendingTTS = null; + } + }, }); // Add assistant response to context @@ -138,7 +154,9 @@ export async function processUserMessage(params: { id: uuidv4(), timestamp: new Date(), type: "error", - content: `Error: ${error instanceof Error ? error.message : String(error)}`, + content: `Error: ${ + error instanceof Error ? error.message : String(error) + }`, }); } throw error; diff --git a/packages/voice-assistant/src/server/agent/tts-manager.ts b/packages/voice-assistant/src/server/agent/tts-manager.ts new file mode 100644 index 000000000..cf0293c37 --- /dev/null +++ b/packages/voice-assistant/src/server/agent/tts-manager.ts @@ -0,0 +1,88 @@ +import { v4 as uuidv4 } from "uuid"; +import { synthesizeSpeech } from "./tts-openai.js"; +import type { VoiceAssistantWebSocketServer } from "../websocket-server.js"; + +interface PendingPlayback { + resolve: () => void; + reject: (error: Error) => void; + timeout: NodeJS.Timeout; +} + +/** + * Store pending playback confirmations + * Maps audio ID -> promise resolve/reject handlers + */ +const pendingPlaybacks = new Map(); + +/** + * Timeout for waiting on client playback confirmation (30 seconds) + */ +const PLAYBACK_TIMEOUT_MS = 30000; + +/** + * Generate TTS audio, broadcast to clients, and wait for playback confirmation + * Returns a Promise that resolves when the client confirms playback completed + */ +export async function generateTTSAndWaitForPlayback( + text: string, + wsServer: VoiceAssistantWebSocketServer +): Promise { + // Generate TTS audio + const { audio, format } = await synthesizeSpeech(text); + + // Create unique ID for this audio segment + const audioId = uuidv4(); + + // Create promise that will be resolved when client confirms playback + const playbackPromise = new Promise((resolve, reject) => { + // Set timeout to prevent hanging forever + const timeout = setTimeout(() => { + pendingPlaybacks.delete(audioId); + reject(new Error(`Audio playback timeout for ${audioId}`)); + }, PLAYBACK_TIMEOUT_MS); + + // Store handlers + pendingPlaybacks.set(audioId, { resolve, reject, timeout }); + }); + + // Broadcast audio to clients + wsServer.broadcast({ + type: "audio_output", + payload: { + id: audioId, + audio: audio.toString("base64"), + format, + }, + }); + + console.log( + `[TTS-Manager] ${new Date().toISOString()} Sent audio ${audioId}, waiting for playback...` + ); + + // Wait for playback confirmation + await playbackPromise; + + console.log( + `[TTS-Manager] ${new Date().toISOString()} Audio ${audioId} playback confirmed` + ); +} + +/** + * Called when client confirms audio playback completed + * Resolves the corresponding promise + */ +export function confirmAudioPlayed(audioId: string): void { + const pending = pendingPlaybacks.get(audioId); + + if (!pending) { + console.warn( + `[TTS-Manager] Received confirmation for unknown audio ID: ${audioId}` + ); + return; + } + + // Clear timeout and resolve promise + clearTimeout(pending.timeout); + pending.resolve(); + pendingPlaybacks.delete(audioId); +} diff --git a/packages/voice-assistant/src/server/agent/tts-openai.ts b/packages/voice-assistant/src/server/agent/tts-openai.ts index 91f0b2e1d..f97890465 100644 --- a/packages/voice-assistant/src/server/agent/tts-openai.ts +++ b/packages/voice-assistant/src/server/agent/tts-openai.ts @@ -54,6 +54,7 @@ export async function synthesizeSpeech(text: string): Promise { model: config.model!, voice: config.voice!, input: text, + // speed: 1.2, response_format: config.responseFormat as "mp3" | "opus" | "aac" | "flac", }); diff --git a/packages/voice-assistant/src/server/daemon/terminal-manager.ts b/packages/voice-assistant/src/server/daemon/terminal-manager.ts index 4b840382b..cb33a84ac 100644 --- a/packages/voice-assistant/src/server/daemon/terminal-manager.ts +++ b/packages/voice-assistant/src/server/daemon/terminal-manager.ts @@ -3,8 +3,11 @@ import { createSession, listWindows, createWindow, + listPanes, + findWindowByName, capturePaneContent, sendText as tmuxSendText, + sendKeys as tmuxSendKeys, renameWindow, killWindow, isWindowNameUnique, @@ -15,13 +18,14 @@ import { const DEFAULT_SESSION = "voice-dev"; // Terminal model: session → windows (single pane per window) -// Terminal ID = window ID (format: @123) +// Terminals are identified by their unique names, not IDs export interface TerminalInfo { - id: string; // window ID name: string; + active: boolean; workingDirectory: string; currentCommand: string; + lastLines?: string; } export interface CreateTerminalParams { @@ -31,7 +35,6 @@ export interface CreateTerminalParams { } export interface Terminal extends TerminalInfo { - active: boolean; sessionId: string; } @@ -49,7 +52,7 @@ export async function initializeDefaultSession(): Promise { /** * List all terminals in the voice-dev session - * Returns terminal info including ID, name, working directory, and current command + * Returns terminal info including name, active status, working directory, and current command */ export async function listTerminals(): Promise { const session = await findSessionByName(DEFAULT_SESSION); @@ -69,20 +72,23 @@ export async function listTerminals(): Promise { try { const workingDirectory = await getCurrentWorkingDirectory(paneId); const currentCommand = await getCurrentCommand(paneId); + const lastLines = await capturePaneContent(paneId, 5, false); terminals.push({ - id: window.id, name: window.name, + active: window.active, workingDirectory, currentCommand, + lastLines, }); } catch (error) { // If we can't get pane info, still include the terminal with empty values terminals.push({ - id: window.id, name: window.name, + active: window.active, workingDirectory: "", currentCommand: "", + lastLines: "", }); } } @@ -126,49 +132,86 @@ export async function createTerminal(params: CreateTerminalParams): Promise { - // Terminal ID is window ID, get the first pane - const paneId = `${terminalId}.0`; + const session = await findSessionByName(DEFAULT_SESSION); + if (!session) { + throw new Error(`Session '${DEFAULT_SESSION}' not found.`); + } + + // Resolve terminal name to window + const window = await findWindowByName(session.id, terminalName); + if (!window) { + const windows = await listWindows(session.id); + const availableNames = windows.map((w) => w.name).join(", "); + throw new Error( + `Terminal '${terminalName}' not found. Available terminals: ${availableNames}` + ); + } + + // Get the first pane + const panes = await listPanes(window.id); + const pane = panes[0]; + if (!pane) { + throw new Error(`No pane found for terminal ${terminalName}`); + } // Optional wait before capture if (wait) { await new Promise((resolve) => setTimeout(resolve, wait)); } - return capturePaneContent(paneId, lines, false); + return capturePaneContent(pane.id, lines, false); } /** - * Send text to a terminal, optionally press Enter, optionally return output + * Send text to a terminal by name, optionally press Enter, optionally return output */ export async function sendText( - terminalId: string, + terminalName: string, text: string, pressEnter: boolean = false, - return_output?: { lines?: number; wait?: number } + return_output?: { lines?: number; waitForSettled?: boolean; maxWait?: number } ): Promise { - // Terminal ID is window ID, get the first pane - const paneId = `${terminalId}.0`; + const session = await findSessionByName(DEFAULT_SESSION); + if (!session) { + throw new Error(`Session '${DEFAULT_SESSION}' not found.`); + } + + // Resolve terminal name to window + const window = await findWindowByName(session.id, terminalName); + if (!window) { + const windows = await listWindows(session.id); + const availableNames = windows.map((w) => w.name).join(", "); + throw new Error( + `Terminal '${terminalName}' not found. Available terminals: ${availableNames}` + ); + } + + // Get the first pane + const panes = await listPanes(window.id); + const pane = panes[0]; + if (!pane) { + throw new Error(`No pane found for terminal ${terminalName}`); + } return tmuxSendText({ - paneId, + paneId: pane.id, text, pressEnter, return_output, @@ -176,11 +219,51 @@ export async function sendText( } /** - * Rename a terminal + * Send special keys or key combinations to a terminal by name + * Useful for TUI navigation, control sequences, and interactive applications + */ +export async function sendKeys( + terminalName: string, + keys: string, + repeat: number = 1, + return_output?: { lines?: number; waitForSettled?: boolean; maxWait?: number } +): Promise { + const session = await findSessionByName(DEFAULT_SESSION); + if (!session) { + throw new Error(`Session '${DEFAULT_SESSION}' not found.`); + } + + // Resolve terminal name to window + const window = await findWindowByName(session.id, terminalName); + if (!window) { + const windows = await listWindows(session.id); + const availableNames = windows.map((w) => w.name).join(", "); + throw new Error( + `Terminal '${terminalName}' not found. Available terminals: ${availableNames}` + ); + } + + // Get the first pane + const panes = await listPanes(window.id); + const pane = panes[0]; + if (!pane) { + throw new Error(`No pane found for terminal ${terminalName}`); + } + + return tmuxSendKeys({ + paneId: pane.id, + keys, + repeat, + return_output, + }); +} + +/** + * Rename a terminal by name * Validates that the new name is unique */ export async function renameTerminal( - terminalId: string, + terminalName: string, newName: string ): Promise { const session = await findSessionByName(DEFAULT_SESSION); @@ -189,13 +272,29 @@ export async function renameTerminal( throw new Error(`Session '${DEFAULT_SESSION}' not found.`); } - // renameWindow handles uniqueness validation internally - await renameWindow(session.id, terminalId, newName); + // renameWindow handles uniqueness validation and name resolution internally + await renameWindow(session.id, terminalName, newName); } /** - * Kill (close/destroy) a terminal + * Kill (close/destroy) a terminal by name */ -export async function killTerminal(terminalId: string): Promise { - await killWindow(terminalId); +export async function killTerminal(terminalName: string): Promise { + const session = await findSessionByName(DEFAULT_SESSION); + + if (!session) { + throw new Error(`Session '${DEFAULT_SESSION}' not found.`); + } + + // Resolve terminal name to window + const window = await findWindowByName(session.id, terminalName); + if (!window) { + const windows = await listWindows(session.id); + const availableNames = windows.map((w) => w.name).join(", "); + throw new Error( + `Terminal '${terminalName}' not found. Available terminals: ${availableNames}` + ); + } + + await killWindow(window.id); } diff --git a/packages/voice-assistant/src/server/daemon/tmux.ts b/packages/voice-assistant/src/server/daemon/tmux.ts index aa5f797d5..9b51e7675 100644 --- a/packages/voice-assistant/src/server/daemon/tmux.ts +++ b/packages/voice-assistant/src/server/daemon/tmux.ts @@ -191,9 +191,18 @@ export async function capturePaneContent( includeColors: boolean = false ): Promise { const colorFlag = includeColors ? "-e" : ""; - return executeTmux( - `capture-pane -p ${colorFlag} -t '${paneId}' -S -${lines} -E -` + // Capture a large range to ensure we have enough content + const captureLines = Math.max(lines, 1000); + const output = await executeTmux( + `capture-pane -p ${colorFlag} -t '${paneId}' -S -${captureLines} -E -` ); + + // Trim trailing whitespace, split by lines, take last N lines, rejoin + const trimmed = output.trimEnd(); + const allLines = trimmed.split('\n'); + const lastLines = allLines.slice(-lines); + + return lastLines.join('\n'); } /** @@ -800,7 +809,7 @@ export async function sendKeys({ paneId: string; keys: string; repeat?: number; - return_output?: { lines?: number; wait?: number }; + return_output?: { lines?: number; waitForSettled?: boolean; maxWait?: number }; }): Promise { // Repeat the key press the specified number of times for (let i = 0; i < repeat; i++) { @@ -810,11 +819,49 @@ export async function sendKeys({ // If return_output is requested, wait and capture pane content if (return_output) { - await new Promise((resolve) => - setTimeout(resolve, return_output.wait ?? 500) - ); const lines = return_output.lines || 200; - return capturePaneContent(paneId, lines, false); + const waitForSettled = return_output.waitForSettled ?? true; + const maxWait = return_output.maxWait ?? 120000; // 2 minutes default + + if (waitForSettled) { + return waitForPaneActivityToSettle(paneId, maxWait, lines); + } else { + return capturePaneContent(paneId, lines, false); + } + } +} + +async function waitForPaneActivityToSettle( + paneId: string, + maxWait: number, + lines: number +): Promise { + const settleTime = 500; // Hardcoded debounce + const pollInterval = 100; // Poll every 100ms + + let lastContent = ''; + let lastChangeTime = Date.now(); + const startTime = Date.now(); + + while (true) { + const elapsed = Date.now() - startTime; + if (elapsed >= maxWait) { + // Timeout - return what we have + return lastContent; + } + + const content = await capturePaneContent(paneId, lines, false); + + if (content !== lastContent) { + // Activity detected - reset settle timer + lastContent = content; + lastChangeTime = Date.now(); + } else if (Date.now() - lastChangeTime >= settleTime) { + // No changes for settleTime ms - settled! + return content; + } + + await new Promise(resolve => setTimeout(resolve, pollInterval)); } } @@ -827,7 +874,7 @@ export async function sendText({ paneId: string; text: string; pressEnter?: boolean; - return_output?: { lines?: number; wait?: number }; + return_output?: { lines?: number; waitForSettled?: boolean; maxWait?: number }; }): Promise { // Send each character with -l flag for literal interpretation for (const char of text) { @@ -843,10 +890,14 @@ export async function sendText({ // If return_output is requested, wait and capture pane content if (return_output) { - if (return_output.wait) { - await new Promise((resolve) => setTimeout(resolve, return_output.wait)); - } const lines = return_output.lines || 200; - return capturePaneContent(paneId, lines, false); + const waitForSettled = return_output.waitForSettled ?? true; + const maxWait = return_output.maxWait ?? 120000; // 2 minutes default + + if (waitForSettled) { + return waitForPaneActivityToSettle(paneId, maxWait, lines); + } else { + return capturePaneContent(paneId, lines, false); + } } } diff --git a/packages/voice-assistant/src/server/daemon/tool-definitions.ts b/packages/voice-assistant/src/server/daemon/tool-definitions.ts index d51577f5a..055bf23ba 100644 --- a/packages/voice-assistant/src/server/daemon/tool-definitions.ts +++ b/packages/voice-assistant/src/server/daemon/tool-definitions.ts @@ -102,16 +102,21 @@ export const terminalTools = [ return_output: { type: "object", description: - "Optional. If provided, will wait and return the terminal output after sending the text.", + "Optional. If provided, will wait for terminal activity to settle and return the output.", properties: { lines: { type: "number", description: "Number of lines to capture. Defaults to 200.", }, - wait: { + waitForSettled: { + type: "boolean", + description: + "Whether to wait for terminal activity to settle before returning output. Polls terminal output and waits 500ms after last change. Defaults to true.", + }, + maxWait: { type: "number", description: - "Milliseconds to wait before capturing output. Defaults to 500ms.", + "Maximum milliseconds to wait for activity to settle. Defaults to 120000 (2 minutes).", }, }, }, diff --git a/packages/voice-assistant/src/server/index.ts b/packages/voice-assistant/src/server/index.ts index f86d13543..4a47c6927 100644 --- a/packages/voice-assistant/src/server/index.ts +++ b/packages/voice-assistant/src/server/index.ts @@ -2,11 +2,7 @@ import "dotenv/config"; import express from "express"; import path from "path"; import { fileURLToPath } from "url"; -import { - createServer as createHTTPServer, - Server as HttpServer, - RequestListener, -} from "http"; +import { createServer as createHTTPServer, Server as HttpServer } from "http"; import { readFile } from "fs/promises"; import { v4 as uuidv4 } from "uuid"; import { createServer as createViteServer } from "vite"; @@ -22,9 +18,8 @@ import { killTerminal, } from "./daemon/terminal-manager.js"; import { initializeSTT, transcribeAudio } from "./agent/stt-openai.js"; -import { initializeTTS, synthesizeSpeech } from "./agent/tts-openai.js"; +import { initializeTTS } from "./agent/tts-openai.js"; import { - createConversation, processUserMessage, cleanupConversations, } from "./agent/orchestrator.js"; @@ -70,20 +65,19 @@ async function createServer(httpServer: HttpServer, config: ServerConfig) { initialCommand: undefined, }); testResults.push( - ` ✓ Created terminal: ${terminal.id} (${terminal.name})` + ` ✓ Created terminal: ${terminal.name}` ); // 4. Send a command to the terminal testResults.push('4. Sending command "echo hello world"...'); - await sendText(terminal.id, 'echo "hello world"', true, { + await sendText(terminal.name, 'echo "hello world"', true, { lines: 20, - wait: 500, }); testResults.push(" ✓ Command sent"); // 5. Capture output testResults.push("5. Capturing terminal output..."); - const output = await captureTerminal(terminal.id, 20); + const output = await captureTerminal(terminal.name, 20); testResults.push(` ✓ Captured ${output.split("\n").length} lines`); testResults.push(` Output preview: ${output.substring(0, 100)}...`); @@ -95,12 +89,12 @@ async function createServer(httpServer: HttpServer, config: ServerConfig) { (t) => t.name === "test-terminal" ); if (testTerminal) { - testResults.push(` ✓ Found test-terminal: ${testTerminal.id}`); + testResults.push(` ✓ Found test-terminal`); } // 7. Kill the test terminal testResults.push("7. Killing test terminal..."); - await killTerminal(terminal.id); + await killTerminal(terminal.name); testResults.push(" ✓ Terminal killed"); // 8. Verify it's gone @@ -231,59 +225,8 @@ async function main() { ); } - // Create default conversation - const defaultConversationId = createConversation(); - console.log(`✓ Default conversation created: ${defaultConversationId}`); - - // Helper function to process message and optionally generate TTS - async function processMessageWithOptionalTTS( - message: string, - enableTTS: boolean = false - ): Promise { - const response = await processUserMessage({ - conversationId: defaultConversationId, - message, - wsServer, - }); - - // Generate TTS for the response if enabled - if (enableTTS && apiKey) { - try { - console.log("[TTS] Generating speech for assistant response..."); - const speechResult = await synthesizeSpeech(response); - - // Convert audio buffer to base64 - const base64Audio = speechResult.audio.toString("base64"); - - // Send audio to client via WebSocket - wsServer.broadcast({ - type: "audio_output", - payload: { - id: uuidv4(), - audio: base64Audio, - format: speechResult.format, - }, - }); - - console.log( - `[TTS] Sent audio to client via WebSocket (${speechResult.audio.length} bytes)` - ); - } catch (error: any) { - console.error("[TTS] Error generating speech:", error); - wsServer.broadcastActivityLog({ - id: uuidv4(), - timestamp: new Date(), - type: "error", - content: `TTS error: ${error.message}`, - }); - } - } - - return response; - } - // Wire orchestrator to WebSocket for text messages - wsServer.setMessageHandler(async (message: string) => { + wsServer.setMessageHandler(async (conversationId: string, message: string, abortSignal: AbortSignal) => { try { // Broadcast user's text message as activity log wsServer.broadcastActivityLog({ @@ -294,9 +237,11 @@ async function main() { }); await processUserMessage({ - conversationId: defaultConversationId, + conversationId, message, wsServer, + enableTTS: true, + abortSignal, }); } catch (error: any) { console.error("[Orchestrator] Error processing message:", error); @@ -311,7 +256,7 @@ async function main() { // Wire audio handler to WebSocket for voice input (STT) wsServer.setAudioHandler( - async (audio: Buffer, format: string): Promise => { + async (conversationId: string, audio: Buffer, format: string, abortSignal: AbortSignal): Promise => { try { // Transcribe audio using OpenAI Whisper const result = await transcribeAudio(audio, format); @@ -330,7 +275,13 @@ async function main() { // Process the transcribed text through the orchestrator WITH TTS enabled // Since this came from voice input, respond with voice output - await processMessageWithOptionalTTS(result.text, true); + await processUserMessage({ + conversationId, + message: result.text, + wsServer, + enableTTS: true, + abortSignal, + }); return result.text; } catch (error: any) { diff --git a/packages/voice-assistant/src/server/types.ts b/packages/voice-assistant/src/server/types.ts index ab72896c3..96328073a 100644 --- a/packages/voice-assistant/src/server/types.ts +++ b/packages/voice-assistant/src/server/types.ts @@ -15,7 +15,7 @@ export interface ActivityLogEntry { export interface WebSocketMessage { type: 'activity_log' | 'status' | 'ping' | 'pong' | 'user_message' | 'assistant_chunk' - | 'audio_chunk' | 'audio_output' | 'recording_state' | 'transcription_result'; + | 'audio_chunk' | 'audio_output' | 'recording_state' | 'transcription_result' | 'audio_played'; payload: unknown; } @@ -40,3 +40,7 @@ export interface AudioOutputPayload { format: string; // 'mp3' id: string; // unique ID for queue management } + +export interface AudioPlayedPayload { + id: string; // unique ID of the audio that finished playing +} diff --git a/packages/voice-assistant/src/server/websocket-server.ts b/packages/voice-assistant/src/server/websocket-server.ts index adc963fa3..dd37e5f45 100644 --- a/packages/voice-assistant/src/server/websocket-server.ts +++ b/packages/voice-assistant/src/server/websocket-server.ts @@ -4,13 +4,18 @@ import type { WebSocketMessage, ActivityLogEntry, AudioChunkPayload, + AudioPlayedPayload, } from "./types.js"; +import { confirmAudioPlayed } from "./agent/tts-manager.js"; +import { createConversation, deleteConversation } from "./agent/orchestrator.js"; export class VoiceAssistantWebSocketServer { private wss: WebSocketServer; private clients: Map = new Map(); // Map ws to client ID - private messageHandler?: (message: string) => Promise; - private audioHandler?: (audio: Buffer, format: string) => Promise; + private conversationIds: Map = new Map(); // Map ws to conversation ID + private abortControllers: Map = new Map(); // Map ws to AbortController + private messageHandler?: (conversationId: string, message: string, abortSignal: AbortSignal) => Promise; + private audioHandler?: (conversationId: string, audio: Buffer, format: string, abortSignal: AbortSignal) => Promise; private audioBuffers: Map = new Map(); private clientIdCounter: number = 0; @@ -29,8 +34,17 @@ export class VoiceAssistantWebSocketServer { // Generate unique client ID const clientId = `client-${++this.clientIdCounter}`; this.clients.set(ws, clientId); + + // Create new conversation for this client + const conversationId = createConversation(); + this.conversationIds.set(ws, conversationId); + + // Create AbortController for this client + const abortController = new AbortController(); + this.abortControllers.set(ws, abortController); + console.log( - `[WS] Client connected: ${clientId} (total: ${this.clients.size})` + `[WS] Client connected: ${clientId} with conversation ${conversationId} (total: ${this.clients.size})` ); // Send welcome message @@ -48,20 +62,50 @@ export class VoiceAssistantWebSocketServer { ws.on("close", () => { const clientId = this.clients.get(ws); + const conversationId = this.conversationIds.get(ws); + const abortController = this.abortControllers.get(ws); + + // Abort any ongoing operations + if (abortController) { + abortController.abort(); + this.abortControllers.delete(ws); + console.log(`[WS] Aborted operations for ${clientId}`); + } + if (clientId) { this.clients.delete(ws); console.log( `[WS] Client disconnected: ${clientId} (total: ${this.clients.size})` ); } + + if (conversationId) { + deleteConversation(conversationId); + this.conversationIds.delete(ws); + console.log(`[WS] Conversation ${conversationId} deleted`); + } }); ws.on("error", (error) => { console.error("[WS] Client error:", error); const clientId = this.clients.get(ws); + const conversationId = this.conversationIds.get(ws); + const abortController = this.abortControllers.get(ws); + + // Abort any ongoing operations + if (abortController) { + abortController.abort(); + this.abortControllers.delete(ws); + } + if (clientId) { this.clients.delete(ws); } + + if (conversationId) { + deleteConversation(conversationId); + this.conversationIds.delete(ws); + } }); } @@ -82,8 +126,18 @@ export class VoiceAssistantWebSocketServer { case "user_message": // Handle user message through orchestrator const payload = message.payload as { message: string }; + const conversationId = this.conversationIds.get(ws); + const abortController = this.abortControllers.get(ws); + if (!conversationId) { + console.error("[WS] No conversation found for client"); + break; + } + if (!abortController) { + console.error("[WS] No abort controller found for client"); + break; + } if (this.messageHandler) { - await this.messageHandler(payload.message); + await this.messageHandler(conversationId, payload.message, abortController.signal); } else { console.warn("[WS] No message handler registered"); } @@ -94,6 +148,12 @@ export class VoiceAssistantWebSocketServer { await this.handleAudioChunk(ws, message.payload as AudioChunkPayload); break; + case "audio_played": + // Handle audio playback confirmation + const audioPlayedPayload = message.payload as AudioPlayedPayload; + confirmAudioPlayed(audioPlayedPayload.id); + break; + default: console.warn(`[WS] Unknown message type: ${message.type}`); } @@ -136,12 +196,12 @@ export class VoiceAssistantWebSocketServer { }); } - public setMessageHandler(handler: (message: string) => Promise): void { + public setMessageHandler(handler: (conversationId: string, message: string, abortSignal: AbortSignal) => Promise): void { this.messageHandler = handler; } public setAudioHandler( - handler: (audio: Buffer, format: string) => Promise + handler: (conversationId: string, audio: Buffer, format: string, abortSignal: AbortSignal) => Promise ): void { this.audioHandler = handler; } @@ -151,8 +211,19 @@ export class VoiceAssistantWebSocketServer { payload: AudioChunkPayload ): Promise { try { - // Use a client-specific key for buffering (in case multiple clients) - const clientId = "default"; // Could be enhanced with per-client tracking + // Use client-specific key for buffering + const clientId = this.clients.get(ws); + if (!clientId) { + console.error("[WS] No client ID found for WebSocket"); + return; + } + + // Get conversation ID for this client + const conversationId = this.conversationIds.get(ws); + if (!conversationId) { + console.error("[WS] No conversation found for client"); + return; + } // Decode base64 audio data const audioBuffer = Buffer.from(payload.audio, "base64"); @@ -188,6 +259,12 @@ export class VoiceAssistantWebSocketServer { this.audioBuffers.delete(clientId); // Process audio through handler (STT) + const abortController = this.abortControllers.get(ws); + if (!abortController) { + console.error("[WS] No abort controller found for client"); + return; + } + if (this.audioHandler) { this.broadcastActivityLog({ id: Date.now().toString(), @@ -196,7 +273,7 @@ export class VoiceAssistantWebSocketServer { content: "Transcribing audio...", }); - const transcript = await this.audioHandler(completeAudio, format); + const transcript = await this.audioHandler(conversationId, completeAudio, format, abortController.signal); // Send transcription result back to client this.sendToClient(ws, { diff --git a/packages/voice-assistant/src/ui/App.css b/packages/voice-assistant/src/ui/App.css index 361ab04f7..70143500a 100644 --- a/packages/voice-assistant/src/ui/App.css +++ b/packages/voice-assistant/src/ui/App.css @@ -1,30 +1,71 @@ .app { display: flex; flex-direction: column; - min-height: 100vh; + height: 100svh; width: 100%; + overflow: hidden; } .header { - display: flex; + display: none; justify-content: space-between; align-items: center; - padding: 1rem 2rem; + padding: 0.5rem 1rem; background-color: #1a1a1a; border-bottom: 1px solid #333; + flex-shrink: 0; + min-height: 0; + overflow: hidden; + max-width: 100%; + min-width: 0; } .header h1 { - font-size: 1.5rem; + font-size: 1rem; font-weight: 600; + margin: 0; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + min-width: 0; + flex: 1; +} + +.header-status { + display: flex; + align-items: center; + gap: 0.5rem; + flex-shrink: 0; + min-width: 0; +} + +.audio-playing-indicator { + width: 8px; + height: 8px; + background-color: #0f766e; + border-radius: 50%; + animation: pulse-audio 1.5s ease-in-out infinite; +} + +@keyframes pulse-audio { + 0%, 100% { + opacity: 1; + transform: scale(1); + } + 50% { + opacity: 0.5; + transform: scale(1.2); + } } .status-indicator { - padding: 0.5rem 1rem; + padding: 0.25rem 0.75rem; border-radius: 4px; font-size: 0.75rem; font-weight: 500; text-transform: uppercase; + flex-shrink: 0; + white-space: nowrap; } .status-indicator.disconnected { @@ -55,20 +96,24 @@ .main { display: flex; flex: 1; - gap: 2rem; -max-height: 100vh; -overflow: hidden; + gap: 0.5rem; + overflow: hidden; + padding: 0.5rem; + min-height: 0; + max-width: 100%; } .chat-interface { flex: 1; background-color: #1a1a1a; border-radius: 8px; - padding: 2rem; + padding: 1rem; display: flex; flex-direction: column; - gap: 1rem; - height: 100%; + gap: 0.5rem; + min-height: 0; + max-width: 100%; + overflow: hidden; } .chat-interface h2 { @@ -113,23 +158,31 @@ overflow: hidden; display: flex; flex-direction: column; overflow: hidden; + min-height: 0; + max-width: 100%; } .log-entries { flex: 1; overflow-y: auto; + overflow-x: hidden; display: flex; flex-direction: column; - gap: 0.5rem; + gap: 0.25rem; + min-height: 0; + max-width: 100%; } .log-entry { - padding: 0.75rem; + padding: 0.5rem; border-radius: 4px; font-size: 0.875rem; line-height: 1.4; display: flex; - gap: 0.75rem; + gap: 0.5rem; + flex-shrink: 0; + max-width: 100%; + overflow: hidden; } .log-time { @@ -140,6 +193,9 @@ overflow: hidden; .log-message { flex: 1; + min-width: 0; + word-wrap: break-word; + overflow-wrap: break-word; } .log-entry.system { @@ -197,6 +253,8 @@ overflow: hidden; .log-metadata { margin-top: 0.5rem; + max-width: 100%; + overflow: hidden; } .log-metadata summary { @@ -212,24 +270,40 @@ overflow: hidden; border-radius: 4px; overflow-x: auto; font-size: 0.75rem; + max-width: 100%; + word-wrap: break-word; + overflow-wrap: break-word; + white-space: pre-wrap; } .message-form { display: flex; + align-items: flex-end; gap: 0.5rem; - padding-top: 1rem; + padding-top: 0.5rem; border-top: 1px solid #333; + flex-shrink: 0; + min-width: 0; + max-width: 100%; + overflow: hidden; } .message-input { flex: 1; - padding: 0.75rem 1rem; + min-width: 0; + padding: 0.5rem 0.75rem; background-color: #2a2a2a; border: 1px solid #444; border-radius: 6px; color: #fff; font-size: 0.875rem; + font-family: inherit; outline: none; + resize: none; + overflow-y: auto; + max-height: 200px; + line-height: 1.5; + min-height: 26px; } .message-input:focus { @@ -242,27 +316,74 @@ overflow: hidden; } .send-button { - padding: 0.75rem 1.5rem; - background-color: #22c55e; - color: #000; + width: 44px; + height: 44px; + min-width: 44px; + min-height: 44px; + padding: 0; + background-color: #3b82f6; + color: #fff; border: none; - border-radius: 6px; - font-size: 0.875rem; - font-weight: 500; + border-radius: 8px; cursor: pointer; - transition: background-color 0.2s; + transition: all 0.2s; + flex-shrink: 0; + display: flex; + align-items: center; + justify-content: center; } -.send-button:hover:not(:disabled) { +.send-button.has-text { + background-color: #22c55e; + color: #000; +} + +.send-button.has-text:hover:not(:disabled) { background-color: #16a34a; } +.send-button:not(.has-text):hover:not(:disabled) { + background-color: #2563eb; +} + +.send-button.recording { + background-color: #ef4444; + animation: pulse-red 1.5s ease-in-out infinite; +} + +.send-button.recording:hover:not(:disabled) { + background-color: #dc2626; +} + +.send-button.processing { + background-color: #f59e0b; +} + .send-button:disabled { background-color: #444; color: #999; cursor: not-allowed; } +.send-button .recording-indicator { + display: inline-block; + width: 12px; + height: 12px; + background-color: #fff; + border-radius: 50%; + animation: pulse 1s ease-in-out infinite; +} + +.send-button .processing-indicator { + display: inline-block; + width: 12px; + height: 12px; + border: 2px solid #fff; + border-top-color: transparent; + border-radius: 50%; + animation: spin 0.8s linear infinite; +} + .test-controls { padding-top: 0.5rem; border-top: 1px solid #333; @@ -274,11 +395,14 @@ overflow: hidden; .voice-controls { display: flex; flex-direction: column; - gap: 0.75rem; - padding: 1rem; + gap: 0.5rem; + padding: 0.75rem; background-color: #2a2a2a; border-radius: 6px; - margin-bottom: 1rem; + flex-shrink: 0; + min-width: 0; + max-width: 100%; + overflow: hidden; } .voice-button { @@ -286,15 +410,17 @@ overflow: hidden; align-items: center; justify-content: center; gap: 0.5rem; - padding: 1rem 2rem; + padding: 0.75rem 1.5rem; background-color: #3b82f6; color: #fff; border: none; border-radius: 8px; - font-size: 1rem; + font-size: 0.875rem; font-weight: 500; cursor: pointer; transition: all 0.2s; + max-width: 100%; + overflow: hidden; } .voice-button:hover:not(:disabled) { @@ -376,13 +502,138 @@ overflow: hidden; font-size: 0.875rem; } -.playing-indicator { - padding: 0.5rem; - background-color: #0f766e; - color: #ccfbf1; - border-radius: 4px; +/* Tool Call Card Styles */ +.tool-call-card { + background-color: #2a2a2a; + border-radius: 6px; + overflow: hidden; + margin: 0.25rem 0; + border: 1px solid #444; + flex-shrink: 0; +} + +.tool-call-card[data-status="executing"] { + border-color: #f59e0b; +} + +.tool-call-card[data-status="completed"] { + border-color: #22c55e; +} + +.tool-call-header { + width: 100%; + display: flex; + align-items: center; + gap: 0.5rem; + padding: 0.75rem; + background: none; + border: none; + color: #e2e8f0; + cursor: pointer; + font-family: inherit; font-size: 0.875rem; - text-align: center; + text-align: left; + transition: background-color 0.2s; +} + +.tool-call-header:hover { + background-color: #333; +} + +.tool-call-icon { + display: flex; + align-items: center; + justify-content: center; + color: #999; + flex-shrink: 0; +} + +.tool-call-name { + flex: 1; + font-family: monospace; + font-weight: 500; + color: #e2e8f0; +} + +.tool-call-status { + display: flex; + align-items: center; + gap: 0.375rem; + padding: 0.25rem 0.5rem; + border-radius: 4px; + font-size: 0.75rem; + flex-shrink: 0; +} + +.tool-call-card[data-status="executing"] .tool-call-status { + background-color: rgba(245, 158, 11, 0.2); + color: #fbbf24; +} + +.tool-call-card[data-status="completed"] .tool-call-status { + background-color: rgba(34, 197, 94, 0.2); + color: #86efac; +} + +.status-icon { + flex-shrink: 0; +} + +.status-icon.spinning { + animation: spin 1s linear infinite; +} + +.status-text { + font-weight: 500; + text-transform: capitalize; +} + +.tool-call-details { + padding: 0 0.75rem 0.75rem 0.75rem; + display: flex; + flex-direction: column; + gap: 0.75rem; + animation: expand 0.2s ease-out; +} + +@keyframes expand { + from { + opacity: 0; + max-height: 0; + } + to { + opacity: 1; + max-height: 1000px; + } +} + +.tool-call-section { + display: flex; + flex-direction: column; + gap: 0.375rem; +} + +.section-header { + font-size: 0.75rem; + font-weight: 600; + color: #999; + text-transform: uppercase; + letter-spacing: 0.025em; +} + +.section-content { + padding: 0.5rem; + background-color: #1a1a1a; + border-radius: 4px; + font-family: monospace; + font-size: 0.75rem; + color: #e2e8f0; + overflow-x: auto; + border: 1px solid #444; + margin: 0; + white-space: pre-wrap; + word-wrap: break-word; + overflow-wrap: break-word; } @media (max-width: 1024px) { @@ -390,3 +641,28 @@ overflow: hidden; flex-direction: column; } } + +@media (max-width: 768px) { + .main { + padding: 0.25rem; + gap: 0.25rem; + } + + .chat-interface { + padding: 0.5rem; + } + + .header { + padding: 0.5rem 0.5rem; + gap: 0.5rem; + } + + .header h1 { + font-size: 0.875rem; + } + + .status-indicator { + padding: 0.25rem 0.5rem; + font-size: 0.7rem; + } +} diff --git a/packages/voice-assistant/src/ui/App.tsx b/packages/voice-assistant/src/ui/App.tsx index ddc071803..14c3cb3f2 100644 --- a/packages/voice-assistant/src/ui/App.tsx +++ b/packages/voice-assistant/src/ui/App.tsx @@ -1,16 +1,28 @@ import { useState, useEffect, useRef } from "react"; +import { Mic, Send } from "lucide-react"; import { useWebSocket } from "./hooks/useWebSocket"; -import { VoiceControls } from "./components/VoiceControls"; import { createAudioPlayer } from "./lib/audio-playback"; +import { createAudioRecorder, type AudioRecorder } from "./lib/audio-capture"; +import { ToolCallCard } from "./components/ToolCallCard"; import "./App.css"; -interface LogEntry { - id: string; - timestamp: number; - type: "system" | "info" | "success" | "error" | "user" | "assistant" | "tool"; - message: string; - metadata?: Record; -} +type LogEntry = + | { + type: "system" | "info" | "success" | "error" | "user" | "assistant"; + id: string; + timestamp: number; + message: string; + metadata?: Record; + } + | { + type: "tool_call"; + id: string; + timestamp: number; + toolName: string; + args: any; + result?: any; + status: "executing" | "completed"; + }; function App() { const [logs, setLogs] = useState([ @@ -25,8 +37,11 @@ function App() { const [currentAssistantMessage, setCurrentAssistantMessage] = useState(""); const [isProcessingAudio, setIsProcessingAudio] = useState(false); const [isPlayingAudio, setIsPlayingAudio] = useState(false); + const [isRecording, setIsRecording] = useState(false); const logEndRef = useRef(null); const audioPlayerRef = useRef(createAudioPlayer()); + const recorderRef = useRef(null); + const textareaRef = useRef(null); // WebSocket URL - use ws://localhost:3000/ws in dev, or construct from current host in prod const wsUrl = `${window.location.protocol === "https:" ? "wss" : "ws"}://${ @@ -35,8 +50,20 @@ function App() { const ws = useWebSocket(wsUrl); + useEffect(() => { + recorderRef.current = createAudioRecorder(); + }, []); + + useEffect(() => { + if (textareaRef.current) { + textareaRef.current.style.height = 'auto'; + const newHeight = Math.max(textareaRef.current.scrollHeight, 44); + textareaRef.current.style.height = `${newHeight}px`; + } + }, [userInput]); + const addLog = ( - type: LogEntry["type"], + type: "system" | "info" | "success" | "error" | "user" | "assistant", message: string, metadata?: Record ) => { @@ -67,22 +94,59 @@ function App() { // Listen for activity log messages const unsubActivity = ws.on("activity_log", (payload: unknown) => { const data = payload as { + id: string; type: string; content: string; metadata?: Record; }; + // Handle tool calls + if (data.type === "tool_call") { + const { toolCallId, toolName, arguments: args } = data.metadata as { + toolCallId: string; + toolName: string; + arguments: any; + }; + + setLogs((prev) => [ + ...prev, + { + type: "tool_call", + id: toolCallId, + timestamp: Date.now(), + toolName, + args, + status: "executing", + }, + ]); + return; + } + + if (data.type === "tool_result") { + const { toolCallId, result } = data.metadata as { + toolCallId: string; + result: any; + }; + + setLogs((prev) => + prev.map((log) => + log.type === "tool_call" && log.id === toolCallId + ? { ...log, result, status: "completed" as const } + : log + ) + ); + return; + } + // Map activity log types to UI log types - let logType: LogEntry["type"] = "info"; + let logType: "system" | "info" | "success" | "error" | "user" | "assistant" = "info"; if (data.type === "transcript") logType = "user"; else if (data.type === "assistant") logType = "assistant"; - else if (data.type === "tool_call" || data.type === "tool_result") - logType = "tool"; else if (data.type === "error") logType = "error"; addLog(logType, data.content, data.metadata); - // Clear streaming state when complete assistant message arrives + // Clear streaming state when assistant segment is received if (data.type === "assistant") { setCurrentAssistantMessage(""); } @@ -97,7 +161,7 @@ function App() { // Listen for transcription results const unsubTranscription = ws.on( "transcription_result", - (payload: unknown) => { + (_payload: unknown) => { // Note: Transcription is already broadcast as activity_log with type "transcript" // No need to log it again here to avoid duplication setIsProcessingAudio(false); @@ -109,7 +173,6 @@ function App() { const data = payload as { audio: string; format: string; id: string }; try { - addLog("info", "Playing assistant audio response..."); setIsPlayingAudio(true); // Decode base64 audio @@ -127,8 +190,13 @@ function App() { // Play audio await audioPlayerRef.current.play(audioBlob); + // Send confirmation back to server + ws.send({ + type: "audio_played", + payload: { id: data.id }, + }); + setIsPlayingAudio(false); - addLog("success", "Audio playback complete"); } catch (error: any) { console.error("[App] Audio playback error:", error); addLog("error", `Audio playback failed: ${error.message}`); @@ -159,13 +227,7 @@ function App() { logEndRef.current?.scrollIntoView({ behavior: "smooth" }); }, [logs, currentAssistantMessage]); - const handlePing = () => { - ws.sendPing(); - addLog("info", "Sent ping to server"); - }; - - const handleSendMessage = (e: React.FormEvent) => { - e.preventDefault(); + const handleSendMessage = () => { if (!userInput.trim() || !ws.isConnected) return; // Send message to server @@ -176,42 +238,74 @@ function App() { setCurrentAssistantMessage(""); }; - const handleAudioRecorded = async (audioBlob: Blob, format: string) => { - if (!ws.isConnected) { - addLog("error", "Cannot send audio - not connected to server"); - return; - } + const handleToggleRecording = async () => { + const recorder = recorderRef.current; + if (!recorder || !ws.isConnected) return; try { - setIsProcessingAudio(true); - addLog("info", "Sending audio to server..."); + if (isRecording) { + console.log('[App] Stopping recording...'); + const audioBlob = await recorder.stop(); + setIsRecording(false); - // Convert blob to base64 - const arrayBuffer = await audioBlob.arrayBuffer(); - const base64Audio = btoa( - new Uint8Array(arrayBuffer).reduce( - (data, byte) => data + String.fromCharCode(byte), - "" - ) - ); + const format = audioBlob.type || 'audio/webm'; + console.log(`[App] Recording complete: ${audioBlob.size} bytes, format: ${format}`); - // Send audio chunk via WebSocket - ws.send({ - type: "audio_chunk", - payload: { - audio: base64Audio, - format: format, - isLast: true, - }, - }); + setIsProcessingAudio(true); + addLog("info", "Sending audio to server..."); - console.log( - `[App] Sent audio: ${audioBlob.size} bytes, format: ${format}` - ); + const arrayBuffer = await audioBlob.arrayBuffer(); + const base64Audio = btoa( + new Uint8Array(arrayBuffer).reduce( + (data, byte) => data + String.fromCharCode(byte), + "" + ) + ); + + ws.send({ + type: "audio_chunk", + payload: { + audio: base64Audio, + format: format, + isLast: true, + }, + }); + + console.log(`[App] Sent audio: ${audioBlob.size} bytes, format: ${format}`); + } else { + console.log('[App] Starting recording...'); + await recorder.start(); + setIsRecording(true); + } } catch (error: any) { - console.error("[App] Error sending audio:", error); - addLog("error", `Failed to send audio: ${error.message}`); - setIsProcessingAudio(false); + console.error('[App] Recording error:', error); + addLog("error", `Failed to record audio: ${error.message}`); + setIsRecording(false); + } + }; + + const handleButtonClick = async (e: React.MouseEvent) => { + e.preventDefault(); + + if (userInput.trim()) { + handleSendMessage(); + } else { + await handleToggleRecording(); + } + }; + + const handleKeyDown = (e: React.KeyboardEvent) => { + // Detect if device is desktop (not mobile/tablet) + const isDesktop = window.matchMedia('(pointer: fine)').matches; + + if (e.key === 'Enter') { + if (isDesktop && !e.shiftKey && userInput.trim()) { + // Desktop: plain Enter sends message + e.preventDefault(); + handleSendMessage(); + } + // Desktop: Shift+Enter creates new line (default behavior) + // Mobile: Enter creates new line (default behavior) } }; @@ -219,12 +313,15 @@ function App() {

Voice Assistant

-
- {ws.isConnected ? "connected" : "disconnected"} +
+ {isPlayingAudio &&
} +
+ {ws.isConnected ? "connected" : "disconnected"} +
@@ -232,25 +329,29 @@ function App() {
- {logs.map((log) => ( -
- - {new Date(log.timestamp).toLocaleTimeString()} - - {log.message} - {log.metadata && ( -
- Details - 
{JSON.stringify(log.metadata, null, 2)}
-
- )} -
- ))} + {logs.map((log) => + log.type === "tool_call" ? ( + + ) : ( +
+ {log.message} + {log.metadata && ( +
+ Details + 
{JSON.stringify(log.metadata, null, 2)}
+
+ )} +
+ ) + )} {currentAssistantMessage && (
- - {new Date().toLocaleTimeString()} - {currentAssistantMessage} ...
@@ -259,29 +360,34 @@ function App() {
-
- +