diff --git a/package-lock.json b/package-lock.json index 7e78e8743..5a4da3286 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2326,6 +2326,33 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "license": "ISC", + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/anymatch/node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -2358,6 +2385,19 @@ "baseline-browser-mapping": "dist/cli.js" } }, + "node_modules/binary-extensions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/body-parser": { "version": "1.20.3", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", @@ -2397,6 +2437,19 @@ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", "license": "MIT" }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/browserslist": { "version": "4.26.3", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.26.3.tgz", @@ -2570,6 +2623,31 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/chokidar": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", + "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, "node_modules/chownr": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", @@ -3198,6 +3276,19 @@ } } }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/finalhandler": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", @@ -3384,6 +3475,19 @@ "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" } }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/gopd": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", @@ -3583,6 +3687,19 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/is-decimal": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-2.0.1.tgz", @@ -3593,6 +3710,16 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", @@ -3603,6 +3730,19 @@ "node": ">=8" } }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-hexadecimal": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-2.0.1.tgz", @@ -3613,6 +3753,16 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, "node_modules/is-plain-obj": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", @@ -4771,6 +4921,16 @@ "dev": true, "license": "MIT" }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -4878,6 +5038,19 @@ "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", "license": "MIT" }, + "node_modules/p-map": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-7.0.3.tgz", + "integrity": "sha512-VkndIv2fIB99swvQoA65bm+fsmt6UNdGeIB0oxBs+WhAhdh08QA04JXpI7rbB9r08/nkbysKoya9rtDERYOYMA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/parse-entities": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.2.tgz", @@ -5179,6 +5352,32 @@ "node": ">=0.10.0" } }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/readdirp/node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/remark-parse": { "version": "11.0.0", "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-11.0.0.tgz", @@ -5661,6 +5860,19 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, "node_modules/toidentifier": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", @@ -6021,6 +6233,25 @@ } } }, + "node_modules/vite-plugin-static-copy": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/vite-plugin-static-copy/-/vite-plugin-static-copy-3.1.4.tgz", + "integrity": "sha512-iCmr4GSw4eSnaB+G8zc2f4dxSuDjbkjwpuBLLGvQYR9IW7rnDzftnUjOH5p4RYR+d4GsiBqXRvzuFhs5bnzVyw==", + "dev": true, + "license": "MIT", + "dependencies": { + "chokidar": "^3.6.0", + "p-map": "^7.0.3", + "picocolors": "^1.1.1", + "tinyglobby": "^0.2.15" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "peerDependencies": { + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0" + } + }, "node_modules/web-streams-polyfill": { "version": "4.0.0-beta.3", "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", @@ -6257,7 +6488,8 @@ "react-dom": "^18.2.0", "tsx": "^4.6.0", "typescript": "^5.2.2", - "vite": "^7.1.10" + "vite": "^7.1.10", + "vite-plugin-static-copy": "^3.1.4" } }, "packages/voice-assistant/node_modules/@types/node": { diff --git a/packages/voice-assistant/package.json b/packages/voice-assistant/package.json index f41ba3b07..bed252f6e 100644 --- a/packages/voice-assistant/package.json +++ b/packages/voice-assistant/package.json @@ -46,6 +46,7 @@ "react-dom": "^18.2.0", "tsx": "^4.6.0", "typescript": "^5.2.2", - "vite": "^7.1.10" + "vite": "^7.1.10", + "vite-plugin-static-copy": "^3.1.4" } } diff --git a/packages/voice-assistant/src/server/agent/playwright-mcp.ts b/packages/voice-assistant/src/server/agent/playwright-mcp.ts index 032152d04..83a5e06e7 100644 --- a/packages/voice-assistant/src/server/agent/playwright-mcp.ts +++ b/packages/voice-assistant/src/server/agent/playwright-mcp.ts @@ -15,7 +15,7 @@ export async function getPlaywrightMCPClient() { const transport = new StdioClientTransport({ command: "npx", - args: ["@playwright/mcp", "--browser", "firefox"], + args: ["@playwright/mcp", "--image-responses", "omit"], }); playwrightMCPClient = await experimental_createMCPClient({ diff --git a/packages/voice-assistant/src/server/websocket-server.ts b/packages/voice-assistant/src/server/websocket-server.ts index dd37e5f45..418467b2f 100644 --- a/packages/voice-assistant/src/server/websocket-server.ts +++ b/packages/voice-assistant/src/server/websocket-server.ts @@ -127,17 +127,23 @@ export class VoiceAssistantWebSocketServer { // Handle user message through orchestrator const payload = message.payload as { message: string }; const conversationId = this.conversationIds.get(ws); - const abortController = this.abortControllers.get(ws); if (!conversationId) { console.error("[WS] No conversation found for client"); break; } - if (!abortController) { - console.error("[WS] No abort controller found for client"); - break; + + // Abort any ongoing request + const oldController = this.abortControllers.get(ws); + if (oldController) { + oldController.abort(); } + + // Create new abort controller for this request + const newController = new AbortController(); + this.abortControllers.set(ws, newController); + if (this.messageHandler) { - await this.messageHandler(conversationId, payload.message, abortController.signal); + await this.messageHandler(conversationId, payload.message, newController.signal); } else { console.warn("[WS] No message handler registered"); } @@ -258,13 +264,16 @@ export class VoiceAssistantWebSocketServer { // Clear buffer this.audioBuffers.delete(clientId); - // Process audio through handler (STT) - const abortController = this.abortControllers.get(ws); - if (!abortController) { - console.error("[WS] No abort controller found for client"); - return; + // Abort any ongoing request + const oldAbortController = this.abortControllers.get(ws); + if (oldAbortController) { + oldAbortController.abort(); } + // Create new abort controller for this request + const newAbortController = new AbortController(); + this.abortControllers.set(ws, newAbortController); + if (this.audioHandler) { this.broadcastActivityLog({ id: Date.now().toString(), @@ -273,7 +282,7 @@ export class VoiceAssistantWebSocketServer { content: "Transcribing audio...", }); - const transcript = await this.audioHandler(conversationId, completeAudio, format, abortController.signal); + const transcript = await this.audioHandler(conversationId, completeAudio, format, newAbortController.signal); // Send transcription result back to client this.sendToClient(ws, { diff --git a/packages/voice-assistant/src/ui/lib/audio-realtime.ts b/packages/voice-assistant/src/ui/lib/audio-realtime.ts index 206fb9a09..9f59c1bdd 100644 --- a/packages/voice-assistant/src/ui/lib/audio-realtime.ts +++ b/packages/voice-assistant/src/ui/lib/audio-realtime.ts @@ -65,9 +65,19 @@ export function createRealtimeVAD(config: RealtimeVADConfig): RealtimeVAD { }, positiveSpeechThreshold: config.positiveSpeechThreshold ?? 0.5, negativeSpeechThreshold: config.negativeSpeechThreshold ?? 0.35, - redemptionMs: config.redemptionFrames ? config.redemptionFrames * 16 : 128, - preSpeechPadMs: config.preSpeechPadFrames ? config.preSpeechPadFrames * 16 : 16, - minSpeechMs: config.minSpeechFrames ? config.minSpeechFrames * 16 : 48, + // Increased redemptionMs to wait longer before deciding speech has ended (allows for natural pauses) + redemptionMs: config.redemptionFrames + ? config.redemptionFrames * 16 + : 2000, + // Padding before speech starts + preSpeechPadMs: config.preSpeechPadFrames + ? config.preSpeechPadFrames * 16 + : 300, + // Minimum speech duration to avoid capturing very short sounds + minSpeechMs: config.minSpeechFrames ? config.minSpeechFrames * 16 : 500, + ortConfig: (ort) => { + ort.env.wasm.wasmPaths = "/"; + }, }); isActive = true; @@ -75,8 +85,7 @@ export function createRealtimeVAD(config: RealtimeVADConfig): RealtimeVAD { console.log("[RealtimeVAD] VAD created and started successfully"); } catch (error) { console.error("[RealtimeVAD] Failed to start VAD:", error); - const err = - error instanceof Error ? error : new Error(String(error)); + const err = error instanceof Error ? error : new Error(String(error)); config.onError?.(err); throw err; } diff --git a/packages/voice-assistant/vite.config.ts b/packages/voice-assistant/vite.config.ts index ba2b26ce0..df7aeea85 100644 --- a/packages/voice-assistant/vite.config.ts +++ b/packages/voice-assistant/vite.config.ts @@ -1,10 +1,45 @@ import { defineConfig } from "vite"; import react from "@vitejs/plugin-react"; import path from "path"; +import { viteStaticCopy } from "vite-plugin-static-copy"; // https://vitejs.dev/config/ export default defineConfig({ - plugins: [react()], + plugins: [ + react(), +viteStaticCopy({ + targets: [ + { + src: path.resolve(__dirname, '../../node_modules/@ricky0123/vad-web/dist/vad.worklet.bundle.min.js'), + dest: './' + }, + { + src: path.resolve(__dirname, '../../node_modules/@ricky0123/vad-web/dist/silero_vad_v5.onnx'), + dest: './' + }, + { + src: path.resolve(__dirname, '../../node_modules/@ricky0123/vad-web/dist/silero_vad_legacy.onnx'), + dest: './' + }, + { + src: path.resolve(__dirname, '../../node_modules/onnxruntime-web/dist/ort-wasm-simd-threaded.mjs'), + dest: './' + }, + { + src: path.resolve(__dirname, '../../node_modules/onnxruntime-web/dist/ort-wasm-simd-threaded.wasm'), + dest: './' + }, + { + src: path.resolve(__dirname, '../../node_modules/onnxruntime-web/dist/ort-wasm-simd-threaded.jsep.mjs'), + dest: './' + }, + { + src: path.resolve(__dirname, '../../node_modules/onnxruntime-web/dist/ort-wasm-simd-threaded.jsep.wasm'), + dest: './' + } + ] + }) + ], root: "src/ui", build: { outDir: path.resolve(__dirname, "dist/ui"),