mirror of
https://github.com/getpaseo/paseo.git
synced 2026-07-29 12:01:31 +00:00
feat: add realtime voice mode, artifact presentation, and Playwright MCP integration
This commit is contained in:
Mohamed Boudra
1863
package-lock.json
generated
1863
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -273,6 +273,10 @@ You interact with the user's machine through **terminals** (isolated shell envir
|
||||
- **rename-terminal(terminalId, name)** - Rename a terminal
|
||||
- **kill-terminal(terminalId)** - Close a terminal
|
||||
|
||||
**Visual Presentation Tools:**
|
||||
|
||||
- **present-artifact(type, content, title?)** - Present visual content to user (plans, diffs, etc.)
|
||||
|
||||
### Creating Terminals with Context
|
||||
|
||||
**CRITICAL**: Always set `workingDirectory` based on context:
|
||||
@@ -323,6 +327,124 @@ You: create-terminal(name="authentication", workingDirectory="WHATEVER THE CURRE
|
||||
- Terminal @124 "tests": ~/dev/voice-dev/packages/web (idle, ready for commands)
|
||||
- Terminal @125 "mcp": ~/dev/voice-dev/packages/mcp-server (running MCP server)
|
||||
|
||||
## Presenting Visual Content
|
||||
|
||||
### Using present_artifact Tool
|
||||
|
||||
**Purpose**: Present information that's hard to convey via voice (plans, diffs, code, images) to the user's screen.
|
||||
|
||||
**Tool Signature:**
|
||||
|
||||
```javascript
|
||||
present_artifact({
|
||||
type: "markdown" | "diff" | "image" | "code",
|
||||
source: {
|
||||
type: "text" | "file" | "command_output",
|
||||
// ... source-specific fields
|
||||
},
|
||||
title: "Display Title"
|
||||
})
|
||||
```
|
||||
|
||||
**Source Types:**
|
||||
|
||||
1. **text** - Direct inline content (string)
|
||||
2. **file** - Read from filesystem (server encodes to base64)
|
||||
3. **command_output** - Execute shell command, use stdout (server encodes to base64)
|
||||
|
||||
**Examples:**
|
||||
|
||||
**Markdown - Claude Code Plan (text source):**
|
||||
```javascript
|
||||
present_artifact({
|
||||
type: "markdown",
|
||||
source: {
|
||||
type: "text",
|
||||
text: "# Authentication Plan\n\n## Steps\n1. Add JWT middleware\n2. ..."
|
||||
},
|
||||
title: "Authentication Feature Plan"
|
||||
})
|
||||
```
|
||||
|
||||
**Diff - Git Changes (command source):**
|
||||
```javascript
|
||||
present_artifact({
|
||||
type: "diff",
|
||||
source: {
|
||||
type: "command_output",
|
||||
command: "git diff HEAD~1"
|
||||
},
|
||||
title: "Recent Changes"
|
||||
})
|
||||
```
|
||||
|
||||
**Code - Package.json (file source):**
|
||||
```javascript
|
||||
present_artifact({
|
||||
type: "code",
|
||||
source: {
|
||||
type: "file",
|
||||
path: "/Users/user/project/package.json"
|
||||
},
|
||||
title: "package.json"
|
||||
})
|
||||
```
|
||||
|
||||
**Image - From URL (command source):**
|
||||
```javascript
|
||||
present_artifact({
|
||||
type: "image",
|
||||
source: {
|
||||
type: "command_output",
|
||||
command: "curl -s https://example.com/image.jpg"
|
||||
},
|
||||
title: "Sample Image"
|
||||
})
|
||||
```
|
||||
|
||||
**Image - From File (file source):**
|
||||
```javascript
|
||||
present_artifact({
|
||||
type: "image",
|
||||
source: {
|
||||
type: "file",
|
||||
path: "/tmp/screenshot.png"
|
||||
},
|
||||
title: "Screenshot"
|
||||
})
|
||||
```
|
||||
|
||||
### When Claude Code Shows a Plan
|
||||
|
||||
**CRITICAL**: When Claude Code presents a plan in plan mode, forward it to the user's screen.
|
||||
|
||||
**Pattern:**
|
||||
|
||||
1. Capture the plan from Claude's terminal output
|
||||
2. Use `present_artifact` with text source
|
||||
3. Tell the user: "Check your screen to review the plan"
|
||||
|
||||
**Example:**
|
||||
|
||||
```
|
||||
User: "Ask Claude to plan the authentication feature"
|
||||
You: [Launch Claude in plan mode with the prompt]
|
||||
You: [Claude presents a plan in the terminal]
|
||||
You: present_artifact({
|
||||
type: "markdown",
|
||||
source: { type: "text", text: "<captured plan markdown>" },
|
||||
title: "Authentication Feature Plan"
|
||||
})
|
||||
You: "Check your screen to review the plan."
|
||||
```
|
||||
|
||||
**Why this matters:**
|
||||
|
||||
- Plans are long and detailed - hard to convey via voice
|
||||
- User can read the formatted markdown plan on their screen
|
||||
- They can accept or reject it visually
|
||||
- You only need to say "Check your screen to review the plan"
|
||||
|
||||
## Claude Code Integration
|
||||
|
||||
### What is Claude Code?
|
||||
|
||||
@@ -17,12 +17,16 @@
|
||||
"dependencies": {
|
||||
"@ai-sdk/openai": "^2.0.52",
|
||||
"@deepgram/sdk": "^3.4.0",
|
||||
"@modelcontextprotocol/sdk": "^1.20.1",
|
||||
"@openrouter/ai-sdk-provider": "^1.2.0",
|
||||
"@ricky0123/vad-web": "^0.0.28",
|
||||
"ai": "^5.0.76",
|
||||
"dotenv": "^17.2.3",
|
||||
"express": "^4.18.2",
|
||||
"lucide-react": "^0.546.0",
|
||||
"openai": "^4.20.0",
|
||||
"playwright": "^1.56.1",
|
||||
"react-markdown": "^10.1.0",
|
||||
"tiny-invariant": "^1.3.3",
|
||||
"uuid": "^9.0.1",
|
||||
"ws": "^8.14.2",
|
||||
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
killTerminal,
|
||||
} from "../daemon/terminal-manager.js";
|
||||
import invariant from "tiny-invariant";
|
||||
import { getPlaywrightTools } from "./playwright-mcp.js";
|
||||
|
||||
/**
|
||||
* Terminal tools using Vercel AI SDK tool() function
|
||||
@@ -208,8 +209,68 @@ export const terminalTools = {
|
||||
return { success: true };
|
||||
},
|
||||
}),
|
||||
|
||||
present_artifact: tool({
|
||||
description:
|
||||
"Present an artifact (plan, diff, screenshot, etc.) to the user for review. Use this when you need to show information that's hard to convey via TTS, such as markdown plans, code diffs, or visual content",
|
||||
inputSchema: z.object({
|
||||
type: z
|
||||
.enum(["markdown", "diff", "image", "code"])
|
||||
.describe("Type of artifact to present."),
|
||||
source: z.union([
|
||||
z.object({
|
||||
type: z.literal("file"),
|
||||
path: z.string(),
|
||||
}),
|
||||
z.object({
|
||||
type: z.literal("command_output"),
|
||||
command: z.string(),
|
||||
}),
|
||||
z.object({
|
||||
type: z.literal("text"),
|
||||
text: z.string(),
|
||||
}),
|
||||
]),
|
||||
title: z
|
||||
.string()
|
||||
.describe(
|
||||
"Title for the artifact (e.g., 'Implementation Plan', 'Refactoring Strategy', '/path/to/project/package.json')."
|
||||
),
|
||||
}),
|
||||
execute: async () => {
|
||||
// Artifact will be broadcast by orchestrator via onToolCall callback
|
||||
// We just return a simple acknowledgment here
|
||||
return {
|
||||
success: true,
|
||||
message: "Artifact presented to user.",
|
||||
};
|
||||
},
|
||||
}),
|
||||
};
|
||||
|
||||
/**
|
||||
* Cache Playwright MCP tools (lazy loaded on first use)
|
||||
*/
|
||||
let playwrightToolsCache: Record<string, any> | null = null;
|
||||
let playwrightToolsPromise: Promise<Record<string, any>> | null = null;
|
||||
|
||||
async function getCachedPlaywrightTools(): Promise<Record<string, any>> {
|
||||
if (playwrightToolsCache) {
|
||||
return playwrightToolsCache;
|
||||
}
|
||||
|
||||
if (playwrightToolsPromise) {
|
||||
return playwrightToolsPromise;
|
||||
}
|
||||
|
||||
playwrightToolsPromise = getPlaywrightTools().then((tools) => {
|
||||
playwrightToolsCache = tools;
|
||||
return tools;
|
||||
});
|
||||
|
||||
return playwrightToolsPromise;
|
||||
}
|
||||
|
||||
/**
|
||||
* Message interface for conversation
|
||||
*/
|
||||
@@ -252,11 +313,18 @@ export async function streamLLM(params: StreamLLMParams): Promise<string> {
|
||||
apiKey: process.env.OPENROUTER_API_KEY,
|
||||
});
|
||||
|
||||
// Merge terminal tools with Playwright MCP tools
|
||||
const pwTools = await getCachedPlaywrightTools();
|
||||
const allTools = {
|
||||
...terminalTools,
|
||||
...pwTools,
|
||||
};
|
||||
|
||||
const result = await streamText({
|
||||
model: openrouter("anthropic/claude-haiku-4.5"),
|
||||
system: params.systemPrompt,
|
||||
messages: params.messages,
|
||||
tools: terminalTools,
|
||||
tools: allTools,
|
||||
abortSignal: params.abortSignal,
|
||||
onChunk: async ({ chunk }) => {
|
||||
// console.log("onChunk", chunk);
|
||||
|
||||
@@ -1,8 +1,14 @@
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { readFile } from "fs/promises";
|
||||
import { exec } from "child_process";
|
||||
import { promisify } from "util";
|
||||
import { getSystemPrompt } from "./system-prompt.js";
|
||||
import { streamLLM, type Message } from "./llm-openai.js";
|
||||
import { generateTTSAndWaitForPlayback } from "./tts-manager.js";
|
||||
import type { VoiceAssistantWebSocketServer } from "../websocket-server.js";
|
||||
import type { ArtifactPayload } from "../types.js";
|
||||
|
||||
const execAsync = promisify(exec);
|
||||
|
||||
interface ConversationContext {
|
||||
id: string;
|
||||
@@ -109,6 +115,60 @@ export async function processUserMessage(params: {
|
||||
await pendingTTS;
|
||||
pendingTTS = null;
|
||||
}
|
||||
|
||||
// Handle present_artifact tool specially
|
||||
if (toolName === "present_artifact" && params.wsServer) {
|
||||
const artifactId = uuidv4();
|
||||
|
||||
// Resolve source to content
|
||||
let content: string;
|
||||
let isBase64 = false;
|
||||
try {
|
||||
if (args.source.type === "file") {
|
||||
const fileBuffer = await readFile(args.source.path);
|
||||
content = fileBuffer.toString("base64");
|
||||
isBase64 = true;
|
||||
} else if (args.source.type === "command_output") {
|
||||
const { stdout } = await execAsync(args.source.command, { encoding: 'buffer' });
|
||||
content = stdout.toString("base64");
|
||||
isBase64 = true;
|
||||
} else if (args.source.type === "text") {
|
||||
content = args.source.text;
|
||||
isBase64 = false;
|
||||
} else {
|
||||
content = "[Unknown source type]";
|
||||
isBase64 = false;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to resolve artifact source:", error);
|
||||
content = `[Error resolving source: ${error instanceof Error ? error.message : String(error)}]`;
|
||||
isBase64 = false;
|
||||
}
|
||||
|
||||
const artifact: ArtifactPayload = {
|
||||
type: args.type,
|
||||
id: artifactId,
|
||||
title: args.title,
|
||||
content,
|
||||
isBase64,
|
||||
};
|
||||
|
||||
// Broadcast artifact to client
|
||||
params.wsServer.broadcast({
|
||||
type: "artifact",
|
||||
payload: artifact,
|
||||
});
|
||||
|
||||
// Broadcast as activity log entry so it appears in the feed
|
||||
params.wsServer.broadcastActivityLog({
|
||||
id: artifactId,
|
||||
timestamp: new Date(),
|
||||
type: "system",
|
||||
content: `${args.type} artifact: ${args.title}`,
|
||||
metadata: { artifactId, artifactType: args.type },
|
||||
});
|
||||
}
|
||||
|
||||
// Broadcast tool call to WebSocket
|
||||
if (params.wsServer) {
|
||||
params.wsServer.broadcastActivityLog({
|
||||
@@ -240,3 +300,4 @@ export function getConversationStats(): {
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
55
packages/voice-assistant/src/server/agent/playwright-mcp.ts
Normal file
55
packages/voice-assistant/src/server/agent/playwright-mcp.ts
Normal file
@@ -0,0 +1,55 @@
|
||||
import { experimental_createMCPClient } from "ai";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
|
||||
|
||||
let playwrightMCPClient: Awaited<
|
||||
ReturnType<typeof experimental_createMCPClient>
|
||||
> | null = null;
|
||||
|
||||
/**
|
||||
* Get or create the Playwright MCP client (singleton)
|
||||
*/
|
||||
export async function getPlaywrightMCPClient() {
|
||||
if (playwrightMCPClient) {
|
||||
return playwrightMCPClient;
|
||||
}
|
||||
|
||||
const transport = new StdioClientTransport({
|
||||
command: "npx",
|
||||
args: ["@playwright/mcp", "--browser", "firefox"],
|
||||
});
|
||||
|
||||
playwrightMCPClient = await experimental_createMCPClient({
|
||||
transport,
|
||||
});
|
||||
|
||||
console.log("Playwright MCP client initialized with Firefox");
|
||||
|
||||
return playwrightMCPClient;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Playwright MCP tools in AI SDK format
|
||||
*/
|
||||
export async function getPlaywrightTools() {
|
||||
try {
|
||||
const client = await getPlaywrightMCPClient();
|
||||
const tools = await client.tools();
|
||||
console.log(`Loaded ${Object.keys(tools).length} Playwright MCP tools`);
|
||||
return tools;
|
||||
} catch (error) {
|
||||
console.error("Failed to initialize Playwright MCP tools:", error);
|
||||
// Return empty object to allow app to continue without Playwright
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup function to close the MCP client
|
||||
*/
|
||||
export async function closePlaywrightMCP() {
|
||||
if (playwrightMCPClient) {
|
||||
await playwrightMCPClient.close();
|
||||
playwrightMCPClient = null;
|
||||
console.log("Playwright MCP client closed");
|
||||
}
|
||||
}
|
||||
@@ -261,6 +261,14 @@ async function main() {
|
||||
// Transcribe audio using OpenAI Whisper
|
||||
const result = await transcribeAudio(audio, format);
|
||||
|
||||
// Check if transcription is empty or only whitespace
|
||||
const transcriptText = result.text.trim();
|
||||
if (!transcriptText) {
|
||||
console.log("[STT] Transcription is empty or silence, skipping LLM processing");
|
||||
// Return empty string without broadcasting or processing
|
||||
return "";
|
||||
}
|
||||
|
||||
// Broadcast transcription result as activity log
|
||||
wsServer.broadcastActivityLog({
|
||||
id: uuidv4(),
|
||||
|
||||
@@ -15,7 +15,8 @@ export interface ActivityLogEntry {
|
||||
|
||||
export interface WebSocketMessage {
|
||||
type: 'activity_log' | 'status' | 'ping' | 'pong' | 'user_message' | 'assistant_chunk'
|
||||
| 'audio_chunk' | 'audio_output' | 'recording_state' | 'transcription_result' | 'audio_played';
|
||||
| 'audio_chunk' | 'audio_output' | 'recording_state' | 'transcription_result' | 'audio_played'
|
||||
| 'artifact';
|
||||
payload: unknown;
|
||||
}
|
||||
|
||||
@@ -44,3 +45,11 @@ export interface AudioOutputPayload {
|
||||
export interface AudioPlayedPayload {
|
||||
id: string; // unique ID of the audio that finished playing
|
||||
}
|
||||
|
||||
export interface ArtifactPayload {
|
||||
type: 'markdown' | 'diff' | 'image' | 'code';
|
||||
id: string;
|
||||
title: string;
|
||||
content: string; // resolved content (base64 for file/command sources, plain text for text source)
|
||||
isBase64: boolean; // true if content is base64 encoded (from file or command_output)
|
||||
}
|
||||
|
||||
@@ -236,6 +236,27 @@
|
||||
font-size: 0.75rem;
|
||||
}
|
||||
|
||||
.log-entry.artifact {
|
||||
background-color: #1e3a5f;
|
||||
color: #93c5fd;
|
||||
border: 1px solid #3b82f6;
|
||||
}
|
||||
|
||||
.log-entry.artifact.clickable {
|
||||
cursor: pointer;
|
||||
transition: all 0.2s ease;
|
||||
}
|
||||
|
||||
.log-entry.artifact.clickable:hover {
|
||||
background-color: #2563eb;
|
||||
color: #fff;
|
||||
transform: translateX(4px);
|
||||
}
|
||||
|
||||
.log-entry.artifact.clickable:active {
|
||||
transform: translateX(2px);
|
||||
}
|
||||
|
||||
.log-entry.streaming {
|
||||
opacity: 0.8;
|
||||
}
|
||||
@@ -384,6 +405,65 @@
|
||||
animation: spin 0.8s linear infinite;
|
||||
}
|
||||
|
||||
/* Realtime mode button styles */
|
||||
.send-button.realtime-button {
|
||||
background-color: #8b5cf6;
|
||||
}
|
||||
|
||||
.send-button.realtime-button:hover:not(:disabled) {
|
||||
background-color: #7c3aed;
|
||||
}
|
||||
|
||||
.send-button.realtime-button.active {
|
||||
background-color: #10b981;
|
||||
animation: pulse-realtime 2s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.send-button.realtime-button.active:hover:not(:disabled) {
|
||||
background-color: #059669;
|
||||
}
|
||||
|
||||
.send-button.realtime-button.speech-detected {
|
||||
background-color: #ef4444;
|
||||
animation: pulse-speech 0.8s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.send-button.realtime-button.loading {
|
||||
background-color: #f59e0b;
|
||||
}
|
||||
|
||||
.send-button .loading-indicator {
|
||||
display: inline-block;
|
||||
width: 12px;
|
||||
height: 12px;
|
||||
border: 2px solid #fff;
|
||||
border-top-color: transparent;
|
||||
border-radius: 50%;
|
||||
animation: spin 0.8s linear infinite;
|
||||
}
|
||||
|
||||
@keyframes pulse-realtime {
|
||||
0%, 100% {
|
||||
opacity: 1;
|
||||
transform: scale(1);
|
||||
}
|
||||
50% {
|
||||
opacity: 0.7;
|
||||
transform: scale(1.05);
|
||||
}
|
||||
}
|
||||
|
||||
@keyframes pulse-speech {
|
||||
0%, 100% {
|
||||
opacity: 1;
|
||||
box-shadow: 0 0 0 0 rgba(239, 68, 68, 0.7);
|
||||
}
|
||||
50% {
|
||||
opacity: 0.9;
|
||||
box-shadow: 0 0 0 8px rgba(239, 68, 68, 0);
|
||||
}
|
||||
}
|
||||
|
||||
.test-controls {
|
||||
padding-top: 0.5rem;
|
||||
border-top: 1px solid #333;
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
import { useState, useEffect, useRef } from "react";
|
||||
import { Mic, Send } from "lucide-react";
|
||||
import { Mic, Send, Radio } from "lucide-react";
|
||||
import { useWebSocket } from "./hooks/useWebSocket";
|
||||
import { createAudioPlayer } from "./lib/audio-playback";
|
||||
import { createAudioRecorder, type AudioRecorder } from "./lib/audio-capture";
|
||||
import { createRealtimeVAD, float32ArrayToBlob, type RealtimeVAD } from "./lib/audio-realtime";
|
||||
import { ToolCallCard } from "./components/ToolCallCard";
|
||||
import { ArtifactDrawer, type Artifact } from "./components/ArtifactDrawer";
|
||||
import "./App.css";
|
||||
|
||||
type LogEntry =
|
||||
@@ -23,6 +25,14 @@ type LogEntry =
|
||||
result?: any;
|
||||
error?: any;
|
||||
status: "executing" | "completed" | "failed";
|
||||
}
|
||||
| {
|
||||
type: "artifact";
|
||||
id: string;
|
||||
timestamp: number;
|
||||
artifactId: string;
|
||||
artifactType: string;
|
||||
title: string;
|
||||
};
|
||||
|
||||
function App() {
|
||||
@@ -39,9 +49,15 @@ function App() {
|
||||
const [isProcessingAudio, setIsProcessingAudio] = useState(false);
|
||||
const [isPlayingAudio, setIsPlayingAudio] = useState(false);
|
||||
const [isRecording, setIsRecording] = useState(false);
|
||||
const [isRealtimeMode, setIsRealtimeMode] = useState(false);
|
||||
const [isSpeechDetected, setIsSpeechDetected] = useState(false);
|
||||
const [isVADLoading, setIsVADLoading] = useState(false);
|
||||
const [currentArtifact, setCurrentArtifact] = useState<Artifact | null>(null);
|
||||
const [artifacts, setArtifacts] = useState<Map<string, Artifact>>(new Map());
|
||||
const logEndRef = useRef<HTMLDivElement>(null);
|
||||
const audioPlayerRef = useRef(createAudioPlayer());
|
||||
const recorderRef = useRef<AudioRecorder | null>(null);
|
||||
const vadRef = useRef<RealtimeVAD | null>(null);
|
||||
const textareaRef = useRef<HTMLTextAreaElement>(null);
|
||||
|
||||
// WebSocket URL - use ws://localhost:3000/ws in dev, or construct from current host in prod
|
||||
@@ -53,6 +69,13 @@ function App() {
|
||||
|
||||
useEffect(() => {
|
||||
recorderRef.current = createAudioRecorder();
|
||||
|
||||
// Cleanup VAD on unmount
|
||||
return () => {
|
||||
if (vadRef.current) {
|
||||
vadRef.current.destroy();
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
@@ -186,6 +209,34 @@ function App() {
|
||||
}
|
||||
);
|
||||
|
||||
// Listen for artifacts
|
||||
const unsubArtifact = ws.on("artifact", (payload: unknown) => {
|
||||
const artifact = payload as Artifact;
|
||||
|
||||
// Add to artifacts map
|
||||
setArtifacts((prev) => {
|
||||
const newMap = new Map(prev);
|
||||
newMap.set(artifact.id, artifact);
|
||||
return newMap;
|
||||
});
|
||||
|
||||
// Add as clickable log entry
|
||||
setLogs((prev) => [
|
||||
...prev,
|
||||
{
|
||||
type: "artifact" as const,
|
||||
id: Date.now().toString(),
|
||||
timestamp: Date.now(),
|
||||
artifactId: artifact.id,
|
||||
artifactType: artifact.type,
|
||||
title: artifact.title,
|
||||
},
|
||||
]);
|
||||
|
||||
// Show drawer by default
|
||||
setCurrentArtifact(artifact);
|
||||
});
|
||||
|
||||
// Listen for audio output (TTS)
|
||||
const unsubAudioOutput = ws.on("audio_output", async (payload: unknown) => {
|
||||
const data = payload as { audio: string; format: string; id: string };
|
||||
@@ -228,6 +279,7 @@ function App() {
|
||||
unsubActivity();
|
||||
unsubChunk();
|
||||
unsubTranscription();
|
||||
unsubArtifact();
|
||||
unsubAudioOutput();
|
||||
};
|
||||
}, [ws]);
|
||||
@@ -336,6 +388,116 @@ function App() {
|
||||
}
|
||||
};
|
||||
|
||||
const handleCloseArtifact = () => {
|
||||
setCurrentArtifact(null);
|
||||
};
|
||||
|
||||
const handleOpenArtifact = (artifactId: string) => {
|
||||
const artifact = artifacts.get(artifactId);
|
||||
if (artifact) {
|
||||
setCurrentArtifact(artifact);
|
||||
}
|
||||
};
|
||||
|
||||
const handleToggleRealtimeMode = async () => {
|
||||
if (!ws.isConnected) return;
|
||||
|
||||
try {
|
||||
if (isRealtimeMode) {
|
||||
// Stop realtime mode
|
||||
if (vadRef.current) {
|
||||
vadRef.current.pause();
|
||||
}
|
||||
setIsRealtimeMode(false);
|
||||
setIsSpeechDetected(false);
|
||||
addLog("info", "Realtime mode stopped");
|
||||
} else {
|
||||
// Start realtime mode
|
||||
setIsVADLoading(true);
|
||||
addLog("info", "Starting realtime mode...");
|
||||
|
||||
// Create VAD instance if it doesn't exist
|
||||
if (!vadRef.current) {
|
||||
vadRef.current = createRealtimeVAD({
|
||||
onModelLoading: () => {
|
||||
console.log("[App] VAD model loading...");
|
||||
setIsVADLoading(true);
|
||||
},
|
||||
onModelLoaded: () => {
|
||||
console.log("[App] VAD model loaded");
|
||||
setIsVADLoading(false);
|
||||
},
|
||||
onSpeechStart: () => {
|
||||
console.log("[App] Speech detected!");
|
||||
setIsSpeechDetected(true);
|
||||
|
||||
// Interrupt playback when speech is detected
|
||||
audioPlayerRef.current.stop();
|
||||
setIsPlayingAudio(false);
|
||||
setCurrentAssistantMessage("");
|
||||
},
|
||||
onSpeechEnd: async (audioData: Float32Array) => {
|
||||
console.log("[App] Speech ended, processing...");
|
||||
setIsSpeechDetected(false);
|
||||
setIsProcessingAudio(true);
|
||||
|
||||
try {
|
||||
// Convert Float32Array to audio blob
|
||||
const audioBlob = float32ArrayToBlob(audioData, 16000);
|
||||
console.log(`[App] Converted to blob: ${audioBlob.size} bytes`);
|
||||
|
||||
// Send to server
|
||||
const arrayBuffer = await audioBlob.arrayBuffer();
|
||||
const base64Audio = btoa(
|
||||
new Uint8Array(arrayBuffer).reduce(
|
||||
(data, byte) => data + String.fromCharCode(byte),
|
||||
""
|
||||
)
|
||||
);
|
||||
|
||||
ws.send({
|
||||
type: "audio_chunk",
|
||||
payload: {
|
||||
audio: base64Audio,
|
||||
format: audioBlob.type,
|
||||
isLast: true,
|
||||
},
|
||||
});
|
||||
|
||||
console.log(`[App] Sent realtime audio to server`);
|
||||
} catch (error: any) {
|
||||
console.error("[App] Failed to process VAD audio:", error);
|
||||
addLog("error", `Failed to process audio: ${error.message}`);
|
||||
setIsProcessingAudio(false);
|
||||
}
|
||||
},
|
||||
onVADMisfire: () => {
|
||||
console.log("[App] VAD misfire");
|
||||
setIsSpeechDetected(false);
|
||||
},
|
||||
onError: (error: Error) => {
|
||||
console.error("[App] VAD error:", error);
|
||||
addLog("error", `VAD error: ${error.message}`);
|
||||
setIsVADLoading(false);
|
||||
setIsRealtimeMode(false);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Start VAD
|
||||
await vadRef.current.start();
|
||||
setIsRealtimeMode(true);
|
||||
setIsVADLoading(false);
|
||||
addLog("success", "Realtime mode started - speak anytime!");
|
||||
}
|
||||
} catch (error: any) {
|
||||
console.error("[App] Realtime mode error:", error);
|
||||
addLog("error", `Failed to toggle realtime mode: ${error.message}`);
|
||||
setIsRealtimeMode(false);
|
||||
setIsVADLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="app">
|
||||
<header className="header">
|
||||
@@ -356,17 +518,42 @@ function App() {
|
||||
<div className="chat-interface">
|
||||
<div className="activity-log">
|
||||
<div className="log-entries">
|
||||
{logs.map((log) =>
|
||||
log.type === "tool_call" ? (
|
||||
<ToolCallCard
|
||||
key={log.id}
|
||||
toolName={log.toolName}
|
||||
args={log.args}
|
||||
result={log.result}
|
||||
error={log.error}
|
||||
status={log.status}
|
||||
/>
|
||||
) : (
|
||||
{logs.map((log) => {
|
||||
if (log.type === "tool_call") {
|
||||
return (
|
||||
<ToolCallCard
|
||||
key={log.id}
|
||||
toolName={log.toolName}
|
||||
args={log.args}
|
||||
result={log.result}
|
||||
error={log.error}
|
||||
status={log.status}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
if (log.type === "artifact") {
|
||||
return (
|
||||
<div
|
||||
key={log.id}
|
||||
className="log-entry artifact clickable"
|
||||
onClick={() => handleOpenArtifact(log.artifactId)}
|
||||
role="button"
|
||||
tabIndex={0}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === "Enter" || e.key === " ") {
|
||||
handleOpenArtifact(log.artifactId);
|
||||
}
|
||||
}}
|
||||
>
|
||||
<span className="log-message">
|
||||
📋 {log.artifactType}: {log.title}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div key={log.id} className={`log-entry ${log.type}`}>
|
||||
<span className="log-message">{log.message}</span>
|
||||
{log.metadata && (
|
||||
@@ -376,8 +563,8 @@ function App() {
|
||||
</details>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
)}
|
||||
);
|
||||
})}
|
||||
{currentAssistantMessage && (
|
||||
<div className="log-entry assistant streaming">
|
||||
<span className="log-message">{currentAssistantMessage}</span>
|
||||
@@ -395,14 +582,27 @@ function App() {
|
||||
onChange={(e) => setUserInput(e.target.value)}
|
||||
onKeyDown={handleKeyDown}
|
||||
placeholder="Type a message or tap mic to talk..."
|
||||
disabled={!ws.isConnected || isRecording}
|
||||
disabled={!ws.isConnected || isRecording || isRealtimeMode}
|
||||
className="message-input"
|
||||
rows={1}
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleToggleRealtimeMode}
|
||||
disabled={!ws.isConnected || isRecording || isVADLoading}
|
||||
className={`send-button realtime-button ${isRealtimeMode ? 'active' : ''} ${isSpeechDetected ? 'speech-detected' : ''} ${isVADLoading ? 'loading' : ''}`}
|
||||
title={isRealtimeMode ? "Stop realtime mode" : "Start realtime mode"}
|
||||
>
|
||||
{isVADLoading ? (
|
||||
<span className="loading-indicator" />
|
||||
) : (
|
||||
<Radio size={20} />
|
||||
)}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleButtonClick}
|
||||
disabled={!ws.isConnected || isProcessingAudio}
|
||||
disabled={!ws.isConnected || isProcessingAudio || isRealtimeMode}
|
||||
className={`send-button ${isRecording ? 'recording' : ''} ${isProcessingAudio ? 'processing' : ''} ${userInput.trim() ? 'has-text' : ''}`}
|
||||
>
|
||||
{isRecording ? (
|
||||
@@ -418,6 +618,11 @@ function App() {
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
<ArtifactDrawer
|
||||
artifact={currentArtifact}
|
||||
onClose={handleCloseArtifact}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
268
packages/voice-assistant/src/ui/components/ArtifactDrawer.css
Normal file
268
packages/voice-assistant/src/ui/components/ArtifactDrawer.css
Normal file
@@ -0,0 +1,268 @@
|
||||
.artifact-drawer-overlay {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
bottom: 0;
|
||||
background-color: rgba(0, 0, 0, 0.5);
|
||||
z-index: 1000;
|
||||
animation: fadeIn 0.2s ease-out;
|
||||
}
|
||||
|
||||
@keyframes fadeIn {
|
||||
from {
|
||||
opacity: 0;
|
||||
}
|
||||
to {
|
||||
opacity: 1;
|
||||
}
|
||||
}
|
||||
|
||||
.artifact-drawer {
|
||||
position: fixed;
|
||||
bottom: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
max-height: 80vh;
|
||||
background: #1a1a1a;
|
||||
border-top-left-radius: 12px;
|
||||
border-top-right-radius: 12px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
animation: slideUp 0.3s ease-out;
|
||||
box-shadow: 0 -4px 20px rgba(0, 0, 0, 0.3);
|
||||
}
|
||||
|
||||
@keyframes slideUp {
|
||||
from {
|
||||
transform: translateY(100%);
|
||||
}
|
||||
to {
|
||||
transform: translateY(0);
|
||||
}
|
||||
}
|
||||
|
||||
.artifact-drawer-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
padding: 20px 24px;
|
||||
border-bottom: 1px solid #333;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.artifact-drawer-title {
|
||||
margin: 0;
|
||||
font-size: 1.25rem;
|
||||
font-weight: 600;
|
||||
color: #fff;
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
.artifact-drawer-header-right {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.artifact-drawer-type-badge {
|
||||
padding: 4px 12px;
|
||||
background-color: #2563eb;
|
||||
color: #fff;
|
||||
border-radius: 12px;
|
||||
font-size: 0.75rem;
|
||||
font-weight: 500;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.5px;
|
||||
}
|
||||
|
||||
.artifact-drawer-close-button {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
padding: 0;
|
||||
background: transparent;
|
||||
border: none;
|
||||
color: #9ca3af;
|
||||
cursor: pointer;
|
||||
border-radius: 6px;
|
||||
transition: all 0.2s ease;
|
||||
}
|
||||
|
||||
.artifact-drawer-close-button:hover {
|
||||
background-color: #2a2a2a;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.artifact-drawer-content {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
padding: 24px;
|
||||
min-height: 200px;
|
||||
}
|
||||
|
||||
.artifact-plan {
|
||||
color: #e5e5e5;
|
||||
line-height: 1.7;
|
||||
}
|
||||
|
||||
/* Markdown styles */
|
||||
.artifact-plan h1 {
|
||||
font-size: 1.75rem;
|
||||
font-weight: 700;
|
||||
margin-top: 0;
|
||||
margin-bottom: 1rem;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.artifact-plan h2 {
|
||||
font-size: 1.5rem;
|
||||
font-weight: 600;
|
||||
margin-top: 1.5rem;
|
||||
margin-bottom: 0.75rem;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.artifact-plan h3 {
|
||||
font-size: 1.25rem;
|
||||
font-weight: 600;
|
||||
margin-top: 1.25rem;
|
||||
margin-bottom: 0.5rem;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.artifact-plan h4,
|
||||
.artifact-plan h5,
|
||||
.artifact-plan h6 {
|
||||
font-size: 1rem;
|
||||
font-weight: 600;
|
||||
margin-top: 1rem;
|
||||
margin-bottom: 0.5rem;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.artifact-plan p {
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
|
||||
.artifact-plan ul,
|
||||
.artifact-plan ol {
|
||||
margin-bottom: 1rem;
|
||||
padding-left: 1.5rem;
|
||||
}
|
||||
|
||||
.artifact-plan li {
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
.artifact-plan code {
|
||||
background-color: #2a2a2a;
|
||||
padding: 2px 6px;
|
||||
border-radius: 4px;
|
||||
font-family: "Monaco", "Courier New", monospace;
|
||||
font-size: 0.9em;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.artifact-plan pre {
|
||||
background-color: #2a2a2a;
|
||||
padding: 12px;
|
||||
border-radius: 6px;
|
||||
overflow-x: auto;
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
|
||||
.artifact-plan pre code {
|
||||
background-color: transparent;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.artifact-plan blockquote {
|
||||
border-left: 4px solid #2563eb;
|
||||
padding-left: 1rem;
|
||||
margin-left: 0;
|
||||
margin-bottom: 1rem;
|
||||
color: #a3a3a3;
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.artifact-plan a {
|
||||
color: #3b82f6;
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
.artifact-plan a:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
/* Diff content styles */
|
||||
.artifact-diff {
|
||||
background-color: #2a2a2a;
|
||||
padding: 16px;
|
||||
border-radius: 6px;
|
||||
overflow-x: auto;
|
||||
color: #e5e5e5;
|
||||
font-family: "Monaco", "Courier New", monospace;
|
||||
font-size: 0.9em;
|
||||
line-height: 1.6;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.artifact-diff code {
|
||||
background-color: transparent;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
/* Code content styles */
|
||||
.artifact-code {
|
||||
background-color: #2a2a2a;
|
||||
padding: 16px;
|
||||
border-radius: 6px;
|
||||
overflow-x: auto;
|
||||
color: #e5e5e5;
|
||||
font-family: "Monaco", "Courier New", monospace;
|
||||
font-size: 0.9em;
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
.artifact-code code {
|
||||
background-color: transparent;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
/* Image content styles */
|
||||
.artifact-image {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
padding: 16px;
|
||||
}
|
||||
|
||||
.artifact-image img {
|
||||
max-width: 100%;
|
||||
max-height: 70vh;
|
||||
border-radius: 8px;
|
||||
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3);
|
||||
}
|
||||
|
||||
/* Mobile responsiveness */
|
||||
@media (max-width: 768px) {
|
||||
.artifact-drawer {
|
||||
max-height: 90vh;
|
||||
}
|
||||
|
||||
.artifact-drawer-header {
|
||||
padding: 16px 20px;
|
||||
}
|
||||
|
||||
.artifact-drawer-content {
|
||||
padding: 20px;
|
||||
}
|
||||
|
||||
.artifact-drawer-title {
|
||||
font-size: 1.1rem;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
import { useEffect, useRef } from "react";
|
||||
import ReactMarkdown from "react-markdown";
|
||||
import { X } from "lucide-react";
|
||||
import "./ArtifactDrawer.css";
|
||||
|
||||
export type Artifact = {
|
||||
type: "markdown" | "diff" | "image" | "code";
|
||||
id: string;
|
||||
title: string;
|
||||
content: string;
|
||||
isBase64: boolean;
|
||||
};
|
||||
|
||||
interface ArtifactDrawerProps {
|
||||
artifact: Artifact | null;
|
||||
onClose: () => void;
|
||||
}
|
||||
|
||||
export function ArtifactDrawer({ artifact, onClose }: ArtifactDrawerProps) {
|
||||
const drawerRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (artifact) {
|
||||
// Prevent body scroll when drawer is open
|
||||
document.body.style.overflow = "hidden";
|
||||
} else {
|
||||
document.body.style.overflow = "";
|
||||
}
|
||||
|
||||
return () => {
|
||||
document.body.style.overflow = "";
|
||||
};
|
||||
}, [artifact]);
|
||||
|
||||
if (!artifact) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="artifact-drawer-overlay" onClick={onClose}>
|
||||
<div ref={drawerRef} className="artifact-drawer" onClick={(e) => e.stopPropagation()}>
|
||||
<div className="artifact-drawer-header">
|
||||
<h2 className="artifact-drawer-title">
|
||||
{artifact.title}
|
||||
</h2>
|
||||
<div className="artifact-drawer-header-right">
|
||||
<div className="artifact-drawer-type-badge">
|
||||
{artifact.type}
|
||||
</div>
|
||||
<button
|
||||
className="artifact-drawer-close-button"
|
||||
onClick={onClose}
|
||||
type="button"
|
||||
title="Close"
|
||||
>
|
||||
<X size={20} />
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="artifact-drawer-content">
|
||||
{artifact.type === "markdown" && (
|
||||
<div className="artifact-plan">
|
||||
<ReactMarkdown>
|
||||
{artifact.isBase64 ? atob(artifact.content) : artifact.content}
|
||||
</ReactMarkdown>
|
||||
</div>
|
||||
)}
|
||||
{artifact.type === "diff" && (
|
||||
<pre className="artifact-diff">
|
||||
<code>{artifact.isBase64 ? atob(artifact.content) : artifact.content}</code>
|
||||
</pre>
|
||||
)}
|
||||
{artifact.type === "code" && (
|
||||
<pre className="artifact-code">
|
||||
<code>{artifact.isBase64 ? atob(artifact.content) : artifact.content}</code>
|
||||
</pre>
|
||||
)}
|
||||
{artifact.type === "image" && (
|
||||
<div className="artifact-image">
|
||||
<img
|
||||
src={`data:image/png;base64,${artifact.content}`}
|
||||
alt={artifact.title}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
176
packages/voice-assistant/src/ui/lib/audio-realtime.ts
Normal file
176
packages/voice-assistant/src/ui/lib/audio-realtime.ts
Normal file
@@ -0,0 +1,176 @@
|
||||
import { MicVAD } from "@ricky0123/vad-web";
|
||||
|
||||
export interface RealtimeVADConfig {
|
||||
onSpeechStart?: () => void;
|
||||
onSpeechEnd?: (audio: Float32Array) => void;
|
||||
onVADMisfire?: () => void;
|
||||
onModelLoading?: () => void;
|
||||
onModelLoaded?: () => void;
|
||||
onError?: (error: Error) => void;
|
||||
positiveSpeechThreshold?: number;
|
||||
negativeSpeechThreshold?: number;
|
||||
redemptionFrames?: number;
|
||||
frameSamples?: number;
|
||||
preSpeechPadFrames?: number;
|
||||
minSpeechFrames?: number;
|
||||
}
|
||||
|
||||
export interface RealtimeVAD {
|
||||
start(): Promise<void>;
|
||||
pause(): void;
|
||||
destroy(): void;
|
||||
isListening(): boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a realtime VAD (Voice Activity Detection) instance using @ricky0123/vad-web
|
||||
* Continuously listens for speech and invokes callbacks when speech is detected
|
||||
*/
|
||||
export function createRealtimeVAD(config: RealtimeVADConfig): RealtimeVAD {
|
||||
let vad: Awaited<ReturnType<typeof MicVAD.new>> | null = null;
|
||||
let isActive = false;
|
||||
|
||||
async function start(): Promise<void> {
|
||||
// If VAD already exists and is paused, just resume it
|
||||
if (vad && !isActive) {
|
||||
vad.start();
|
||||
isActive = true;
|
||||
console.log("[RealtimeVAD] VAD resumed");
|
||||
return;
|
||||
}
|
||||
|
||||
if (isActive) {
|
||||
throw new Error("VAD is already active");
|
||||
}
|
||||
|
||||
// Create new VAD instance
|
||||
try {
|
||||
config.onModelLoading?.();
|
||||
|
||||
vad = await MicVAD.new({
|
||||
onSpeechStart: () => {
|
||||
console.log("[RealtimeVAD] Speech start detected");
|
||||
config.onSpeechStart?.();
|
||||
},
|
||||
onSpeechEnd: (audio: Float32Array) => {
|
||||
console.log("[RealtimeVAD] Speech end detected", {
|
||||
samples: audio.length,
|
||||
durationSeconds: audio.length / 16000,
|
||||
});
|
||||
config.onSpeechEnd?.(audio);
|
||||
},
|
||||
onVADMisfire: () => {
|
||||
console.log("[RealtimeVAD] VAD misfire (false positive)");
|
||||
config.onVADMisfire?.();
|
||||
},
|
||||
positiveSpeechThreshold: config.positiveSpeechThreshold ?? 0.5,
|
||||
negativeSpeechThreshold: config.negativeSpeechThreshold ?? 0.35,
|
||||
redemptionMs: config.redemptionFrames ? config.redemptionFrames * 16 : 128,
|
||||
preSpeechPadMs: config.preSpeechPadFrames ? config.preSpeechPadFrames * 16 : 16,
|
||||
minSpeechMs: config.minSpeechFrames ? config.minSpeechFrames * 16 : 48,
|
||||
});
|
||||
|
||||
isActive = true;
|
||||
config.onModelLoaded?.();
|
||||
console.log("[RealtimeVAD] VAD created and started successfully");
|
||||
} catch (error) {
|
||||
console.error("[RealtimeVAD] Failed to start VAD:", error);
|
||||
const err =
|
||||
error instanceof Error ? error : new Error(String(error));
|
||||
config.onError?.(err);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
function pause(): void {
|
||||
if (vad && isActive) {
|
||||
vad.pause();
|
||||
isActive = false;
|
||||
console.log("[RealtimeVAD] VAD paused");
|
||||
}
|
||||
}
|
||||
|
||||
function destroy(): void {
|
||||
if (vad) {
|
||||
vad.destroy();
|
||||
vad = null;
|
||||
isActive = false;
|
||||
console.log("[RealtimeVAD] VAD destroyed");
|
||||
}
|
||||
}
|
||||
|
||||
function isListening(): boolean {
|
||||
return isActive;
|
||||
}
|
||||
|
||||
return {
|
||||
start,
|
||||
pause,
|
||||
destroy,
|
||||
isListening,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert Float32Array PCM audio to audio blob
|
||||
* VAD returns Float32Array at 16kHz sample rate
|
||||
*/
|
||||
export function float32ArrayToBlob(
|
||||
audio: Float32Array,
|
||||
sampleRate: number = 16000
|
||||
): Blob {
|
||||
// Convert Float32Array to 16-bit PCM
|
||||
const pcm = new Int16Array(audio.length);
|
||||
for (let i = 0; i < audio.length; i++) {
|
||||
const s = Math.max(-1, Math.min(1, audio[i]));
|
||||
pcm[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
|
||||
}
|
||||
|
||||
// Create WAV header
|
||||
const wavHeader = createWavHeader(pcm.length * 2, sampleRate);
|
||||
|
||||
// Combine header and data
|
||||
const wavBytes = new Uint8Array(wavHeader.length + pcm.length * 2);
|
||||
wavBytes.set(wavHeader, 0);
|
||||
wavBytes.set(new Uint8Array(pcm.buffer), wavHeader.length);
|
||||
|
||||
return new Blob([wavBytes], { type: "audio/wav" });
|
||||
}
|
||||
|
||||
/**
|
||||
* Create WAV file header
|
||||
*/
|
||||
function createWavHeader(dataLength: number, sampleRate: number): Uint8Array {
|
||||
const header = new ArrayBuffer(44);
|
||||
const view = new DataView(header);
|
||||
|
||||
// "RIFF" chunk descriptor
|
||||
writeString(view, 0, "RIFF");
|
||||
view.setUint32(4, 36 + dataLength, true);
|
||||
writeString(view, 8, "WAVE");
|
||||
|
||||
// "fmt " sub-chunk
|
||||
writeString(view, 12, "fmt ");
|
||||
view.setUint32(16, 16, true); // SubChunk1Size (16 for PCM)
|
||||
view.setUint16(20, 1, true); // AudioFormat (1 for PCM)
|
||||
view.setUint16(22, 1, true); // NumChannels (1 = mono)
|
||||
view.setUint32(24, sampleRate, true); // SampleRate
|
||||
view.setUint32(28, sampleRate * 2, true); // ByteRate
|
||||
view.setUint16(32, 2, true); // BlockAlign
|
||||
view.setUint16(34, 16, true); // BitsPerSample
|
||||
|
||||
// "data" sub-chunk
|
||||
writeString(view, 36, "data");
|
||||
view.setUint32(40, dataLength, true);
|
||||
|
||||
return new Uint8Array(header);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write string to DataView
|
||||
*/
|
||||
function writeString(view: DataView, offset: number, str: string): void {
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
view.setUint8(offset + i, str.charCodeAt(i));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user