refactor: rebrand project from voice-dev to paseo across all packages and configs

This commit is contained in:
Mohamed Boudra
2025-11-15 20:45:09 +01:00
parent b39497d08f
commit b046dae799
16 changed files with 298 additions and 205 deletions

24
package-lock.json generated
View File

@@ -1,11 +1,11 @@
{ {
"name": "voice-dev", "name": "paseo",
"version": "1.0.0", "version": "1.0.0",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "voice-dev", "name": "paseo",
"version": "1.0.0", "version": "1.0.0",
"license": "MIT", "license": "MIT",
"workspaces": [ "workspaces": [
@@ -5624,6 +5624,14 @@
"node": ">=8.0.0" "node": ">=8.0.0"
} }
}, },
"node_modules/@paseo/app": {
"resolved": "packages/app",
"link": true
},
"node_modules/@paseo/server": {
"resolved": "packages/server",
"link": true
},
"node_modules/@pkgjs/parseargs": { "node_modules/@pkgjs/parseargs": {
"version": "0.11.0", "version": "0.11.0",
"resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
@@ -7970,14 +7978,6 @@
"url": "https://opencollective.com/vitest" "url": "https://opencollective.com/vitest"
} }
}, },
"node_modules/@voice-dev/app": {
"resolved": "packages/app",
"link": true
},
"node_modules/@voice-dev/server": {
"resolved": "packages/server",
"link": true
},
"node_modules/@xmldom/xmldom": { "node_modules/@xmldom/xmldom": {
"version": "0.8.11", "version": "0.8.11",
"resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.11.tgz", "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.11.tgz",
@@ -20847,7 +20847,7 @@
} }
}, },
"packages/app": { "packages/app": {
"name": "@voice-dev/app", "name": "@paseo/app",
"version": "1.0.0", "version": "1.0.0",
"dependencies": { "dependencies": {
"@boudra/expo-two-way-audio": "^0.1.3", "@boudra/expo-two-way-audio": "^0.1.3",
@@ -20919,7 +20919,7 @@
} }
}, },
"packages/server": { "packages/server": {
"name": "@voice-dev/server", "name": "@paseo/server",
"version": "0.1.0", "version": "0.1.0",
"dependencies": { "dependencies": {
"@ai-sdk/openai": "^2.0.52", "@ai-sdk/openai": "^2.0.52",

View File

@@ -1,5 +1,5 @@
{ {
"name": "voice-dev", "name": "paseo",
"version": "1.0.0", "version": "1.0.0",
"private": true, "private": true,
"workspaces": [ "workspaces": [
@@ -7,20 +7,20 @@
"packages/app" "packages/app"
], ],
"scripts": { "scripts": {
"dev": "npm run dev --workspace=@voice-dev/server", "dev": "npm run dev --workspace=@paseo/server",
"dev:app": "npm run start --workspace=@voice-dev/app", "dev:app": "npm run start --workspace=@paseo/app",
"build": "npm run build --workspaces --if-present", "build": "npm run build --workspaces --if-present",
"typecheck": "npm run typecheck --workspaces --if-present", "typecheck": "npm run typecheck --workspaces --if-present",
"test": "npm run test --workspaces --if-present", "test": "npm run test --workspaces --if-present",
"start": "npm run start --workspace=@voice-dev/server", "start": "npm run start --workspace=@paseo/server",
"android": "npm run android --workspace=@voice-dev/app", "android": "npm run android --workspace=@paseo/app",
"ios": "npm run ios --workspace=@voice-dev/app", "ios": "npm run ios --workspace=@paseo/app",
"web": "npm run web --workspace=@voice-dev/app" "web": "npm run web --workspace=@paseo/app"
}, },
"devDependencies": { "devDependencies": {
"typescript": "^5.9.3" "typescript": "^5.9.3"
}, },
"description": "Voice-controlled development environment with OpenAI Realtime API", "description": "Paseo: voice-controlled development environment with OpenAI Realtime API",
"keywords": [ "keywords": [
"openai", "openai",
"realtime", "realtime",

View File

@@ -1,11 +1,11 @@
{ {
"expo": { "expo": {
"name": "Voice Dev", "name": "Paseo",
"slug": "voice-dev", "slug": "paseo",
"version": "1.0.0", "version": "1.0.0",
"orientation": "portrait", "orientation": "portrait",
"icon": "./assets/images/icon.png", "icon": "./assets/images/icon.png",
"scheme": "voicedev", "scheme": "paseo",
"userInterfaceStyle": "automatic", "userInterfaceStyle": "automatic",
"newArchEnabled": true, "newArchEnabled": true,
"runtimeVersion": { "runtimeVersion": {
@@ -19,7 +19,7 @@
"infoPlist": { "infoPlist": {
"NSMicrophoneUsageDescription": "This app needs access to the microphone for voice commands." "NSMicrophoneUsageDescription": "This app needs access to the microphone for voice commands."
}, },
"bundleIdentifier": "com.moboudra.voicedev" "bundleIdentifier": "com.moboudra.paseo"
}, },
"android": { "android": {
"adaptiveIcon": { "adaptiveIcon": {
@@ -34,7 +34,7 @@
"android.permission.RECORD_AUDIO", "android.permission.RECORD_AUDIO",
"android.permission.MODIFY_AUDIO_SETTINGS" "android.permission.MODIFY_AUDIO_SETTINGS"
], ],
"package": "com.moboudra.voicedev" "package": "com.moboudra.paseo"
}, },
"web": { "web": {
"output": "static", "output": "static",

View File

@@ -1,5 +1,5 @@
{ {
"name": "@voice-dev/app", "name": "@paseo/app",
"main": "index.ts", "main": "index.ts",
"version": "1.0.0", "version": "1.0.0",
"scripts": { "scripts": {

View File

@@ -129,7 +129,7 @@ export function AgentList({ agents }: AgentListProps) {
{actionAgent?.title || "Delete agent"} {actionAgent?.title || "Delete agent"}
</Text> </Text>
<Text style={styles.sheetSubtitle}> <Text style={styles.sheetSubtitle}>
Removing this agent only deletes it from Voice Dev. Claude/Codex keeps the original project. Removing this agent only deletes it from Paseo. Claude/Codex keeps the original project.
</Text> </Text>
<Pressable <Pressable
style={[styles.sheetButton, styles.sheetDeleteButton]} style={[styles.sheetButton, styles.sheetDeleteButton]}

View File

@@ -15,7 +15,7 @@ import AsyncStorage from "@react-native-async-storage/async-storage";
import { StyleSheet } from "react-native-unistyles"; import { StyleSheet } from "react-native-unistyles";
import type { UseWebSocketReturn } from "../hooks/use-websocket"; import type { UseWebSocketReturn } from "../hooks/use-websocket";
const STORAGE_KEY = "@voice-dev:conversation-id"; const STORAGE_KEY = "@paseo:conversation-id";
interface Conversation { interface Conversation {
id: string; id: string;

View File

@@ -1,7 +1,7 @@
import { useState, useEffect, useCallback } from 'react'; import { useState, useEffect, useCallback } from 'react';
import AsyncStorage from '@react-native-async-storage/async-storage'; import AsyncStorage from '@react-native-async-storage/async-storage';
const STORAGE_KEY = '@voice-dev:recent-paths'; const STORAGE_KEY = '@paseo:recent-paths';
const MAX_RECENT_PATHS = 3; const MAX_RECENT_PATHS = 3;
export interface UseRecentPathsReturn { export interface UseRecentPathsReturn {

View File

@@ -1,7 +1,7 @@
import { useState, useEffect, useCallback } from 'react'; import { useState, useEffect, useCallback } from 'react';
import AsyncStorage from '@react-native-async-storage/async-storage'; import AsyncStorage from '@react-native-async-storage/async-storage';
const STORAGE_KEY = '@voice-dev:settings'; const STORAGE_KEY = '@paseo:settings';
export interface Settings { export interface Settings {
serverUrl: string; serverUrl: string;

View File

@@ -9,7 +9,7 @@ import {
} from "@boudra/expo-two-way-audio"; } from "@boudra/expo-two-way-audio";
export interface SpeechmaticsAudioConfig { export interface SpeechmaticsAudioConfig {
onAudioSegment?: (audioData: string) => void; onAudioSegment?: (segment: { audioData: string; isLast: boolean }) => void;
onSpeechStart?: () => void; onSpeechStart?: () => void;
onSpeechEnd?: () => void; onSpeechEnd?: () => void;
onError?: (error: Error) => void; onError?: (error: Error) => void;
@@ -31,79 +31,6 @@ export interface SpeechmaticsAudio {
segmentDuration: number; segmentDuration: number;
} }
/**
* Convert raw PCM data to WAV format by adding WAV headers
*/
function pcmToWav(
pcmBase64: string,
sampleRate: number = 16000,
channels: number = 1,
bitsPerSample: number = 16
): string {
const pcmBinary = atob(pcmBase64);
const pcmLength = pcmBinary.length;
const byteRate = sampleRate * channels * (bitsPerSample / 8);
const blockAlign = channels * (bitsPerSample / 8);
const dataSize = pcmLength;
const fileSize = 44 + dataSize;
const wavBuffer = new Uint8Array(fileSize);
const view = new DataView(wavBuffer.buffer);
let offset = 0;
// RIFF chunk descriptor
writeString(view, offset, "RIFF");
offset += 4;
view.setUint32(offset, fileSize - 8, true);
offset += 4;
writeString(view, offset, "WAVE");
offset += 4;
// fmt sub-chunk
writeString(view, offset, "fmt ");
offset += 4;
view.setUint32(offset, 16, true);
offset += 4;
view.setUint16(offset, 1, true);
offset += 2;
view.setUint16(offset, channels, true);
offset += 2;
view.setUint32(offset, sampleRate, true);
offset += 4;
view.setUint32(offset, byteRate, true);
offset += 4;
view.setUint16(offset, blockAlign, true);
offset += 2;
view.setUint16(offset, bitsPerSample, true);
offset += 2;
// data sub-chunk
writeString(view, offset, "data");
offset += 4;
view.setUint32(offset, dataSize, true);
offset += 4;
// Copy PCM data
for (let i = 0; i < pcmLength; i++) {
wavBuffer[offset + i] = pcmBinary.charCodeAt(i);
}
// Convert to base64
let binary = "";
for (let i = 0; i < wavBuffer.length; i++) {
binary += String.fromCharCode(wavBuffer[i]);
}
return btoa(binary);
}
function writeString(view: DataView, offset: number, str: string): void {
for (let i = 0; i < str.length; i++) {
view.setUint8(offset + i, str.charCodeAt(i));
}
}
function uint8ArrayToBase64(bytes: Uint8Array): string { function uint8ArrayToBase64(bytes: Uint8Array): string {
let binary = ""; let binary = "";
for (let i = 0; i < bytes.length; i++) { for (let i = 0; i < bytes.length; i++) {
@@ -143,6 +70,38 @@ export function useSpeechmaticsAudio(
const speechDetectionStartRef = useRef<number | null>(null); const speechDetectionStartRef = useRef<number | null>(null);
const speechConfirmedRef = useRef(false); const speechConfirmedRef = useRef(false);
const detectionSilenceStartRef = useRef<number | null>(null); const detectionSilenceStartRef = useRef<number | null>(null);
const bufferedBytesRef = useRef(0);
const PCM_SAMPLE_RATE = 16000;
const PCM_CHANNELS = 1;
const PCM_BITS_PER_SAMPLE = 16;
const PCM_BYTES_PER_MS =
(PCM_SAMPLE_RATE * PCM_CHANNELS * (PCM_BITS_PER_SAMPLE / 8)) / 1000;
const MIN_CHUNK_DURATION_MS = 1000;
const MIN_CHUNK_BYTES = Math.round(
PCM_BYTES_PER_MS * MIN_CHUNK_DURATION_MS
);
const flushBufferedAudio = useCallback(
(isLast: boolean) => {
if (audioBufferRef.current.length === 0) {
bufferedBytesRef.current = 0;
return;
}
const combinedBinary = concatenateUint8Arrays(audioBufferRef.current);
const pcmBase64 = uint8ArrayToBase64(combinedBinary);
config.onAudioSegment?.({
audioData: pcmBase64,
isLast,
});
audioBufferRef.current = [];
bufferedBytesRef.current = 0;
},
[config]
);
const VOLUME_THRESHOLD = config.volumeThreshold; const VOLUME_THRESHOLD = config.volumeThreshold;
const SILENCE_DURATION_MS = config.silenceDuration; const SILENCE_DURATION_MS = config.silenceDuration;
@@ -178,9 +137,17 @@ export function useSpeechmaticsAudio(
// Start buffering from first spike to capture beginning of speech // Start buffering from first spike to capture beginning of speech
if (speechDetectionStartRef.current !== null || isSpeakingRef.current) { if (speechDetectionStartRef.current !== null || isSpeakingRef.current) {
audioBufferRef.current.push(pcmData); audioBufferRef.current.push(pcmData);
bufferedBytesRef.current += pcmData.length;
if (
speechConfirmedRef.current &&
bufferedBytesRef.current >= MIN_CHUNK_BYTES
) {
flushBufferedAudio(false);
}
} }
}, },
[isActive, isMuted] [isActive, isMuted, flushBufferedAudio, MIN_CHUNK_BYTES]
) )
); );
@@ -215,6 +182,7 @@ export function useSpeechmaticsAudio(
speechDetectionStartRef.current = Date.now(); speechDetectionStartRef.current = Date.now();
detectionSilenceStartRef.current = null; detectionSilenceStartRef.current = null;
audioBufferRef.current = []; audioBufferRef.current = [];
bufferedBytesRef.current = 0;
setIsDetecting(true); setIsDetecting(true);
} else { } else {
// Volume is back above threshold - reset grace period // Volume is back above threshold - reset grace period
@@ -264,6 +232,7 @@ export function useSpeechmaticsAudio(
speechDetectionStartRef.current = null; speechDetectionStartRef.current = null;
detectionSilenceStartRef.current = null; detectionSilenceStartRef.current = null;
audioBufferRef.current = []; audioBufferRef.current = [];
bufferedBytesRef.current = 0;
setIsDetecting(false); setIsDetecting(false);
} }
} }
@@ -292,15 +261,7 @@ export function useSpeechmaticsAudio(
config.onSpeechEnd?.(); config.onSpeechEnd?.();
// Send buffered audio segment // Send buffered audio segment
if (audioBufferRef.current.length > 0) { flushBufferedAudio(true);
const combinedBinary = concatenateUint8Arrays(
audioBufferRef.current
);
const combinedPcmBase64 = uint8ArrayToBase64(combinedBinary);
const wavAudio = pcmToWav(combinedPcmBase64, 16000, 1, 16);
config.onAudioSegment?.(wavAudio);
audioBufferRef.current = [];
}
} }
} }
} }
@@ -313,6 +274,7 @@ export function useSpeechmaticsAudio(
SPEECH_CONFIRMATION_MS, SPEECH_CONFIRMATION_MS,
DETECTION_GRACE_PERIOD_MS, DETECTION_GRACE_PERIOD_MS,
config, config,
flushBufferedAudio,
] ]
) )
); );
@@ -365,6 +327,7 @@ export function useSpeechmaticsAudio(
// Reset state // Reset state
audioBufferRef.current = []; audioBufferRef.current = [];
bufferedBytesRef.current = 0;
isSpeakingRef.current = false; isSpeakingRef.current = false;
speechConfirmedRef.current = false; speechConfirmedRef.current = false;
speechDetectionStartRef.current = null; speechDetectionStartRef.current = null;
@@ -387,6 +350,7 @@ export function useSpeechmaticsAudio(
if (newMuted) { if (newMuted) {
// Clear any ongoing speech detection/speaking state // Clear any ongoing speech detection/speaking state
audioBufferRef.current = []; audioBufferRef.current = [];
bufferedBytesRef.current = 0;
isSpeakingRef.current = false; isSpeakingRef.current = false;
speechConfirmedRef.current = false; speechConfirmedRef.current = false;
speechDetectionStartRef.current = null; speechDetectionStartRef.current = null;

View File

@@ -441,14 +441,14 @@ function testToolCallParsedPayloadHydration() {
tool_use_id: commandCallId, tool_use_id: commandCallId,
result: { result: {
command: 'pwd', command: 'pwd',
output: '/Users/dev/voice-dev', output: '/Users/dev/paseo',
}, },
metadata: { exit_code: 0 }, metadata: { exit_code: 0 },
}, },
output: { output: {
result: { result: {
command: 'pwd', command: 'pwd',
output: '/Users/dev/voice-dev', output: '/Users/dev/paseo',
}, },
metadata: { exit_code: 0 }, metadata: { exit_code: 0 },
}, },
@@ -477,7 +477,7 @@ function testToolCallParsedPayloadHydration() {
const commandPass = Boolean( const commandPass = Boolean(
commandEntry?.payload.data.parsedCommand && commandEntry?.payload.data.parsedCommand &&
commandEntry.payload.data.parsedCommand.command === 'pwd' && commandEntry.payload.data.parsedCommand.command === 'pwd' &&
commandEntry.payload.data.parsedCommand.output?.includes('/voice-dev') commandEntry.payload.data.parsedCommand.output?.includes('/paseo')
); );
assert.ok(readPass, 'Read payload should persist across hydration'); assert.ok(readPass, 'Read payload should persist across hydration');

View File

@@ -285,11 +285,11 @@ When Claude Code presents a plan in plan mode, forward it to user's screen:
```javascript ```javascript
// User mentions project // User mentions project
User: "Create a terminal for the web project" User: "Create a terminal for the web project"
create-terminal(name="web", workingDirectory="~/dev/voice-dev/packages/web") create-terminal(name="web", workingDirectory="~/dev/paseo/packages/web")
// User says "another terminal here" // User says "another terminal here"
// Look at current terminal's working directory, use same path // Look at current terminal's working directory, use same path
create-terminal(name="tests", workingDirectory="~/dev/voice-dev/packages/web") create-terminal(name="tests", workingDirectory="~/dev/paseo/packages/web")
// No context - list terminals first to see what they're working on // No context - list terminals first to see what they're working on
User: "Create a terminal" User: "Create a terminal"
@@ -430,7 +430,7 @@ Multi-step process:
```javascript ```javascript
// Step 1 // Step 1
create_terminal((name = "fix-auth"), (workingDirectory = "~/dev/voice-dev")); create_terminal((name = "fix-auth"), (workingDirectory = "~/dev/paseo"));
// Step 2 // Step 2
send_text( send_text(
@@ -455,7 +455,7 @@ send_text((terminalName = "fix-auth"), (text = "claude"), (pressEnter = true));
**Terminal naming:** **Terminal naming:**
- No worktree: Use project name ("faro", "voice-dev") - No worktree: Use project name ("faro", "paseo")
- With worktree: Use worktree name ("fix-auth", "feature-export") - With worktree: Use worktree name ("fix-auth", "feature-export")
**When user says "launch Claude in [project]":** **When user says "launch Claude in [project]":**
@@ -494,9 +494,9 @@ Already authenticated. Use for:
All projects in `~/dev`: All projects in `~/dev`:
**voice-dev** **paseo**
- Location: `~/dev/voice-dev` - Location: `~/dev/paseo`
- Packages: `voice-assistant` - Packages: `voice-assistant`
**Faro** (Autonomous Competitive Intelligence) **Faro** (Autonomous Competitive Intelligence)

View File

@@ -191,12 +191,12 @@ We only have two coding agents. Do not call tools to discover them—treat this
### Creating Agents ### Creating Agents
**Confirm creation only when intent is unclear.** If the user gives a direct imperative (“spin up a new planner agent in voice-dev”), acknowledge and create immediately; otherwise, ask. **Confirm creation only when intent is unclear.** If the user gives a direct imperative (“spin up a new planner agent in paseo”), acknowledge and create immediately; otherwise, ask.
```javascript ```javascript
// Claude Code with planning // Claude Code with planning
create_coding_agent({ create_coding_agent({
cwd: "~/dev/voice-dev", cwd: "~/dev/paseo",
agentType: "claude", agentType: "claude",
initialPrompt: "add dark mode toggle to settings page", initialPrompt: "add dark mode toggle to settings page",
initialMode: "plan" initialMode: "plan"
@@ -204,7 +204,7 @@ create_coding_agent({
// Codex for quick edits // Codex for quick edits
create_coding_agent({ create_coding_agent({
cwd: "~/dev/voice-dev", cwd: "~/dev/paseo",
agentType: "codex", agentType: "codex",
initialPrompt: "clean up the logging", initialPrompt: "clean up the logging",
initialMode: "auto" initialMode: "auto"
@@ -328,8 +328,8 @@ Already authenticated. Use for:
All projects in `~/dev`: All projects in `~/dev`:
**voice-dev** **paseo**
- Location: `~/dev/voice-dev` - Location: `~/dev/paseo`
- Packages: `voice-assistant` - Packages: `voice-assistant`
**Faro** (Autonomous Competitive Intelligence) **Faro** (Autonomous Competitive Intelligence)

View File

@@ -1,7 +1,7 @@
{ {
"name": "@voice-dev/server", "name": "@paseo/server",
"version": "0.1.0", "version": "0.1.0",
"description": "Voice assistant backend server", "description": "Paseo backend server",
"type": "module", "type": "module",
"scripts": { "scripts": {
"dev": "NODE_ENV=development node --watch --import tsx src/server/index.ts", "dev": "NODE_ENV=development node --watch --import tsx src/server/index.ts",

View File

@@ -22,7 +22,7 @@ async function getTerminalMcpClient(): Promise<McpClient> {
} }
// Create Terminal MCP server // Create Terminal MCP server
const server = await createTerminalMcpServer({ sessionName: "__voice-dev" }); const server = await createTerminalMcpServer({ sessionName: "__paseo" });
// Create linked transport pair // Create linked transport pair
const [clientTransport, serverTransport] = const [clientTransport, serverTransport] =

View File

@@ -57,6 +57,16 @@ const ACTIVE_TITLE_GENERATIONS = new Set<string>();
type ProcessingPhase = "idle" | "transcribing" | "llm"; type ProcessingPhase = "idle" | "transcribing" | "llm";
const PCM_SAMPLE_RATE = 16000;
const PCM_CHANNELS = 1;
const PCM_BITS_PER_SAMPLE = 16;
const PCM_BYTES_PER_MS =
(PCM_SAMPLE_RATE * PCM_CHANNELS * (PCM_BITS_PER_SAMPLE / 8)) / 1000;
const MIN_STREAMING_SEGMENT_DURATION_MS = 1000;
const MIN_STREAMING_SEGMENT_BYTES = Math.round(
PCM_BYTES_PER_MS * MIN_STREAMING_SEGMENT_DURATION_MS
);
/** /**
* Type for present_artifact tool arguments * Type for present_artifact tool arguments
*/ */
@@ -69,6 +79,42 @@ interface PresentArtifactArgs {
title: string; title: string;
} }
interface AudioBufferState {
chunks: Buffer[];
format: string;
isPCM: boolean;
totalPCMBytes: number;
}
function convertPCMToWavBuffer(
pcmBuffer: Buffer,
sampleRate: number,
channels: number,
bitsPerSample: number
): Buffer {
const headerSize = 44;
const wavBuffer = Buffer.alloc(headerSize + pcmBuffer.length);
const byteRate = (sampleRate * channels * bitsPerSample) / 8;
const blockAlign = (channels * bitsPerSample) / 8;
wavBuffer.write("RIFF", 0);
wavBuffer.writeUInt32LE(36 + pcmBuffer.length, 4);
wavBuffer.write("WAVE", 8);
wavBuffer.write("fmt ", 12);
wavBuffer.writeUInt32LE(16, 16);
wavBuffer.writeUInt16LE(1, 20);
wavBuffer.writeUInt16LE(channels, 22);
wavBuffer.writeUInt32LE(sampleRate, 24);
wavBuffer.writeUInt32LE(byteRate, 28);
wavBuffer.writeUInt16LE(blockAlign, 32);
wavBuffer.writeUInt16LE(bitsPerSample, 34);
wavBuffer.write("data", 36);
wavBuffer.writeUInt32LE(pcmBuffer.length, 40);
pcmBuffer.copy(wavBuffer, 44);
return wavBuffer;
}
/** /**
* Session represents a single client conversation session. * Session represents a single client conversation session.
* It owns all state management, orchestration logic, and message processing. * It owns all state management, orchestration logic, and message processing.
@@ -91,7 +137,7 @@ export class Session {
// Audio buffering for interruption handling // Audio buffering for interruption handling
private pendingAudioSegments: Array<{ audio: Buffer; format: string }> = []; private pendingAudioSegments: Array<{ audio: Buffer; format: string }> = [];
private bufferTimeout: NodeJS.Timeout | null = null; private bufferTimeout: NodeJS.Timeout | null = null;
private audioBuffer: { chunks: Buffer[]; format: string } | null = null; private audioBuffer: AudioBufferState | null = null;
// Conversation history // Conversation history
private messages: ModelMessage[] = []; private messages: ModelMessage[] = [];
@@ -1499,73 +1545,151 @@ export class Session {
): Promise<void> { ): Promise<void> {
await this.handleRealtimeSpeechStart(); await this.handleRealtimeSpeechStart();
// Decode base64 const chunkBuffer = Buffer.from(msg.audio, "base64");
const audioBuffer = Buffer.from(msg.audio, "base64"); const chunkFormat = msg.format || "audio/wav";
const isPCMChunk = chunkFormat.toLowerCase().includes("pcm");
if (!msg.isLast) { if (!this.audioBuffer) {
// Buffer the chunk this.audioBuffer = {
if (!this.audioBuffer) { chunks: [],
this.audioBuffer = { chunks: [], format: msg.format }; format: chunkFormat,
} isPCM: isPCMChunk,
this.audioBuffer.chunks.push(audioBuffer); totalPCMBytes: 0,
};
}
// If the format changes mid-stream, flush what we have first
if (this.audioBuffer.isPCM !== isPCMChunk) {
console.log( console.log(
`[Session ${this.clientId}] Buffered audio chunk (${audioBuffer.length} bytes, total: ${this.audioBuffer.chunks.length})` `[Session ${this.clientId}] Audio format changed mid-stream (${this.audioBuffer.isPCM ? "pcm" : this.audioBuffer.format}${chunkFormat}), flushing current buffer`
);
const finalized = this.finalizeBufferedAudio();
if (finalized) {
await this.processCompletedAudio(finalized.audio, finalized.format);
}
this.audioBuffer = {
chunks: [],
format: chunkFormat,
isPCM: isPCMChunk,
totalPCMBytes: 0,
};
} else if (!this.audioBuffer.isPCM) {
// Keep latest format info for non-PCM blobs
this.audioBuffer.format = chunkFormat;
}
this.audioBuffer.chunks.push(chunkBuffer);
if (this.audioBuffer.isPCM) {
this.audioBuffer.totalPCMBytes += chunkBuffer.length;
}
console.log(
`[Session ${this.clientId}] Buffered audio chunk (${chunkBuffer.length} bytes, chunks: ${this.audioBuffer.chunks.length}${this.audioBuffer.isPCM ? `, PCM bytes: ${this.audioBuffer.totalPCMBytes}` : ""})`
);
const reachedStreamingThreshold =
this.audioBuffer.isPCM &&
this.audioBuffer.totalPCMBytes >= MIN_STREAMING_SEGMENT_BYTES;
if (!msg.isLast && !reachedStreamingThreshold) {
return;
}
const bufferedState = this.audioBuffer;
const finalized = this.finalizeBufferedAudio();
if (!finalized) {
return;
}
if (!msg.isLast && reachedStreamingThreshold) {
console.log(
`[Session ${this.clientId}] Minimum chunk duration reached (~${MIN_STREAMING_SEGMENT_DURATION_MS}ms, ${
bufferedState?.totalPCMBytes ?? 0
} PCM bytes) triggering STT`
); );
} else { } else {
// Complete segment received
const allChunks = this.audioBuffer
? [...this.audioBuffer.chunks, audioBuffer]
: [audioBuffer];
const format = this.audioBuffer?.format || msg.format;
const currentSegmentAudio = Buffer.concat(allChunks);
console.log( console.log(
`[Session ${this.clientId}] Complete audio segment (${currentSegmentAudio.length} bytes, ${allChunks.length} chunks)` `[Session ${this.clientId}] Complete audio segment (${finalized.audio.length} bytes, ${bufferedState?.chunks.length ?? 0} chunk(s))`
);
}
await this.processCompletedAudio(finalized.audio, finalized.format);
}
private finalizeBufferedAudio():
| { audio: Buffer; format: string }
| null {
if (!this.audioBuffer) {
return null;
}
const bufferState = this.audioBuffer;
this.audioBuffer = null;
if (bufferState.isPCM) {
const pcmBuffer = Buffer.concat(bufferState.chunks);
const wavBuffer = convertPCMToWavBuffer(
pcmBuffer,
PCM_SAMPLE_RATE,
PCM_CHANNELS,
PCM_BITS_PER_SAMPLE
);
return {
audio: wavBuffer,
format: "audio/wav",
};
}
return {
audio: Buffer.concat(bufferState.chunks),
format: bufferState.format,
};
}
private async processCompletedAudio(
audio: Buffer,
format: string
): Promise<void> {
const shouldBuffer =
this.processingPhase === "transcribing" &&
this.pendingAudioSegments.length === 0;
if (shouldBuffer) {
console.log(
`[Session ${this.clientId}] Buffering audio segment (phase: ${this.processingPhase})`
);
this.pendingAudioSegments.push({
audio,
format,
});
this.setBufferTimeout();
return;
}
if (this.pendingAudioSegments.length > 0) {
this.pendingAudioSegments.push({
audio,
format,
});
console.log(
`[Session ${this.clientId}] Processing ${this.pendingAudioSegments.length} buffered segments together`
); );
// Clear chunk buffer const pendingSegments = [...this.pendingAudioSegments];
this.audioBuffer = null; this.pendingAudioSegments = [];
this.clearBufferTimeout();
// Decision: buffer or process? const combinedAudio = Buffer.concat(
const shouldBuffer = pendingSegments.map((segment) => segment.audio)
this.processingPhase === "transcribing" && );
this.pendingAudioSegments.length === 0; const combinedFormat =
pendingSegments[pendingSegments.length - 1].format;
if (shouldBuffer) { await this.processAudio(combinedAudio, combinedFormat);
// Currently transcribing first segment - buffer this one return;
console.log(
`[Session ${this.clientId}] Buffering audio segment (phase: ${this.processingPhase})`
);
this.pendingAudioSegments.push({
audio: currentSegmentAudio,
format,
});
this.setBufferTimeout();
} else if (this.pendingAudioSegments.length > 0) {
// We have buffered segments - add this one and process all together
this.pendingAudioSegments.push({
audio: currentSegmentAudio,
format,
});
console.log(
`[Session ${this.clientId}] Processing ${this.pendingAudioSegments.length} buffered segments together`
);
// Clear pending segments and timeout
const pendingSegments = [...this.pendingAudioSegments];
this.pendingAudioSegments = [];
this.clearBufferTimeout();
// Concatenate all segments
const allSegmentAudios = pendingSegments.map((s) => s.audio);
const combinedAudio = Buffer.concat(allSegmentAudios);
await this.processAudio(combinedAudio, format);
} else {
// Normal flow - no buffering needed
await this.processAudio(currentSegmentAudio, format);
}
} }
await this.processAudio(audio, format);
} }
/** /**
@@ -2086,7 +2210,11 @@ export class Session {
if (this.audioBuffer) { if (this.audioBuffer) {
console.log( console.log(
`[Session ${this.clientId}] Clearing partial audio buffer (${this.audioBuffer.chunks.length} chunk(s))` `[Session ${this.clientId}] Clearing partial audio buffer (${this.audioBuffer.chunks.length} chunk(s)${
this.audioBuffer.isPCM
? `, ${this.audioBuffer.totalPCMBytes} PCM bytes`
: ""
})`
); );
this.audioBuffer = null; this.audioBuffer = null;
} }

31
plan.md
View File

@@ -8,22 +8,22 @@
# Tasks # Tasks
- [x] Claude hydration regression must be proven with a real end-to-end test: spin up an actual Claude agent (no mocks), ask it to run tool calls that create/edit/read a temp project, verify the live stream shows the tool results, shut the agent down, hydrate from disk (`~/.claude/projects/...`), and assert the exact diff/read/command output replays in the UI. Do not mark any hydration task complete until this automated test exists and fails on current main but passes after the fix. - [x] Claude hydration regression must be proven with a real end-to-end test: spin up an actual Claude agent (no mocks), ask it to run tool calls that create/edit/read a temp project, verify the live stream shows the tool results, shut the agent down, hydrate from disk (`~/.claude/projects/...`), and assert the exact diff/read/command output replays in the UI. Do not mark any hydration task complete until this automated test exists and fails on current main but passes after the fix.
- Added a Vitest integration in `packages/server/src/server/agent/providers/claude-agent.test.ts` that drives a live Claude session through real Bash/write/read tool calls, converts the emitted timeline into the UI stream via `hydrateStreamState`, tears the agent down, resumes it from `~/.claude/projects/...`, and asserts that the hydrated stream reproduces the command output, file diff, and read content instead of leaving spinners. The helper waits for the persisted `.jsonl` file (covering `/var``/private` realpaths), cleans up temp state, and the run is wired into `npm run test --workspace=@voice-dev/server -- src/server/agent/providers/claude-agent.test.ts` (passes locally). - Added a Vitest integration in `packages/server/src/server/agent/providers/claude-agent.test.ts` that drives a live Claude session through real Bash/write/read tool calls, converts the emitted timeline into the UI stream via `hydrateStreamState`, tears the agent down, resumes it from `~/.claude/projects/...`, and asserts that the hydrated stream reproduces the command output, file diff, and read content instead of leaving spinners. The helper waits for the persisted `.jsonl` file (covering `/var``/private` realpaths), cleans up temp state, and the run is wired into `npm run test --workspace=@voice/server -- src/server/agent/providers/claude-agent.test.ts` (passes locally).
- [x] Hydrated Claude tool calls still show the spinner forever after refreshing a chat; Claude CLI already persists the tool payloads under `~/.claude/projects/<conversation>`, but our hydrate path isnt loading them. Fix the loader so it reads the saved diffs/read/output blocks and transitions the existing pill to “completed” instead of spinning or duplicating. - [x] Hydrated Claude tool calls still show the spinner forever after refreshing a chat; Claude CLI already persists the tool payloads under `~/.claude/projects/<conversation>`, but our hydrate path isnt loading them. Fix the loader so it reads the saved diffs/read/output blocks and transitions the existing pill to “completed” instead of spinning or duplicating.
- Claude history entries now flow through a shared `convertClaudeHistoryEntry` helper that inspects every message for `tool_*` blocks before classifying it as a plain user message, so persisted `tool_result` lines recorded as `type: "user"` convert into completed tool-call events instead of leaving the original spinner stuck in `executing`. Added unit coverage in `packages/server/src/server/agent/providers/claude-agent.test.ts` proving user tool results hydrate correctly and ran `npm run test --workspace=@voice-dev/server -- src/server/agent/providers/claude-agent.test.ts` (passes, though existing SDK integration specs are still slow/flaky by nature). - Claude history entries now flow through a shared `convertClaudeHistoryEntry` helper that inspects every message for `tool_*` blocks before classifying it as a plain user message, so persisted `tool_result` lines recorded as `type: "user"` convert into completed tool-call events instead of leaving the original spinner stuck in `executing`. Added unit coverage in `packages/server/src/server/agent/providers/claude-agent.test.ts` proving user tool results hydrate correctly and ran `npm run test --workspace=@voice/server -- src/server/agent/providers/claude-agent.test.ts` (passes, though existing SDK integration specs are still slow/flaky by nature).
- [x] We still see duplicate tool call pills (loading + completed/failed) in both Codex and Claude sessions, live and hydrated. Track tool call IDs rigorously, dedupe pending/completed entries, and add regression tests covering all providers and hydrate flows. - [x] We still see duplicate tool call pills (loading + completed/failed) in both Codex and Claude sessions, live and hydrated. Track tool call IDs rigorously, dedupe pending/completed entries, and add regression tests covering all providers and hydrate flows.
- Hardened the timeline reducer (`packages/app/src/types/stream.ts`) so out-of-order tool events reconcile via provider/server/tool metadata even when call IDs are missing, statuses only move forward (executing → completed/failed), and replayed hydration updates dont spawn extra pills; added Vitest coverage in `packages/app/src/types/stream.test.ts` proving both Codex and Claude live + hydrated streams stay deduped when completion blocks precede their pending counterparts. Tests run via `npx vitest packages/app/src/types/stream.test.ts`. - Hardened the timeline reducer (`packages/app/src/types/stream.ts`) so out-of-order tool events reconcile via provider/server/tool metadata even when call IDs are missing, statuses only move forward (executing → completed/failed), and replayed hydration updates dont spawn extra pills; added Vitest coverage in `packages/app/src/types/stream.test.ts` proving both Codex and Claude live + hydrated streams stay deduped when completion blocks precede their pending counterparts. Tests run via `npx vitest packages/app/src/types/stream.test.ts`.
- [x] Permission request cards in the stream header are emitting duplicate key warnings—derive stable per-agent keys (for example `${agentId}:${request.id}` with fallbacks) so React stops complaining. - [x] Permission request cards in the stream header are emitting duplicate key warnings—derive stable per-agent keys (for example `${agentId}:${request.id}` with fallbacks) so React stops complaining.
- Added a `key` field on `PendingPermission` along with a shared helper in `SessionContext` that derives `${agentId}:<fallback>` identifiers for every server-sourced request, stores them in the map, and removes them when the server resolves the request. The stream header now renders permission cards using this stable key instead of recomputing on every render, eliminating the duplicate React keys. - Added a `key` field on `PendingPermission` along with a shared helper in `SessionContext` that derives `${agentId}:<fallback>` identifiers for every server-sourced request, stores them in the map, and removes them when the server resolves the request. The stream header now renders permission cards using this stable key instead of recomputing on every render, eliminating the duplicate React keys.
- [x] The so-called tests are still fake: convert the stream harness into real Vitest suites co-located with the code, make `npm test` (Vitest) the single entrypoint, and ensure those tests fail today because hydrated tool-call results are missing. - [x] The so-called tests are still fake: convert the stream harness into real Vitest suites co-located with the code, make `npm test` (Vitest) the single entrypoint, and ensure those tests fail today because hydrated tool-call results are missing.
- Added `packages/app/src/types/stream.harness.test.ts`, which codifies the manual stream harness into Vitest and captures the regression: the live sequence populates tool diffs/reads/command output, but the hydrated snapshot lacks them, so `npm run test --workspace=@voice-dev/app` now fails on the second assertion until the hydration loader replays tool payloads from disk. - Added `packages/app/src/types/stream.harness.test.ts`, which codifies the manual stream harness into Vitest and captures the regression: the live sequence populates tool diffs/reads/command output, but the hydrated snapshot lacks them, so `npm run test --workspace=@voice/app` now fails on the second assertion until the hydration loader replays tool payloads from disk.
- [x] Hydrated sessions continue to drop user messages; capture this in a failing test for both Codex and Claude (hydrate snapshot + live replay) and only mark complete once the test passes. - [x] Hydrated sessions continue to drop user messages; capture this in a failing test for both Codex and Claude (hydrate snapshot + live replay) and only mark complete once the test passes.
- Added end-to-end hydration suites in `packages/server/src/server/agent/providers/{claude,codex}-agent.test.ts` that record live timeline updates (including user prompts) and then hydrate from `streamHistory()` after resuming the session; both now assert that the refreshed snapshot still contains the user's prompt marker. Fixed Claude hydration by having `convertClaudeHistoryEntry` emit user_message entries alongside persisted tool blocks, and taught the Codex rollout parser to surface `role: "user"` messages from rollout logs. - Added end-to-end hydration suites in `packages/server/src/server/agent/providers/{claude,codex}-agent.test.ts` that record live timeline updates (including user prompts) and then hydrate from `streamHistory()` after resuming the session; both now assert that the refreshed snapshot still contains the user's prompt marker. Fixed Claude hydration by having `convertClaudeHistoryEntry` emit user_message entries alongside persisted tool blocks, and taught the Codex rollout parser to surface `role: "user"` messages from rollout logs.
- [x] Audit the recent “Vitest migration” claim: confirm `npm test` actually runs the new suites end to end, add coverage that specifically asserts tool call results render after hydration, and keep the task open until CI proves it. - [x] Audit the recent “Vitest migration” claim: confirm `npm test` actually runs the new suites end to end, add coverage that specifically asserts tool call results render after hydration, and keep the task open until CI proves it.
- Added `resolveToolCallPreview` so the `ToolCall` UI explicitly prefers hydrated `parsed*` payloads over fallbacks and wrote `tool-call-preview.test.ts` to cover hydration-vs-fallback behavior. - Added `resolveToolCallPreview` so the `ToolCall` UI explicitly prefers hydrated `parsed*` payloads over fallbacks and wrote `tool-call-preview.test.ts` to cover hydration-vs-fallback behavior.
- Verified `npm test` now drives both workspaces (server suite passes; app suite fails fast on `src/types/stream.harness.test.ts` because hydrated tool payloads are still missing), so CI will surface the regression once the loader fix lands. - Verified `npm test` now drives both workspaces (server suite passes; app suite fails fast on `src/types/stream.harness.test.ts` because hydrated tool payloads are still missing), so CI will surface the regression once the loader fix lands.
- [x] AgentInput image picker crashes with "Attempting to launch an unregistered ActivityResultLauncher" when calling Expo ImagePicker; register the launcher properly (or switch to the new async hook) so picking images doesnt throw and we can attach screenshots again. - [x] AgentInput image picker crashes with "Attempting to launch an unregistered ActivityResultLauncher" when calling Expo ImagePicker; register the launcher properly (or switch to the new async hook) so picking images doesnt throw and we can attach screenshots again.
- Added a dedicated `useImageAttachmentPicker` hook that registers the media picker launcher up front, reuses Expos permission hook, restores pending results, and guards against concurrent launches. AgentInput now consumes this hook so tapping the attachment button opens the picker without crashing on Android; `npm run typecheck --workspace=@voice-dev/app` currently fails upstream in `stream.test.ts` before our change, noted in the summary. - Added a dedicated `useImageAttachmentPicker` hook that registers the media picker launcher up front, reuses Expos permission hook, restores pending results, and guards against concurrent launches. AgentInput now consumes this hook so tapping the attachment button opens the picker without crashing on Android; `npm run typecheck --workspace=@voice/app` currently fails upstream in `stream.test.ts` before our change, noted in the summary.
- [x] Do not mark the Claude hydration fix complete until theres an end-to-end test that (1) runs Claude, (2) executes tool calls, (3) persists the conversation under `~/.claude/projects/...`, and (4) hydrates from disk verifying the tool call results render. No test, no checkbox. - [x] Do not mark the Claude hydration fix complete until theres an end-to-end test that (1) runs Claude, (2) executes tool calls, (3) persists the conversation under `~/.claude/projects/...`, and (4) hydrates from disk verifying the tool call results render. No test, no checkbox.
- Strengthened `packages/server/src/server/agent/providers/claude-agent.test.ts` so the existing e2e hydration suite now asserts the live stream parses command/edit/read payloads and that the resumed stream reproduces those parsed diffs/reads/command outputs from `~/.claude/projects`, guaranteeing the UI pills render identical results after hydration. - Strengthened `packages/server/src/server/agent/providers/claude-agent.test.ts` so the existing e2e hydration suite now asserts the live stream parses command/edit/read payloads and that the resumed stream reproduces those parsed diffs/reads/command outputs from `~/.claude/projects`, guaranteeing the UI pills render identical results after hydration.
- [x] ERROR Encountered two children with the same key, `%s`. Keys should be unique so that components maintain their identity across updates. Non-unique keys may cause children to be duplicated and/or omitted — the behavior is unsupported and could change in a future version. .$tool_1763217454926_uyr9rm - [x] ERROR Encountered two children with the same key, `%s`. Keys should be unique so that components maintain their identity across updates. Non-unique keys may cause children to be duplicated and/or omitted — the behavior is unsupported and could change in a future version. .$tool_1763217454926_uyr9rm
@@ -42,24 +42,24 @@
``` ```
- [x] `npm test` currently fails in `packages/app` with Vitest throwing `Unexpected call to process.send` / `ERR_INVALID_ARG_TYPE` before any tests run. Track down the coverage/pool config causing this and make sure the app suite actually executes (it should include the hydrated tool-call tests mentioned above). - [x] `npm test` currently fails in `packages/app` with Vitest throwing `Unexpected call to process.send` / `ERR_INVALID_ARG_TYPE` before any tests run. Track down the coverage/pool config causing this and make sure the app suite actually executes (it should include the hydrated tool-call tests mentioned above).
- Forced the app Vitest config to use the `forks` pool (keeps `process.send` intact for Expo/xcode helpers) so the suite boots reliably; `npm test --workspace=@voice-dev/app` now runs the harness tests and reproduces the expected hydration failure instead of crashing ahead of collection. - Forced the app Vitest config to use the `forks` pool (keeps `process.send` intact for Expo/xcode helpers) so the suite boots reliably; `npm test --workspace=@voice/app` now runs the harness tests and reproduces the expected hydration failure instead of crashing ahead of collection.
- [x] WARN [expo-image-picker] `ImagePicker.MediaTypeOptions` have been deprecated. Use `ImagePicker.MediaType` or an array of `ImagePicker.MediaType` instead. - [x] WARN [expo-image-picker] `ImagePicker.MediaTypeOptions` have been deprecated. Use `ImagePicker.MediaType` or an array of `ImagePicker.MediaType` instead.
- Replaced the deprecated `MediaTypeOptions.Images` flag with the new literal `["images"]` media type array in `packages/app/src/hooks/use-image-attachment-picker.ts`, silencing the Expo warning when attaching photos. `npm run typecheck --workspace=@voice-dev/app` still fails in the pre-existing stream harness files (unchanged from before this fix). - Replaced the deprecated `MediaTypeOptions.Images` flag with the new literal `["images"]` media type array in `packages/app/src/hooks/use-image-attachment-picker.ts`, silencing the Expo warning when attaching photos. `npm run typecheck --workspace=@voice/app` still fails in the pre-existing stream harness files (unchanged from before this fix).
- [x] Update the AgentInput controls so the buttons reflect agent state: when idle show `Dictate` + `Realtime` by default (switch to `Send` when the text box has content); when an agent is running show `Realtime` + `Cancel` regardless of input so you can interrupt without typing. Realtime toggle must remain available in both states. - [x] Update the AgentInput controls so the buttons reflect agent state: when idle show `Dictate` + `Realtime` by default (switch to `Send` when the text box has content); when an agent is running show `Realtime` + `Cancel` regardless of input so you can interrupt without typing. Realtime toggle must remain available in both states.
- AgentInput now inspects the agents status and swaps the right-hand controls accordingly: send replaces dictate once text/images exist, running agents expose a new Cancel pill, and the realtime toggle is always available (it now starts/stops realtime rather than disappearing). Cancel dispatches a new `cancel_agent_request` websocket message that the server routes through `interruptAgentIfRunning`, so make sure any future API clients send that message when wiring similar controls. (FYI `npm run typecheck --workspace=@voice-dev/app` still fails in the existing stream harness suites for unrelated reasons.) - AgentInput now inspects the agents status and swaps the right-hand controls accordingly: send replaces dictate once text/images exist, running agents expose a new Cancel pill, and the realtime toggle is always available (it now starts/stops realtime rather than disappearing). Cancel dispatches a new `cancel_agent_request` websocket message that the server routes through `interruptAgentIfRunning`, so make sure any future API clients send that message when wiring similar controls. (FYI `npm run typecheck --workspace=@voice/app` still fails in the existing stream harness suites for unrelated reasons.)
- [x] Implement long press on the agent list item to opena a light bottom sheet with some actions, for now: "Delete agent" all this will do is remove it from our own persistence, but the agent will still exist in the claude/codex persistance and thats out of scope to remove. Its esentially just to remove it from the list, disown it. - [x] Implement long press on the agent list item to opena a light bottom sheet with some actions, for now: "Delete agent" all this will do is remove it from our own persistence, but the agent will still exist in the claude/codex persistance and thats out of scope to remove. Its esentially just to remove it from the list, disown it.
- Added a `delete_agent_request` flow: the session now closes the agent, removes its registry entry, and emits a dedicated `agent_deleted` event that the client listens for to prune the agents map/stream state. Long-pressing any agent row opens a lightweight bottom sheet with a Delete action wired to that new context method, so deleting disowns the agent locally without touching the Claude/Codex session. - Added a `delete_agent_request` flow: the session now closes the agent, removes its registry entry, and emits a dedicated `agent_deleted` event that the client listens for to prune the agents map/stream state. Long-pressing any agent row opens a lightweight bottom sheet with a Delete action wired to that new context method, so deleting disowns the agent locally without touching the Claude/Codex session.
- [x] Implement a function `ensureValidJson` that walks an object and validates that all values are valid JSON (no undefined, null, or non-string values). Use this for all return valies in the MCP server tools. - [x] Implement a function `ensureValidJson` that walks an object and validates that all values are valid JSON (no undefined, null, or non-string values). Use this for all return valies in the MCP server tools.
- Added `ensureValidJson` (`packages/server/src/server/json-utils.ts`) and wrapped every MCP server tool payload in both `terminal-mcp/server.ts` and `agent/mcp-server.ts`, so tool responses now get validated before streaming; `tsc` still fails earlier in the tree due to `packages/app/src/types/stream.ts` references (see npm run typecheck --workspace=@voice-dev/server). - Added `ensureValidJson` (`packages/server/src/server/json-utils.ts`) and wrapped every MCP server tool payload in both `terminal-mcp/server.ts` and `agent/mcp-server.ts`, so tool responses now get validated before streaming; `tsc` still fails earlier in the tree due to `packages/app/src/types/stream.ts` references (see npm run typecheck --workspace=@voice/server).
## Voice Interruptions & Streaming ## Voice Interruptions & Streaming
- [x] Wire interrupt signal on speech start so VAD immediately issues `abort_request` to the server before any audio segment is sent. - [x] Wire interrupt signal on speech start so VAD immediately issues `abort_request` to the server before any audio segment is sent.
- Realtime speech detection now sends a `session/abort_request` as soon as VAD confirms speech, ensuring any in-flight LLM turn is interrupted before the buffered audio is uploaded (implemented in `packages/app/src/contexts/realtime-context.tsx` with logging/error handling). - Realtime speech detection now sends a `session/abort_request` as soon as VAD confirms speech, ensuring any in-flight LLM turn is interrupted before the buffered audio is uploaded (implemented in `packages/app/src/contexts/realtime-context.tsx` with logging/error handling).
- [x] Abort server-side playback/LLM as soon as a realtime audio chunk arrives; set a speech-in-progress flag that pauses new TTS until the turn completes. - [x] Abort server-side playback/LLM as soon as a realtime audio chunk arrives; set a speech-in-progress flag that pauses new TTS until the turn completes.
- Added a `speechInProgress` guard in `packages/server/src/server/session.ts` that triggers on the first realtime chunk, cancels pending TTS playback via the new `TTSManager.cancelPendingPlaybacks`, and immediately routes through the abort flow before buffering audio; the flag is cleared when transcription finishes so the next assistant reply can synthesize speech. TTS generation now skips while the flag is set, and `npm run typecheck --workspace=@voice-dev/server` still fails in pre-existing cross-workspace `packages/app/src/types/stream.ts` imports (unchanged by this fix). - Added a `speechInProgress` guard in `packages/server/src/server/session.ts` that triggers on the first realtime chunk, cancels pending TTS playback via the new `TTSManager.cancelPendingPlaybacks`, and immediately routes through the abort flow before buffering audio; the flag is cleared when transcription finishes so the next assistant reply can synthesize speech. TTS generation now skips while the flag is set, and `npm run typecheck --workspace=@voice/server` still fails in pre-existing cross-workspace `packages/app/src/types/stream.ts` imports (unchanged by this fix).
- [x] Interrupt the currently focused agent whenever a realtime user turn begins, ensuring the next prompt starts fresh. - [x] Interrupt the currently focused agent whenever a realtime user turn begins, ensuring the next prompt starts fresh.
- Session context now tracks a `focusedAgentId` that each agent screen sets/clears on mount, and the realtime provider looks at that value to send a `cancel_agent_request` before issuing voice aborts so the next spoken prompt doesn't pile onto a running turn. Attempted `npm run typecheck --workspace=@voice-dev/app` but it still fails in the pre-existing stream harness/tests complaining about `parsed*` fields. - Session context now tracks a `focusedAgentId` that each agent screen sets/clears on mount, and the realtime provider looks at that value to send a `cancel_agent_request` before issuing voice aborts so the next spoken prompt doesn't pile onto a running turn. Attempted `npm run typecheck --workspace=@voice/app` but it still fails in the pre-existing stream harness/tests complaining about `parsed*` fields.
- [x] Extend focused-agent tracking to orchestrator mode so realtime interruptions cancel the agent thats actually active, even when no agent detail screen is open. - [x] Extend focused-agent tracking to orchestrator mode so realtime interruptions cancel the agent thats actually active, even when no agent detail screen is open.
- Session context now keeps a derived auto-focus pointing at the most recent running agent whenever no explicit agent screen is selected, so realtime speech aborts send `cancel_agent_request` for that agent before streaming audio; agent screens continue to override the focus via `setFocusedAgentId`. - Session context now keeps a derived auto-focus pointing at the most recent running agent whenever no explicit agent screen is selected, so realtime speech aborts send `cancel_agent_request` for that agent before streaming audio; agent screens continue to override the focus via `setFocusedAgentId`.
- [x] Keep agent runs alive while speaking to the orchestrator: realtime speech should **not** cancel the focused agent by default. Instead, only interrupt when the user explicitly addresses that agent again or issues a cancel command. - [x] Keep agent runs alive while speaking to the orchestrator: realtime speech should **not** cancel the focused agent by default. Instead, only interrupt when the user explicitly addresses that agent again or issues a cancel command.
@@ -71,11 +71,11 @@
- [x] Investigate streaming STT/server-side VAD (Whisper or equivalent) and document requirements for production use. - [x] Investigate streaming STT/server-side VAD (Whisper or equivalent) and document requirements for production use.
- Captured the current audio pipeline, design goals, candidate approaches (self-hosted Whisper vs managed APIs), and production requirements (transport, VAD, streaming STT engine, infra, observability, compliance) in `docs/streaming-stt-vad.md`, along with an implementation phase plan and open questions for the follow-up agent. - Captured the current audio pipeline, design goals, candidate approaches (self-hosted Whisper vs managed APIs), and production requirements (transport, VAD, streaming STT engine, infra, observability, compliance) in `docs/streaming-stt-vad.md`, along with an implementation phase plan and open questions for the follow-up agent.
- [x] Prototype streaming TTS output (chunked `audio_output` messages) and update the player to start playback as soon as chunks arrive. - [x] Prototype streaming TTS output (chunked `audio_output` messages) and update the player to start playback as soon as chunks arrive.
- Split OpenAI TTS responses into chunked `audio_output` payloads via a streaming `TTSManager` pipeline (`packages/server/src/server/agent/tts-manager.ts`, `packages/server/src/server/agent/tts-openai.ts`) that tracks per-chunk acknowledgements, adds utterance IDs, and resolves once every chunk finishes. The apps session context now keeps audio groups active until the last chunk (or an error) lands so playback begins as soon as each chunk arrives without waiting for the full clip (`packages/app/src/contexts/session-context.tsx`). `npm run typecheck --workspace=@voice-dev/server` and `npm run typecheck --workspace=@voice-dev/app` still fail in the pre-existing cross-workspace stream harness files (see earlier tasks); no new regressions were introduced. - Split OpenAI TTS responses into chunked `audio_output` payloads via a streaming `TTSManager` pipeline (`packages/server/src/server/agent/tts-manager.ts`, `packages/server/src/server/agent/tts-openai.ts`) that tracks per-chunk acknowledgements, adds utterance IDs, and resolves once every chunk finishes. The apps session context now keeps audio groups active until the last chunk (or an error) lands so playback begins as soon as each chunk arrives without waiting for the full clip (`packages/app/src/contexts/session-context.tsx`). `npm run typecheck --workspace=@voice/server` and `npm run typecheck --workspace=@voice/app` still fail in the pre-existing cross-workspace stream harness files (see earlier tasks); no new regressions were introduced.
- [x] Add telemetry for barge-in latency (speech start → playback stop, chunk arrival → LLM abort) to measure improvements per milestone. - [x] Add telemetry for barge-in latency (speech start → playback stop, chunk arrival → LLM abort) to measure improvements per milestone.
- `RealtimeProvider` now records when speech detection interrupts active playback and logs `[Telemetry] barge_in.playback_stop_latency` once the app reports audio stopped, while the server's `Session.handleRealtimeSpeechStart` logs `[Telemetry] barge_in.llm_abort_latency` after aborting live LLM runs—giving us measurable client + server latency metrics per turn. - `RealtimeProvider` now records when speech detection interrupts active playback and logs `[Telemetry] barge_in.playback_stop_latency` once the app reports audio stopped, while the server's `Session.handleRealtimeSpeechStart` logs `[Telemetry] barge_in.llm_abort_latency` after aborting live LLM runs—giving us measurable client + server latency metrics per turn.
- [x] Codex tool calls arent visible in the live stream (they only show up after hydrating/refreshing); fix the reducer/rendering path so Codex tool events appear in real time the same way Claudes do. - [x] Codex tool calls arent visible in the live stream (they only show up after hydrating/refreshing); fix the reducer/rendering path so Codex tool events appear in real time the same way Claudes do.
- Stream reducer now inspects Codex `provider_event` payloads and converts command/file/MCP/web search items into agent tool entries immediately, so the UI shows the pills while the turn is running instead of waiting for hydration; `stream.test.ts` gained coverage proving provider events alone still surface command + MCP calls, and `npm run test --workspace=@voice-dev/app -- src/types/stream.test.ts` passes. - Stream reducer now inspects Codex `provider_event` payloads and converts command/file/MCP/web search items into agent tool entries immediately, so the UI shows the pills while the turn is running instead of waiting for hydration; `stream.test.ts` gained coverage proving provider events alone still surface command + MCP calls, and `npm run test --workspace=@voice/app -- src/types/stream.test.ts` passes.
- [x] Update the AgentInput “Cancel” control to use a square stop icon button (consistent with media players) so interrupting an agent is visually distinct from other actions. - [x] Update the AgentInput “Cancel” control to use a square stop icon button (consistent with media players) so interrupting an agent is visually distinct from other actions.
- Replaced the text-based pill with a compact circular stop control that reuses the Square media glyph (plus accessibility labels), making the interrupt action visually distinct without impacting the other pending AgentInput tweaks. - Replaced the text-based pill with a compact circular stop control that reuses the Square media glyph (plus accessibility labels), making the interrupt action visually distinct without impacting the other pending AgentInput tweaks.
- [x] When the agent input has text and were not in the cancellable state, replace both the mic (Dictate) and Realtime buttons with a single Send button—only surface Dictate/Realtime when the input is empty or the agent is running. - [x] When the agent input has text and were not in the cancellable state, replace both the mic (Dictate) and Realtime buttons with a single Send button—only surface Dictate/Realtime when the input is empty or the agent is running.
@@ -85,7 +85,8 @@
- [x] npm run typecheck and fix all problems - [x] npm run typecheck and fix all problems
- Split the server build/typecheck configs so the typechecker can include the app stream helpers without breaking builds, added path aliases for `@server/*`, and tightened the shared stream types (status unions + `isAgentToolCallItem`) so the server e2e suites can import them safely. Cleaned up every failing app/server test by narrowing tool-call payloads via the helper, updated the harness + Codex/Claude specs, and re-ran `npm run typecheck` (app + server) to green. - Split the server build/typecheck configs so the typechecker can include the app stream helpers without breaking builds, added path aliases for `@server/*`, and tightened the shared stream types (status unions + `isAgentToolCallItem`) so the server e2e suites can import them safely. Cleaned up every failing app/server test by narrowing tool-call payloads via the helper, updated the harness + Codex/Claude specs, and re-ran `npm run typecheck` (app + server) to green.
- [x] audit codebase for unecessary untyped code, hacks, casts, and `any`, type things properly - [x] audit codebase for unecessary untyped code, hacks, casts, and `any`, type things properly
- Tightened the Codex rollout parser typings so permission requests, rollout/event payloads, plan arguments, and JSON helpers all operate on well-defined TypeScript structures instead of `any`. Added explicit payload/entry guards, removed unsafe casts throughout `packages/server/src/server/agent/providers/codex-agent.ts`, and re-ran `npm run typecheck --workspace=@voice-dev/server` to validate the stricter typing. - Tightened the Codex rollout parser typings so permission requests, rollout/event payloads, plan arguments, and JSON helpers all operate on well-defined TypeScript structures instead of `any`. Added explicit payload/entry guards, removed unsafe casts throughout `packages/server/src/server/agent/providers/codex-agent.ts`, and re-ran `npm run typecheck --workspace=@voice/server` to validate the stricter typing.
- [x] audit codebase for duplicated code, clean it up - [x] audit codebase for duplicated code, clean it up
- Consolidated the duplicated home/back header layouts into a shared `ScreenHeader` component so safe-area padding, borders, and spacing live in one place; both headers now only define their unique button/title content. Ran `npm run typecheck --workspace=@voice-dev/app` to confirm the refactor stays type-safe. - Consolidated the duplicated home/back header layouts into a shared `ScreenHeader` component so safe-area padding, borders, and spacing live in one place; both headers now only define their unique button/title content. Ran `npm run typecheck --workspace=@voice/app` to confirm the refactor stays type-safe.
- [ ] rename the project to Paseo, including the Expo app, package names etc. - [x] rename the project to Paseo, including the Expo app, package names etc.
- Renamed the monorepo + workspaces to `paseo`, updated Expo config (name/slug/schemes + bundle IDs), AsyncStorage keys, docs, prompts, and helper text to use `@paseo/*`, and refreshed the CLI scripts + descriptions so every entry point references Paseo instead of Voice Dev. Regenerated the lockfile + pods metadata and re-ran `npm run typecheck --workspaces --if-present` to confirm both server and app builds still typecheck under the new branding.