mirror of
https://github.com/getpaseo/paseo.git
synced 2026-07-29 12:01:31 +00:00
refactor: rebrand project from voice-dev to paseo across all packages and configs
This commit is contained in:
Mohamed Boudra
24
package-lock.json
generated
24
package-lock.json
generated
@@ -1,11 +1,11 @@
|
||||
{
|
||||
"name": "voice-dev",
|
||||
"name": "paseo",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "voice-dev",
|
||||
"name": "paseo",
|
||||
"version": "1.0.0",
|
||||
"license": "MIT",
|
||||
"workspaces": [
|
||||
@@ -5624,6 +5624,14 @@
|
||||
"node": ">=8.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@paseo/app": {
|
||||
"resolved": "packages/app",
|
||||
"link": true
|
||||
},
|
||||
"node_modules/@paseo/server": {
|
||||
"resolved": "packages/server",
|
||||
"link": true
|
||||
},
|
||||
"node_modules/@pkgjs/parseargs": {
|
||||
"version": "0.11.0",
|
||||
"resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
|
||||
@@ -7970,14 +7978,6 @@
|
||||
"url": "https://opencollective.com/vitest"
|
||||
}
|
||||
},
|
||||
"node_modules/@voice-dev/app": {
|
||||
"resolved": "packages/app",
|
||||
"link": true
|
||||
},
|
||||
"node_modules/@voice-dev/server": {
|
||||
"resolved": "packages/server",
|
||||
"link": true
|
||||
},
|
||||
"node_modules/@xmldom/xmldom": {
|
||||
"version": "0.8.11",
|
||||
"resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.11.tgz",
|
||||
@@ -20847,7 +20847,7 @@
|
||||
}
|
||||
},
|
||||
"packages/app": {
|
||||
"name": "@voice-dev/app",
|
||||
"name": "@paseo/app",
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"@boudra/expo-two-way-audio": "^0.1.3",
|
||||
@@ -20919,7 +20919,7 @@
|
||||
}
|
||||
},
|
||||
"packages/server": {
|
||||
"name": "@voice-dev/server",
|
||||
"name": "@paseo/server",
|
||||
"version": "0.1.0",
|
||||
"dependencies": {
|
||||
"@ai-sdk/openai": "^2.0.52",
|
||||
|
||||
16
package.json
16
package.json
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"name": "voice-dev",
|
||||
"name": "paseo",
|
||||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"workspaces": [
|
||||
@@ -7,20 +7,20 @@
|
||||
"packages/app"
|
||||
],
|
||||
"scripts": {
|
||||
"dev": "npm run dev --workspace=@voice-dev/server",
|
||||
"dev:app": "npm run start --workspace=@voice-dev/app",
|
||||
"dev": "npm run dev --workspace=@paseo/server",
|
||||
"dev:app": "npm run start --workspace=@paseo/app",
|
||||
"build": "npm run build --workspaces --if-present",
|
||||
"typecheck": "npm run typecheck --workspaces --if-present",
|
||||
"test": "npm run test --workspaces --if-present",
|
||||
"start": "npm run start --workspace=@voice-dev/server",
|
||||
"android": "npm run android --workspace=@voice-dev/app",
|
||||
"ios": "npm run ios --workspace=@voice-dev/app",
|
||||
"web": "npm run web --workspace=@voice-dev/app"
|
||||
"start": "npm run start --workspace=@paseo/server",
|
||||
"android": "npm run android --workspace=@paseo/app",
|
||||
"ios": "npm run ios --workspace=@paseo/app",
|
||||
"web": "npm run web --workspace=@paseo/app"
|
||||
},
|
||||
"devDependencies": {
|
||||
"typescript": "^5.9.3"
|
||||
},
|
||||
"description": "Voice-controlled development environment with OpenAI Realtime API",
|
||||
"description": "Paseo: voice-controlled development environment with OpenAI Realtime API",
|
||||
"keywords": [
|
||||
"openai",
|
||||
"realtime",
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
{
|
||||
"expo": {
|
||||
"name": "Voice Dev",
|
||||
"slug": "voice-dev",
|
||||
"name": "Paseo",
|
||||
"slug": "paseo",
|
||||
"version": "1.0.0",
|
||||
"orientation": "portrait",
|
||||
"icon": "./assets/images/icon.png",
|
||||
"scheme": "voicedev",
|
||||
"scheme": "paseo",
|
||||
"userInterfaceStyle": "automatic",
|
||||
"newArchEnabled": true,
|
||||
"runtimeVersion": {
|
||||
@@ -19,7 +19,7 @@
|
||||
"infoPlist": {
|
||||
"NSMicrophoneUsageDescription": "This app needs access to the microphone for voice commands."
|
||||
},
|
||||
"bundleIdentifier": "com.moboudra.voicedev"
|
||||
"bundleIdentifier": "com.moboudra.paseo"
|
||||
},
|
||||
"android": {
|
||||
"adaptiveIcon": {
|
||||
@@ -34,7 +34,7 @@
|
||||
"android.permission.RECORD_AUDIO",
|
||||
"android.permission.MODIFY_AUDIO_SETTINGS"
|
||||
],
|
||||
"package": "com.moboudra.voicedev"
|
||||
"package": "com.moboudra.paseo"
|
||||
},
|
||||
"web": {
|
||||
"output": "static",
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"name": "@voice-dev/app",
|
||||
"name": "@paseo/app",
|
||||
"main": "index.ts",
|
||||
"version": "1.0.0",
|
||||
"scripts": {
|
||||
|
||||
@@ -129,7 +129,7 @@ export function AgentList({ agents }: AgentListProps) {
|
||||
{actionAgent?.title || "Delete agent"}
|
||||
</Text>
|
||||
<Text style={styles.sheetSubtitle}>
|
||||
Removing this agent only deletes it from Voice Dev. Claude/Codex keeps the original project.
|
||||
Removing this agent only deletes it from Paseo. Claude/Codex keeps the original project.
|
||||
</Text>
|
||||
<Pressable
|
||||
style={[styles.sheetButton, styles.sheetDeleteButton]}
|
||||
|
||||
@@ -15,7 +15,7 @@ import AsyncStorage from "@react-native-async-storage/async-storage";
|
||||
import { StyleSheet } from "react-native-unistyles";
|
||||
import type { UseWebSocketReturn } from "../hooks/use-websocket";
|
||||
|
||||
const STORAGE_KEY = "@voice-dev:conversation-id";
|
||||
const STORAGE_KEY = "@paseo:conversation-id";
|
||||
|
||||
interface Conversation {
|
||||
id: string;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { useState, useEffect, useCallback } from 'react';
|
||||
import AsyncStorage from '@react-native-async-storage/async-storage';
|
||||
|
||||
const STORAGE_KEY = '@voice-dev:recent-paths';
|
||||
const STORAGE_KEY = '@paseo:recent-paths';
|
||||
const MAX_RECENT_PATHS = 3;
|
||||
|
||||
export interface UseRecentPathsReturn {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { useState, useEffect, useCallback } from 'react';
|
||||
import AsyncStorage from '@react-native-async-storage/async-storage';
|
||||
|
||||
const STORAGE_KEY = '@voice-dev:settings';
|
||||
const STORAGE_KEY = '@paseo:settings';
|
||||
|
||||
export interface Settings {
|
||||
serverUrl: string;
|
||||
|
||||
@@ -9,7 +9,7 @@ import {
|
||||
} from "@boudra/expo-two-way-audio";
|
||||
|
||||
export interface SpeechmaticsAudioConfig {
|
||||
onAudioSegment?: (audioData: string) => void;
|
||||
onAudioSegment?: (segment: { audioData: string; isLast: boolean }) => void;
|
||||
onSpeechStart?: () => void;
|
||||
onSpeechEnd?: () => void;
|
||||
onError?: (error: Error) => void;
|
||||
@@ -31,79 +31,6 @@ export interface SpeechmaticsAudio {
|
||||
segmentDuration: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert raw PCM data to WAV format by adding WAV headers
|
||||
*/
|
||||
function pcmToWav(
|
||||
pcmBase64: string,
|
||||
sampleRate: number = 16000,
|
||||
channels: number = 1,
|
||||
bitsPerSample: number = 16
|
||||
): string {
|
||||
const pcmBinary = atob(pcmBase64);
|
||||
const pcmLength = pcmBinary.length;
|
||||
|
||||
const byteRate = sampleRate * channels * (bitsPerSample / 8);
|
||||
const blockAlign = channels * (bitsPerSample / 8);
|
||||
const dataSize = pcmLength;
|
||||
const fileSize = 44 + dataSize;
|
||||
|
||||
const wavBuffer = new Uint8Array(fileSize);
|
||||
const view = new DataView(wavBuffer.buffer);
|
||||
|
||||
let offset = 0;
|
||||
|
||||
// RIFF chunk descriptor
|
||||
writeString(view, offset, "RIFF");
|
||||
offset += 4;
|
||||
view.setUint32(offset, fileSize - 8, true);
|
||||
offset += 4;
|
||||
writeString(view, offset, "WAVE");
|
||||
offset += 4;
|
||||
|
||||
// fmt sub-chunk
|
||||
writeString(view, offset, "fmt ");
|
||||
offset += 4;
|
||||
view.setUint32(offset, 16, true);
|
||||
offset += 4;
|
||||
view.setUint16(offset, 1, true);
|
||||
offset += 2;
|
||||
view.setUint16(offset, channels, true);
|
||||
offset += 2;
|
||||
view.setUint32(offset, sampleRate, true);
|
||||
offset += 4;
|
||||
view.setUint32(offset, byteRate, true);
|
||||
offset += 4;
|
||||
view.setUint16(offset, blockAlign, true);
|
||||
offset += 2;
|
||||
view.setUint16(offset, bitsPerSample, true);
|
||||
offset += 2;
|
||||
|
||||
// data sub-chunk
|
||||
writeString(view, offset, "data");
|
||||
offset += 4;
|
||||
view.setUint32(offset, dataSize, true);
|
||||
offset += 4;
|
||||
|
||||
// Copy PCM data
|
||||
for (let i = 0; i < pcmLength; i++) {
|
||||
wavBuffer[offset + i] = pcmBinary.charCodeAt(i);
|
||||
}
|
||||
|
||||
// Convert to base64
|
||||
let binary = "";
|
||||
for (let i = 0; i < wavBuffer.length; i++) {
|
||||
binary += String.fromCharCode(wavBuffer[i]);
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
function writeString(view: DataView, offset: number, str: string): void {
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
view.setUint8(offset + i, str.charCodeAt(i));
|
||||
}
|
||||
}
|
||||
|
||||
function uint8ArrayToBase64(bytes: Uint8Array): string {
|
||||
let binary = "";
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
@@ -143,6 +70,38 @@ export function useSpeechmaticsAudio(
|
||||
const speechDetectionStartRef = useRef<number | null>(null);
|
||||
const speechConfirmedRef = useRef(false);
|
||||
const detectionSilenceStartRef = useRef<number | null>(null);
|
||||
const bufferedBytesRef = useRef(0);
|
||||
|
||||
const PCM_SAMPLE_RATE = 16000;
|
||||
const PCM_CHANNELS = 1;
|
||||
const PCM_BITS_PER_SAMPLE = 16;
|
||||
const PCM_BYTES_PER_MS =
|
||||
(PCM_SAMPLE_RATE * PCM_CHANNELS * (PCM_BITS_PER_SAMPLE / 8)) / 1000;
|
||||
const MIN_CHUNK_DURATION_MS = 1000;
|
||||
const MIN_CHUNK_BYTES = Math.round(
|
||||
PCM_BYTES_PER_MS * MIN_CHUNK_DURATION_MS
|
||||
);
|
||||
|
||||
const flushBufferedAudio = useCallback(
|
||||
(isLast: boolean) => {
|
||||
if (audioBufferRef.current.length === 0) {
|
||||
bufferedBytesRef.current = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
const combinedBinary = concatenateUint8Arrays(audioBufferRef.current);
|
||||
const pcmBase64 = uint8ArrayToBase64(combinedBinary);
|
||||
|
||||
config.onAudioSegment?.({
|
||||
audioData: pcmBase64,
|
||||
isLast,
|
||||
});
|
||||
|
||||
audioBufferRef.current = [];
|
||||
bufferedBytesRef.current = 0;
|
||||
},
|
||||
[config]
|
||||
);
|
||||
|
||||
const VOLUME_THRESHOLD = config.volumeThreshold;
|
||||
const SILENCE_DURATION_MS = config.silenceDuration;
|
||||
@@ -178,9 +137,17 @@ export function useSpeechmaticsAudio(
|
||||
// Start buffering from first spike to capture beginning of speech
|
||||
if (speechDetectionStartRef.current !== null || isSpeakingRef.current) {
|
||||
audioBufferRef.current.push(pcmData);
|
||||
bufferedBytesRef.current += pcmData.length;
|
||||
|
||||
if (
|
||||
speechConfirmedRef.current &&
|
||||
bufferedBytesRef.current >= MIN_CHUNK_BYTES
|
||||
) {
|
||||
flushBufferedAudio(false);
|
||||
}
|
||||
}
|
||||
},
|
||||
[isActive, isMuted]
|
||||
[isActive, isMuted, flushBufferedAudio, MIN_CHUNK_BYTES]
|
||||
)
|
||||
);
|
||||
|
||||
@@ -215,6 +182,7 @@ export function useSpeechmaticsAudio(
|
||||
speechDetectionStartRef.current = Date.now();
|
||||
detectionSilenceStartRef.current = null;
|
||||
audioBufferRef.current = [];
|
||||
bufferedBytesRef.current = 0;
|
||||
setIsDetecting(true);
|
||||
} else {
|
||||
// Volume is back above threshold - reset grace period
|
||||
@@ -264,6 +232,7 @@ export function useSpeechmaticsAudio(
|
||||
speechDetectionStartRef.current = null;
|
||||
detectionSilenceStartRef.current = null;
|
||||
audioBufferRef.current = [];
|
||||
bufferedBytesRef.current = 0;
|
||||
setIsDetecting(false);
|
||||
}
|
||||
}
|
||||
@@ -292,15 +261,7 @@ export function useSpeechmaticsAudio(
|
||||
config.onSpeechEnd?.();
|
||||
|
||||
// Send buffered audio segment
|
||||
if (audioBufferRef.current.length > 0) {
|
||||
const combinedBinary = concatenateUint8Arrays(
|
||||
audioBufferRef.current
|
||||
);
|
||||
const combinedPcmBase64 = uint8ArrayToBase64(combinedBinary);
|
||||
const wavAudio = pcmToWav(combinedPcmBase64, 16000, 1, 16);
|
||||
config.onAudioSegment?.(wavAudio);
|
||||
audioBufferRef.current = [];
|
||||
}
|
||||
flushBufferedAudio(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -313,6 +274,7 @@ export function useSpeechmaticsAudio(
|
||||
SPEECH_CONFIRMATION_MS,
|
||||
DETECTION_GRACE_PERIOD_MS,
|
||||
config,
|
||||
flushBufferedAudio,
|
||||
]
|
||||
)
|
||||
);
|
||||
@@ -365,6 +327,7 @@ export function useSpeechmaticsAudio(
|
||||
|
||||
// Reset state
|
||||
audioBufferRef.current = [];
|
||||
bufferedBytesRef.current = 0;
|
||||
isSpeakingRef.current = false;
|
||||
speechConfirmedRef.current = false;
|
||||
speechDetectionStartRef.current = null;
|
||||
@@ -387,6 +350,7 @@ export function useSpeechmaticsAudio(
|
||||
if (newMuted) {
|
||||
// Clear any ongoing speech detection/speaking state
|
||||
audioBufferRef.current = [];
|
||||
bufferedBytesRef.current = 0;
|
||||
isSpeakingRef.current = false;
|
||||
speechConfirmedRef.current = false;
|
||||
speechDetectionStartRef.current = null;
|
||||
|
||||
@@ -441,14 +441,14 @@ function testToolCallParsedPayloadHydration() {
|
||||
tool_use_id: commandCallId,
|
||||
result: {
|
||||
command: 'pwd',
|
||||
output: '/Users/dev/voice-dev',
|
||||
output: '/Users/dev/paseo',
|
||||
},
|
||||
metadata: { exit_code: 0 },
|
||||
},
|
||||
output: {
|
||||
result: {
|
||||
command: 'pwd',
|
||||
output: '/Users/dev/voice-dev',
|
||||
output: '/Users/dev/paseo',
|
||||
},
|
||||
metadata: { exit_code: 0 },
|
||||
},
|
||||
@@ -477,7 +477,7 @@ function testToolCallParsedPayloadHydration() {
|
||||
const commandPass = Boolean(
|
||||
commandEntry?.payload.data.parsedCommand &&
|
||||
commandEntry.payload.data.parsedCommand.command === 'pwd' &&
|
||||
commandEntry.payload.data.parsedCommand.output?.includes('/voice-dev')
|
||||
commandEntry.payload.data.parsedCommand.output?.includes('/paseo')
|
||||
);
|
||||
|
||||
assert.ok(readPass, 'Read payload should persist across hydration');
|
||||
|
||||
@@ -285,11 +285,11 @@ When Claude Code presents a plan in plan mode, forward it to user's screen:
|
||||
```javascript
|
||||
// User mentions project
|
||||
User: "Create a terminal for the web project"
|
||||
create-terminal(name="web", workingDirectory="~/dev/voice-dev/packages/web")
|
||||
create-terminal(name="web", workingDirectory="~/dev/paseo/packages/web")
|
||||
|
||||
// User says "another terminal here"
|
||||
// Look at current terminal's working directory, use same path
|
||||
create-terminal(name="tests", workingDirectory="~/dev/voice-dev/packages/web")
|
||||
create-terminal(name="tests", workingDirectory="~/dev/paseo/packages/web")
|
||||
|
||||
// No context - list terminals first to see what they're working on
|
||||
User: "Create a terminal"
|
||||
@@ -430,7 +430,7 @@ Multi-step process:
|
||||
|
||||
```javascript
|
||||
// Step 1
|
||||
create_terminal((name = "fix-auth"), (workingDirectory = "~/dev/voice-dev"));
|
||||
create_terminal((name = "fix-auth"), (workingDirectory = "~/dev/paseo"));
|
||||
|
||||
// Step 2
|
||||
send_text(
|
||||
@@ -455,7 +455,7 @@ send_text((terminalName = "fix-auth"), (text = "claude"), (pressEnter = true));
|
||||
|
||||
**Terminal naming:**
|
||||
|
||||
- No worktree: Use project name ("faro", "voice-dev")
|
||||
- No worktree: Use project name ("faro", "paseo")
|
||||
- With worktree: Use worktree name ("fix-auth", "feature-export")
|
||||
|
||||
**When user says "launch Claude in [project]":**
|
||||
@@ -494,9 +494,9 @@ Already authenticated. Use for:
|
||||
|
||||
All projects in `~/dev`:
|
||||
|
||||
**voice-dev**
|
||||
**paseo**
|
||||
|
||||
- Location: `~/dev/voice-dev`
|
||||
- Location: `~/dev/paseo`
|
||||
- Packages: `voice-assistant`
|
||||
|
||||
**Faro** (Autonomous Competitive Intelligence)
|
||||
|
||||
@@ -191,12 +191,12 @@ We only have two coding agents. Do not call tools to discover them—treat this
|
||||
|
||||
### Creating Agents
|
||||
|
||||
**Confirm creation only when intent is unclear.** If the user gives a direct imperative (“spin up a new planner agent in voice-dev”), acknowledge and create immediately; otherwise, ask.
|
||||
**Confirm creation only when intent is unclear.** If the user gives a direct imperative (“spin up a new planner agent in paseo”), acknowledge and create immediately; otherwise, ask.
|
||||
|
||||
```javascript
|
||||
// Claude Code with planning
|
||||
create_coding_agent({
|
||||
cwd: "~/dev/voice-dev",
|
||||
cwd: "~/dev/paseo",
|
||||
agentType: "claude",
|
||||
initialPrompt: "add dark mode toggle to settings page",
|
||||
initialMode: "plan"
|
||||
@@ -204,7 +204,7 @@ create_coding_agent({
|
||||
|
||||
// Codex for quick edits
|
||||
create_coding_agent({
|
||||
cwd: "~/dev/voice-dev",
|
||||
cwd: "~/dev/paseo",
|
||||
agentType: "codex",
|
||||
initialPrompt: "clean up the logging",
|
||||
initialMode: "auto"
|
||||
@@ -328,8 +328,8 @@ Already authenticated. Use for:
|
||||
|
||||
All projects in `~/dev`:
|
||||
|
||||
**voice-dev**
|
||||
- Location: `~/dev/voice-dev`
|
||||
**paseo**
|
||||
- Location: `~/dev/paseo`
|
||||
- Packages: `voice-assistant`
|
||||
|
||||
**Faro** (Autonomous Competitive Intelligence)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@voice-dev/server",
|
||||
"name": "@paseo/server",
|
||||
"version": "0.1.0",
|
||||
"description": "Voice assistant backend server",
|
||||
"description": "Paseo backend server",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "NODE_ENV=development node --watch --import tsx src/server/index.ts",
|
||||
|
||||
@@ -22,7 +22,7 @@ async function getTerminalMcpClient(): Promise<McpClient> {
|
||||
}
|
||||
|
||||
// Create Terminal MCP server
|
||||
const server = await createTerminalMcpServer({ sessionName: "__voice-dev" });
|
||||
const server = await createTerminalMcpServer({ sessionName: "__paseo" });
|
||||
|
||||
// Create linked transport pair
|
||||
const [clientTransport, serverTransport] =
|
||||
|
||||
@@ -57,6 +57,16 @@ const ACTIVE_TITLE_GENERATIONS = new Set<string>();
|
||||
|
||||
type ProcessingPhase = "idle" | "transcribing" | "llm";
|
||||
|
||||
const PCM_SAMPLE_RATE = 16000;
|
||||
const PCM_CHANNELS = 1;
|
||||
const PCM_BITS_PER_SAMPLE = 16;
|
||||
const PCM_BYTES_PER_MS =
|
||||
(PCM_SAMPLE_RATE * PCM_CHANNELS * (PCM_BITS_PER_SAMPLE / 8)) / 1000;
|
||||
const MIN_STREAMING_SEGMENT_DURATION_MS = 1000;
|
||||
const MIN_STREAMING_SEGMENT_BYTES = Math.round(
|
||||
PCM_BYTES_PER_MS * MIN_STREAMING_SEGMENT_DURATION_MS
|
||||
);
|
||||
|
||||
/**
|
||||
* Type for present_artifact tool arguments
|
||||
*/
|
||||
@@ -69,6 +79,42 @@ interface PresentArtifactArgs {
|
||||
title: string;
|
||||
}
|
||||
|
||||
interface AudioBufferState {
|
||||
chunks: Buffer[];
|
||||
format: string;
|
||||
isPCM: boolean;
|
||||
totalPCMBytes: number;
|
||||
}
|
||||
|
||||
function convertPCMToWavBuffer(
|
||||
pcmBuffer: Buffer,
|
||||
sampleRate: number,
|
||||
channels: number,
|
||||
bitsPerSample: number
|
||||
): Buffer {
|
||||
const headerSize = 44;
|
||||
const wavBuffer = Buffer.alloc(headerSize + pcmBuffer.length);
|
||||
const byteRate = (sampleRate * channels * bitsPerSample) / 8;
|
||||
const blockAlign = (channels * bitsPerSample) / 8;
|
||||
|
||||
wavBuffer.write("RIFF", 0);
|
||||
wavBuffer.writeUInt32LE(36 + pcmBuffer.length, 4);
|
||||
wavBuffer.write("WAVE", 8);
|
||||
wavBuffer.write("fmt ", 12);
|
||||
wavBuffer.writeUInt32LE(16, 16);
|
||||
wavBuffer.writeUInt16LE(1, 20);
|
||||
wavBuffer.writeUInt16LE(channels, 22);
|
||||
wavBuffer.writeUInt32LE(sampleRate, 24);
|
||||
wavBuffer.writeUInt32LE(byteRate, 28);
|
||||
wavBuffer.writeUInt16LE(blockAlign, 32);
|
||||
wavBuffer.writeUInt16LE(bitsPerSample, 34);
|
||||
wavBuffer.write("data", 36);
|
||||
wavBuffer.writeUInt32LE(pcmBuffer.length, 40);
|
||||
pcmBuffer.copy(wavBuffer, 44);
|
||||
|
||||
return wavBuffer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Session represents a single client conversation session.
|
||||
* It owns all state management, orchestration logic, and message processing.
|
||||
@@ -91,7 +137,7 @@ export class Session {
|
||||
// Audio buffering for interruption handling
|
||||
private pendingAudioSegments: Array<{ audio: Buffer; format: string }> = [];
|
||||
private bufferTimeout: NodeJS.Timeout | null = null;
|
||||
private audioBuffer: { chunks: Buffer[]; format: string } | null = null;
|
||||
private audioBuffer: AudioBufferState | null = null;
|
||||
|
||||
// Conversation history
|
||||
private messages: ModelMessage[] = [];
|
||||
@@ -1499,73 +1545,151 @@ export class Session {
|
||||
): Promise<void> {
|
||||
await this.handleRealtimeSpeechStart();
|
||||
|
||||
// Decode base64
|
||||
const audioBuffer = Buffer.from(msg.audio, "base64");
|
||||
const chunkBuffer = Buffer.from(msg.audio, "base64");
|
||||
const chunkFormat = msg.format || "audio/wav";
|
||||
const isPCMChunk = chunkFormat.toLowerCase().includes("pcm");
|
||||
|
||||
if (!msg.isLast) {
|
||||
// Buffer the chunk
|
||||
if (!this.audioBuffer) {
|
||||
this.audioBuffer = { chunks: [], format: msg.format };
|
||||
}
|
||||
this.audioBuffer.chunks.push(audioBuffer);
|
||||
if (!this.audioBuffer) {
|
||||
this.audioBuffer = {
|
||||
chunks: [],
|
||||
format: chunkFormat,
|
||||
isPCM: isPCMChunk,
|
||||
totalPCMBytes: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// If the format changes mid-stream, flush what we have first
|
||||
if (this.audioBuffer.isPCM !== isPCMChunk) {
|
||||
console.log(
|
||||
`[Session ${this.clientId}] Buffered audio chunk (${audioBuffer.length} bytes, total: ${this.audioBuffer.chunks.length})`
|
||||
`[Session ${this.clientId}] Audio format changed mid-stream (${this.audioBuffer.isPCM ? "pcm" : this.audioBuffer.format} → ${chunkFormat}), flushing current buffer`
|
||||
);
|
||||
const finalized = this.finalizeBufferedAudio();
|
||||
if (finalized) {
|
||||
await this.processCompletedAudio(finalized.audio, finalized.format);
|
||||
}
|
||||
this.audioBuffer = {
|
||||
chunks: [],
|
||||
format: chunkFormat,
|
||||
isPCM: isPCMChunk,
|
||||
totalPCMBytes: 0,
|
||||
};
|
||||
} else if (!this.audioBuffer.isPCM) {
|
||||
// Keep latest format info for non-PCM blobs
|
||||
this.audioBuffer.format = chunkFormat;
|
||||
}
|
||||
|
||||
this.audioBuffer.chunks.push(chunkBuffer);
|
||||
if (this.audioBuffer.isPCM) {
|
||||
this.audioBuffer.totalPCMBytes += chunkBuffer.length;
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[Session ${this.clientId}] Buffered audio chunk (${chunkBuffer.length} bytes, chunks: ${this.audioBuffer.chunks.length}${this.audioBuffer.isPCM ? `, PCM bytes: ${this.audioBuffer.totalPCMBytes}` : ""})`
|
||||
);
|
||||
|
||||
const reachedStreamingThreshold =
|
||||
this.audioBuffer.isPCM &&
|
||||
this.audioBuffer.totalPCMBytes >= MIN_STREAMING_SEGMENT_BYTES;
|
||||
|
||||
if (!msg.isLast && !reachedStreamingThreshold) {
|
||||
return;
|
||||
}
|
||||
|
||||
const bufferedState = this.audioBuffer;
|
||||
const finalized = this.finalizeBufferedAudio();
|
||||
if (!finalized) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!msg.isLast && reachedStreamingThreshold) {
|
||||
console.log(
|
||||
`[Session ${this.clientId}] Minimum chunk duration reached (~${MIN_STREAMING_SEGMENT_DURATION_MS}ms, ${
|
||||
bufferedState?.totalPCMBytes ?? 0
|
||||
} PCM bytes) – triggering STT`
|
||||
);
|
||||
} else {
|
||||
// Complete segment received
|
||||
const allChunks = this.audioBuffer
|
||||
? [...this.audioBuffer.chunks, audioBuffer]
|
||||
: [audioBuffer];
|
||||
const format = this.audioBuffer?.format || msg.format;
|
||||
const currentSegmentAudio = Buffer.concat(allChunks);
|
||||
|
||||
console.log(
|
||||
`[Session ${this.clientId}] Complete audio segment (${currentSegmentAudio.length} bytes, ${allChunks.length} chunks)`
|
||||
`[Session ${this.clientId}] Complete audio segment (${finalized.audio.length} bytes, ${bufferedState?.chunks.length ?? 0} chunk(s))`
|
||||
);
|
||||
}
|
||||
|
||||
await this.processCompletedAudio(finalized.audio, finalized.format);
|
||||
}
|
||||
|
||||
private finalizeBufferedAudio():
|
||||
| { audio: Buffer; format: string }
|
||||
| null {
|
||||
if (!this.audioBuffer) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const bufferState = this.audioBuffer;
|
||||
this.audioBuffer = null;
|
||||
|
||||
if (bufferState.isPCM) {
|
||||
const pcmBuffer = Buffer.concat(bufferState.chunks);
|
||||
const wavBuffer = convertPCMToWavBuffer(
|
||||
pcmBuffer,
|
||||
PCM_SAMPLE_RATE,
|
||||
PCM_CHANNELS,
|
||||
PCM_BITS_PER_SAMPLE
|
||||
);
|
||||
return {
|
||||
audio: wavBuffer,
|
||||
format: "audio/wav",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
audio: Buffer.concat(bufferState.chunks),
|
||||
format: bufferState.format,
|
||||
};
|
||||
}
|
||||
|
||||
private async processCompletedAudio(
|
||||
audio: Buffer,
|
||||
format: string
|
||||
): Promise<void> {
|
||||
const shouldBuffer =
|
||||
this.processingPhase === "transcribing" &&
|
||||
this.pendingAudioSegments.length === 0;
|
||||
|
||||
if (shouldBuffer) {
|
||||
console.log(
|
||||
`[Session ${this.clientId}] Buffering audio segment (phase: ${this.processingPhase})`
|
||||
);
|
||||
this.pendingAudioSegments.push({
|
||||
audio,
|
||||
format,
|
||||
});
|
||||
this.setBufferTimeout();
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.pendingAudioSegments.length > 0) {
|
||||
this.pendingAudioSegments.push({
|
||||
audio,
|
||||
format,
|
||||
});
|
||||
console.log(
|
||||
`[Session ${this.clientId}] Processing ${this.pendingAudioSegments.length} buffered segments together`
|
||||
);
|
||||
|
||||
// Clear chunk buffer
|
||||
this.audioBuffer = null;
|
||||
const pendingSegments = [...this.pendingAudioSegments];
|
||||
this.pendingAudioSegments = [];
|
||||
this.clearBufferTimeout();
|
||||
|
||||
// Decision: buffer or process?
|
||||
const shouldBuffer =
|
||||
this.processingPhase === "transcribing" &&
|
||||
this.pendingAudioSegments.length === 0;
|
||||
const combinedAudio = Buffer.concat(
|
||||
pendingSegments.map((segment) => segment.audio)
|
||||
);
|
||||
const combinedFormat =
|
||||
pendingSegments[pendingSegments.length - 1].format;
|
||||
|
||||
if (shouldBuffer) {
|
||||
// Currently transcribing first segment - buffer this one
|
||||
console.log(
|
||||
`[Session ${this.clientId}] Buffering audio segment (phase: ${this.processingPhase})`
|
||||
);
|
||||
this.pendingAudioSegments.push({
|
||||
audio: currentSegmentAudio,
|
||||
format,
|
||||
});
|
||||
this.setBufferTimeout();
|
||||
} else if (this.pendingAudioSegments.length > 0) {
|
||||
// We have buffered segments - add this one and process all together
|
||||
this.pendingAudioSegments.push({
|
||||
audio: currentSegmentAudio,
|
||||
format,
|
||||
});
|
||||
console.log(
|
||||
`[Session ${this.clientId}] Processing ${this.pendingAudioSegments.length} buffered segments together`
|
||||
);
|
||||
|
||||
// Clear pending segments and timeout
|
||||
const pendingSegments = [...this.pendingAudioSegments];
|
||||
this.pendingAudioSegments = [];
|
||||
this.clearBufferTimeout();
|
||||
|
||||
// Concatenate all segments
|
||||
const allSegmentAudios = pendingSegments.map((s) => s.audio);
|
||||
const combinedAudio = Buffer.concat(allSegmentAudios);
|
||||
|
||||
await this.processAudio(combinedAudio, format);
|
||||
} else {
|
||||
// Normal flow - no buffering needed
|
||||
await this.processAudio(currentSegmentAudio, format);
|
||||
}
|
||||
await this.processAudio(combinedAudio, combinedFormat);
|
||||
return;
|
||||
}
|
||||
|
||||
await this.processAudio(audio, format);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -2086,7 +2210,11 @@ export class Session {
|
||||
|
||||
if (this.audioBuffer) {
|
||||
console.log(
|
||||
`[Session ${this.clientId}] Clearing partial audio buffer (${this.audioBuffer.chunks.length} chunk(s))`
|
||||
`[Session ${this.clientId}] Clearing partial audio buffer (${this.audioBuffer.chunks.length} chunk(s)${
|
||||
this.audioBuffer.isPCM
|
||||
? `, ${this.audioBuffer.totalPCMBytes} PCM bytes`
|
||||
: ""
|
||||
})`
|
||||
);
|
||||
this.audioBuffer = null;
|
||||
}
|
||||
|
||||
31
plan.md
31
plan.md
@@ -8,22 +8,22 @@
|
||||
# Tasks
|
||||
|
||||
- [x] Claude hydration regression must be proven with a real end-to-end test: spin up an actual Claude agent (no mocks), ask it to run tool calls that create/edit/read a temp project, verify the live stream shows the tool results, shut the agent down, hydrate from disk (`~/.claude/projects/...`), and assert the exact diff/read/command output replays in the UI. Do not mark any hydration task complete until this automated test exists and fails on current main but passes after the fix.
|
||||
- Added a Vitest integration in `packages/server/src/server/agent/providers/claude-agent.test.ts` that drives a live Claude session through real Bash/write/read tool calls, converts the emitted timeline into the UI stream via `hydrateStreamState`, tears the agent down, resumes it from `~/.claude/projects/...`, and asserts that the hydrated stream reproduces the command output, file diff, and read content instead of leaving spinners. The helper waits for the persisted `.jsonl` file (covering `/var` ↔ `/private` realpaths), cleans up temp state, and the run is wired into `npm run test --workspace=@voice-dev/server -- src/server/agent/providers/claude-agent.test.ts` (passes locally).
|
||||
- Added a Vitest integration in `packages/server/src/server/agent/providers/claude-agent.test.ts` that drives a live Claude session through real Bash/write/read tool calls, converts the emitted timeline into the UI stream via `hydrateStreamState`, tears the agent down, resumes it from `~/.claude/projects/...`, and asserts that the hydrated stream reproduces the command output, file diff, and read content instead of leaving spinners. The helper waits for the persisted `.jsonl` file (covering `/var` ↔ `/private` realpaths), cleans up temp state, and the run is wired into `npm run test --workspace=@voice/server -- src/server/agent/providers/claude-agent.test.ts` (passes locally).
|
||||
- [x] Hydrated Claude tool calls still show the spinner forever after refreshing a chat; Claude CLI already persists the tool payloads under `~/.claude/projects/<conversation>`, but our hydrate path isn’t loading them. Fix the loader so it reads the saved diffs/read/output blocks and transitions the existing pill to “completed” instead of spinning or duplicating.
|
||||
- Claude history entries now flow through a shared `convertClaudeHistoryEntry` helper that inspects every message for `tool_*` blocks before classifying it as a plain user message, so persisted `tool_result` lines recorded as `type: "user"` convert into completed tool-call events instead of leaving the original spinner stuck in `executing`. Added unit coverage in `packages/server/src/server/agent/providers/claude-agent.test.ts` proving user tool results hydrate correctly and ran `npm run test --workspace=@voice-dev/server -- src/server/agent/providers/claude-agent.test.ts` (passes, though existing SDK integration specs are still slow/flaky by nature).
|
||||
- Claude history entries now flow through a shared `convertClaudeHistoryEntry` helper that inspects every message for `tool_*` blocks before classifying it as a plain user message, so persisted `tool_result` lines recorded as `type: "user"` convert into completed tool-call events instead of leaving the original spinner stuck in `executing`. Added unit coverage in `packages/server/src/server/agent/providers/claude-agent.test.ts` proving user tool results hydrate correctly and ran `npm run test --workspace=@voice/server -- src/server/agent/providers/claude-agent.test.ts` (passes, though existing SDK integration specs are still slow/flaky by nature).
|
||||
- [x] We still see duplicate tool call pills (loading + completed/failed) in both Codex and Claude sessions, live and hydrated. Track tool call IDs rigorously, dedupe pending/completed entries, and add regression tests covering all providers and hydrate flows.
|
||||
- Hardened the timeline reducer (`packages/app/src/types/stream.ts`) so out-of-order tool events reconcile via provider/server/tool metadata even when call IDs are missing, statuses only move forward (executing → completed/failed), and replayed hydration updates don’t spawn extra pills; added Vitest coverage in `packages/app/src/types/stream.test.ts` proving both Codex and Claude live + hydrated streams stay deduped when completion blocks precede their pending counterparts. Tests run via `npx vitest packages/app/src/types/stream.test.ts`.
|
||||
- [x] Permission request cards in the stream header are emitting duplicate key warnings—derive stable per-agent keys (for example `${agentId}:${request.id}` with fallbacks) so React stops complaining.
|
||||
- Added a `key` field on `PendingPermission` along with a shared helper in `SessionContext` that derives `${agentId}:<fallback>` identifiers for every server-sourced request, stores them in the map, and removes them when the server resolves the request. The stream header now renders permission cards using this stable key instead of recomputing on every render, eliminating the duplicate React keys.
|
||||
- [x] The so-called tests are still fake: convert the stream harness into real Vitest suites co-located with the code, make `npm test` (Vitest) the single entrypoint, and ensure those tests fail today because hydrated tool-call results are missing.
|
||||
- Added `packages/app/src/types/stream.harness.test.ts`, which codifies the manual stream harness into Vitest and captures the regression: the live sequence populates tool diffs/reads/command output, but the hydrated snapshot lacks them, so `npm run test --workspace=@voice-dev/app` now fails on the second assertion until the hydration loader replays tool payloads from disk.
|
||||
- Added `packages/app/src/types/stream.harness.test.ts`, which codifies the manual stream harness into Vitest and captures the regression: the live sequence populates tool diffs/reads/command output, but the hydrated snapshot lacks them, so `npm run test --workspace=@voice/app` now fails on the second assertion until the hydration loader replays tool payloads from disk.
|
||||
- [x] Hydrated sessions continue to drop user messages; capture this in a failing test for both Codex and Claude (hydrate snapshot + live replay) and only mark complete once the test passes.
|
||||
- Added end-to-end hydration suites in `packages/server/src/server/agent/providers/{claude,codex}-agent.test.ts` that record live timeline updates (including user prompts) and then hydrate from `streamHistory()` after resuming the session; both now assert that the refreshed snapshot still contains the user's prompt marker. Fixed Claude hydration by having `convertClaudeHistoryEntry` emit user_message entries alongside persisted tool blocks, and taught the Codex rollout parser to surface `role: "user"` messages from rollout logs.
|
||||
- [x] Audit the recent “Vitest migration” claim: confirm `npm test` actually runs the new suites end to end, add coverage that specifically asserts tool call results render after hydration, and keep the task open until CI proves it.
|
||||
- Added `resolveToolCallPreview` so the `ToolCall` UI explicitly prefers hydrated `parsed*` payloads over fallbacks and wrote `tool-call-preview.test.ts` to cover hydration-vs-fallback behavior.
|
||||
- Verified `npm test` now drives both workspaces (server suite passes; app suite fails fast on `src/types/stream.harness.test.ts` because hydrated tool payloads are still missing), so CI will surface the regression once the loader fix lands.
|
||||
- [x] AgentInput image picker crashes with "Attempting to launch an unregistered ActivityResultLauncher" when calling Expo ImagePicker; register the launcher properly (or switch to the new async hook) so picking images doesn’t throw and we can attach screenshots again.
|
||||
- Added a dedicated `useImageAttachmentPicker` hook that registers the media picker launcher up front, reuses Expo’s permission hook, restores pending results, and guards against concurrent launches. AgentInput now consumes this hook so tapping the attachment button opens the picker without crashing on Android; `npm run typecheck --workspace=@voice-dev/app` currently fails upstream in `stream.test.ts` before our change, noted in the summary.
|
||||
- Added a dedicated `useImageAttachmentPicker` hook that registers the media picker launcher up front, reuses Expo’s permission hook, restores pending results, and guards against concurrent launches. AgentInput now consumes this hook so tapping the attachment button opens the picker without crashing on Android; `npm run typecheck --workspace=@voice/app` currently fails upstream in `stream.test.ts` before our change, noted in the summary.
|
||||
- [x] Do not mark the Claude hydration fix complete until there’s an end-to-end test that (1) runs Claude, (2) executes tool calls, (3) persists the conversation under `~/.claude/projects/...`, and (4) hydrates from disk verifying the tool call results render. No test, no checkbox.
|
||||
- Strengthened `packages/server/src/server/agent/providers/claude-agent.test.ts` so the existing e2e hydration suite now asserts the live stream parses command/edit/read payloads and that the resumed stream reproduces those parsed diffs/reads/command outputs from `~/.claude/projects`, guaranteeing the UI pills render identical results after hydration.
|
||||
- [x] ERROR Encountered two children with the same key, `%s`. Keys should be unique so that components maintain their identity across updates. Non-unique keys may cause children to be duplicated and/or omitted — the behavior is unsupported and could change in a future version. .$tool_1763217454926_uyr9rm
|
||||
@@ -42,24 +42,24 @@
|
||||
```
|
||||
|
||||
- [x] `npm test` currently fails in `packages/app` with Vitest throwing `Unexpected call to process.send` / `ERR_INVALID_ARG_TYPE` before any tests run. Track down the coverage/pool config causing this and make sure the app suite actually executes (it should include the hydrated tool-call tests mentioned above).
|
||||
- Forced the app Vitest config to use the `forks` pool (keeps `process.send` intact for Expo/xcode helpers) so the suite boots reliably; `npm test --workspace=@voice-dev/app` now runs the harness tests and reproduces the expected hydration failure instead of crashing ahead of collection.
|
||||
- Forced the app Vitest config to use the `forks` pool (keeps `process.send` intact for Expo/xcode helpers) so the suite boots reliably; `npm test --workspace=@voice/app` now runs the harness tests and reproduces the expected hydration failure instead of crashing ahead of collection.
|
||||
- [x] WARN [expo-image-picker] `ImagePicker.MediaTypeOptions` have been deprecated. Use `ImagePicker.MediaType` or an array of `ImagePicker.MediaType` instead.
|
||||
- Replaced the deprecated `MediaTypeOptions.Images` flag with the new literal `["images"]` media type array in `packages/app/src/hooks/use-image-attachment-picker.ts`, silencing the Expo warning when attaching photos. `npm run typecheck --workspace=@voice-dev/app` still fails in the pre-existing stream harness files (unchanged from before this fix).
|
||||
- Replaced the deprecated `MediaTypeOptions.Images` flag with the new literal `["images"]` media type array in `packages/app/src/hooks/use-image-attachment-picker.ts`, silencing the Expo warning when attaching photos. `npm run typecheck --workspace=@voice/app` still fails in the pre-existing stream harness files (unchanged from before this fix).
|
||||
- [x] Update the AgentInput controls so the buttons reflect agent state: when idle show `Dictate` + `Realtime` by default (switch to `Send` when the text box has content); when an agent is running show `Realtime` + `Cancel` regardless of input so you can interrupt without typing. Realtime toggle must remain available in both states.
|
||||
- AgentInput now inspects the agent’s status and swaps the right-hand controls accordingly: send replaces dictate once text/images exist, running agents expose a new Cancel pill, and the realtime toggle is always available (it now starts/stops realtime rather than disappearing). Cancel dispatches a new `cancel_agent_request` websocket message that the server routes through `interruptAgentIfRunning`, so make sure any future API clients send that message when wiring similar controls. (FYI `npm run typecheck --workspace=@voice-dev/app` still fails in the existing stream harness suites for unrelated reasons.)
|
||||
- AgentInput now inspects the agent’s status and swaps the right-hand controls accordingly: send replaces dictate once text/images exist, running agents expose a new Cancel pill, and the realtime toggle is always available (it now starts/stops realtime rather than disappearing). Cancel dispatches a new `cancel_agent_request` websocket message that the server routes through `interruptAgentIfRunning`, so make sure any future API clients send that message when wiring similar controls. (FYI `npm run typecheck --workspace=@voice/app` still fails in the existing stream harness suites for unrelated reasons.)
|
||||
- [x] Implement long press on the agent list item to opena a light bottom sheet with some actions, for now: "Delete agent" all this will do is remove it from our own persistence, but the agent will still exist in the claude/codex persistance and thats out of scope to remove. Its esentially just to remove it from the list, disown it.
|
||||
- Added a `delete_agent_request` flow: the session now closes the agent, removes its registry entry, and emits a dedicated `agent_deleted` event that the client listens for to prune the agents map/stream state. Long-pressing any agent row opens a lightweight bottom sheet with a Delete action wired to that new context method, so deleting disowns the agent locally without touching the Claude/Codex session.
|
||||
- [x] Implement a function `ensureValidJson` that walks an object and validates that all values are valid JSON (no undefined, null, or non-string values). Use this for all return valies in the MCP server tools.
|
||||
- Added `ensureValidJson` (`packages/server/src/server/json-utils.ts`) and wrapped every MCP server tool payload in both `terminal-mcp/server.ts` and `agent/mcp-server.ts`, so tool responses now get validated before streaming; `tsc` still fails earlier in the tree due to `packages/app/src/types/stream.ts` references (see npm run typecheck --workspace=@voice-dev/server).
|
||||
- Added `ensureValidJson` (`packages/server/src/server/json-utils.ts`) and wrapped every MCP server tool payload in both `terminal-mcp/server.ts` and `agent/mcp-server.ts`, so tool responses now get validated before streaming; `tsc` still fails earlier in the tree due to `packages/app/src/types/stream.ts` references (see npm run typecheck --workspace=@voice/server).
|
||||
|
||||
## Voice Interruptions & Streaming
|
||||
|
||||
- [x] Wire interrupt signal on speech start so VAD immediately issues `abort_request` to the server before any audio segment is sent.
|
||||
- Realtime speech detection now sends a `session/abort_request` as soon as VAD confirms speech, ensuring any in-flight LLM turn is interrupted before the buffered audio is uploaded (implemented in `packages/app/src/contexts/realtime-context.tsx` with logging/error handling).
|
||||
- [x] Abort server-side playback/LLM as soon as a realtime audio chunk arrives; set a speech-in-progress flag that pauses new TTS until the turn completes.
|
||||
- Added a `speechInProgress` guard in `packages/server/src/server/session.ts` that triggers on the first realtime chunk, cancels pending TTS playback via the new `TTSManager.cancelPendingPlaybacks`, and immediately routes through the abort flow before buffering audio; the flag is cleared when transcription finishes so the next assistant reply can synthesize speech. TTS generation now skips while the flag is set, and `npm run typecheck --workspace=@voice-dev/server` still fails in pre-existing cross-workspace `packages/app/src/types/stream.ts` imports (unchanged by this fix).
|
||||
- Added a `speechInProgress` guard in `packages/server/src/server/session.ts` that triggers on the first realtime chunk, cancels pending TTS playback via the new `TTSManager.cancelPendingPlaybacks`, and immediately routes through the abort flow before buffering audio; the flag is cleared when transcription finishes so the next assistant reply can synthesize speech. TTS generation now skips while the flag is set, and `npm run typecheck --workspace=@voice/server` still fails in pre-existing cross-workspace `packages/app/src/types/stream.ts` imports (unchanged by this fix).
|
||||
- [x] Interrupt the currently focused agent whenever a realtime user turn begins, ensuring the next prompt starts fresh.
|
||||
- Session context now tracks a `focusedAgentId` that each agent screen sets/clears on mount, and the realtime provider looks at that value to send a `cancel_agent_request` before issuing voice aborts so the next spoken prompt doesn't pile onto a running turn. Attempted `npm run typecheck --workspace=@voice-dev/app` but it still fails in the pre-existing stream harness/tests complaining about `parsed*` fields.
|
||||
- Session context now tracks a `focusedAgentId` that each agent screen sets/clears on mount, and the realtime provider looks at that value to send a `cancel_agent_request` before issuing voice aborts so the next spoken prompt doesn't pile onto a running turn. Attempted `npm run typecheck --workspace=@voice/app` but it still fails in the pre-existing stream harness/tests complaining about `parsed*` fields.
|
||||
- [x] Extend focused-agent tracking to orchestrator mode so realtime interruptions cancel the agent that’s actually active, even when no agent detail screen is open.
|
||||
- Session context now keeps a derived auto-focus pointing at the most recent running agent whenever no explicit agent screen is selected, so realtime speech aborts send `cancel_agent_request` for that agent before streaming audio; agent screens continue to override the focus via `setFocusedAgentId`.
|
||||
- [x] Keep agent runs alive while speaking to the orchestrator: realtime speech should **not** cancel the focused agent by default. Instead, only interrupt when the user explicitly addresses that agent again or issues a cancel command.
|
||||
@@ -71,11 +71,11 @@
|
||||
- [x] Investigate streaming STT/server-side VAD (Whisper or equivalent) and document requirements for production use.
|
||||
- Captured the current audio pipeline, design goals, candidate approaches (self-hosted Whisper vs managed APIs), and production requirements (transport, VAD, streaming STT engine, infra, observability, compliance) in `docs/streaming-stt-vad.md`, along with an implementation phase plan and open questions for the follow-up agent.
|
||||
- [x] Prototype streaming TTS output (chunked `audio_output` messages) and update the player to start playback as soon as chunks arrive.
|
||||
- Split OpenAI TTS responses into chunked `audio_output` payloads via a streaming `TTSManager` pipeline (`packages/server/src/server/agent/tts-manager.ts`, `packages/server/src/server/agent/tts-openai.ts`) that tracks per-chunk acknowledgements, adds utterance IDs, and resolves once every chunk finishes. The app’s session context now keeps audio groups active until the last chunk (or an error) lands so playback begins as soon as each chunk arrives without waiting for the full clip (`packages/app/src/contexts/session-context.tsx`). `npm run typecheck --workspace=@voice-dev/server` and `npm run typecheck --workspace=@voice-dev/app` still fail in the pre-existing cross-workspace stream harness files (see earlier tasks); no new regressions were introduced.
|
||||
- Split OpenAI TTS responses into chunked `audio_output` payloads via a streaming `TTSManager` pipeline (`packages/server/src/server/agent/tts-manager.ts`, `packages/server/src/server/agent/tts-openai.ts`) that tracks per-chunk acknowledgements, adds utterance IDs, and resolves once every chunk finishes. The app’s session context now keeps audio groups active until the last chunk (or an error) lands so playback begins as soon as each chunk arrives without waiting for the full clip (`packages/app/src/contexts/session-context.tsx`). `npm run typecheck --workspace=@voice/server` and `npm run typecheck --workspace=@voice/app` still fail in the pre-existing cross-workspace stream harness files (see earlier tasks); no new regressions were introduced.
|
||||
- [x] Add telemetry for barge-in latency (speech start → playback stop, chunk arrival → LLM abort) to measure improvements per milestone.
|
||||
- `RealtimeProvider` now records when speech detection interrupts active playback and logs `[Telemetry] barge_in.playback_stop_latency` once the app reports audio stopped, while the server's `Session.handleRealtimeSpeechStart` logs `[Telemetry] barge_in.llm_abort_latency` after aborting live LLM runs—giving us measurable client + server latency metrics per turn.
|
||||
- [x] Codex tool calls aren’t visible in the live stream (they only show up after hydrating/refreshing); fix the reducer/rendering path so Codex tool events appear in real time the same way Claude’s do.
|
||||
- Stream reducer now inspects Codex `provider_event` payloads and converts command/file/MCP/web search items into agent tool entries immediately, so the UI shows the pills while the turn is running instead of waiting for hydration; `stream.test.ts` gained coverage proving provider events alone still surface command + MCP calls, and `npm run test --workspace=@voice-dev/app -- src/types/stream.test.ts` passes.
|
||||
- Stream reducer now inspects Codex `provider_event` payloads and converts command/file/MCP/web search items into agent tool entries immediately, so the UI shows the pills while the turn is running instead of waiting for hydration; `stream.test.ts` gained coverage proving provider events alone still surface command + MCP calls, and `npm run test --workspace=@voice/app -- src/types/stream.test.ts` passes.
|
||||
- [x] Update the AgentInput “Cancel” control to use a square stop icon button (consistent with media players) so interrupting an agent is visually distinct from other actions.
|
||||
- Replaced the text-based pill with a compact circular stop control that reuses the Square media glyph (plus accessibility labels), making the interrupt action visually distinct without impacting the other pending AgentInput tweaks.
|
||||
- [x] When the agent input has text and we’re not in the cancellable state, replace both the mic (Dictate) and Realtime buttons with a single Send button—only surface Dictate/Realtime when the input is empty or the agent is running.
|
||||
@@ -85,7 +85,8 @@
|
||||
- [x] npm run typecheck and fix all problems
|
||||
- Split the server build/typecheck configs so the typechecker can include the app stream helpers without breaking builds, added path aliases for `@server/*`, and tightened the shared stream types (status unions + `isAgentToolCallItem`) so the server e2e suites can import them safely. Cleaned up every failing app/server test by narrowing tool-call payloads via the helper, updated the harness + Codex/Claude specs, and re-ran `npm run typecheck` (app + server) to green.
|
||||
- [x] audit codebase for unecessary untyped code, hacks, casts, and `any`, type things properly
|
||||
- Tightened the Codex rollout parser typings so permission requests, rollout/event payloads, plan arguments, and JSON helpers all operate on well-defined TypeScript structures instead of `any`. Added explicit payload/entry guards, removed unsafe casts throughout `packages/server/src/server/agent/providers/codex-agent.ts`, and re-ran `npm run typecheck --workspace=@voice-dev/server` to validate the stricter typing.
|
||||
- Tightened the Codex rollout parser typings so permission requests, rollout/event payloads, plan arguments, and JSON helpers all operate on well-defined TypeScript structures instead of `any`. Added explicit payload/entry guards, removed unsafe casts throughout `packages/server/src/server/agent/providers/codex-agent.ts`, and re-ran `npm run typecheck --workspace=@voice/server` to validate the stricter typing.
|
||||
- [x] audit codebase for duplicated code, clean it up
|
||||
- Consolidated the duplicated home/back header layouts into a shared `ScreenHeader` component so safe-area padding, borders, and spacing live in one place; both headers now only define their unique button/title content. Ran `npm run typecheck --workspace=@voice-dev/app` to confirm the refactor stays type-safe.
|
||||
- [ ] rename the project to Paseo, including the Expo app, package names etc.
|
||||
- Consolidated the duplicated home/back header layouts into a shared `ScreenHeader` component so safe-area padding, borders, and spacing live in one place; both headers now only define their unique button/title content. Ran `npm run typecheck --workspace=@voice/app` to confirm the refactor stays type-safe.
|
||||
- [x] rename the project to Paseo, including the Expo app, package names etc.
|
||||
- Renamed the monorepo + workspaces to `paseo`, updated Expo config (name/slug/schemes + bundle IDs), AsyncStorage keys, docs, prompts, and helper text to use `@paseo/*`, and refreshed the CLI scripts + descriptions so every entry point references Paseo instead of Voice Dev. Regenerated the lockfile + pods metadata and re-ran `npm run typecheck --workspaces --if-present` to confirm both server and app builds still typecheck under the new branding.
|
||||
|
||||
Reference in New Issue
Block a user