Compare commits

...

27 Commits

Author SHA1 Message Date
Mohamed Boudra
c45b414476 chore(release): cut 0.1.27 2026-03-13 21:09:06 +07:00
Mohamed Boudra
73bd0c1dba docs: add 0.1.27 changelog 2026-03-13 21:08:53 +07:00
Mohamed Boudra
502dd2d2e2 refactor(app): remove debug logging from voice and UI components 2026-03-13 21:07:18 +07:00
Mohamed Boudra
80a2f8052c refactor(app): replace voice hooks with voice runtime and audio engine architecture 2026-03-13 20:53:55 +07:00
Mohamed Boudra
730f34753a fix(app): use platform-specific components for markdown links 2026-03-13 13:27:24 +07:00
Mohamed Boudra
013f573cff refactor: migrate to shell-env and custom desktop notifications bridge 2026-03-13 12:26:14 +07:00
Mohamed Boudra
ee8a81d2e1 feat(server): support Grep tool in Claude tool-call mapping 2026-03-13 10:32:33 +07:00
Mohamed Boudra
bed03be332 feat(app): support file:// protocol URLs for opening workspace files 2026-03-13 10:24:01 +07:00
Mohamed Boudra
debe4b543b fix(desktop): resolve linux appimage cli resources 2026-03-13 09:58:01 +07:00
Mohamed Boudra
1c601e09b6 fix(release): repair linux appimage symlinks 2026-03-13 09:38:59 +07:00
Mohamed Boudra
ba54aae206 fix(server): pin sherpa runtime deps 2026-03-13 08:48:28 +07:00
Mohamed Boudra
36961f83bf fix(release): recover desktop r8 failures 2026-03-12 23:49:07 +07:00
Mohamed Boudra
6db99c03cb feat(app): allow agents to open workspace files from chat 2026-03-12 23:43:59 +07:00
Mohamed Boudra
999d100464 fix(release): recover desktop runtime packaging 2026-03-12 23:33:50 +07:00
Mohamed Boudra
9190d86148 feat(app): improve image picker, markdown rendering, and UI interactions 2026-03-12 23:29:26 +07:00
Mohamed Boudra
4b5cb5a3e2 fix(release): recover desktop packaging artifacts 2026-03-12 23:20:26 +07:00
Mohamed Boudra
847e1d2c60 fix(app): avoid metro exclusionList breakage on windows 2026-03-12 23:06:36 +07:00
Mohamed Boudra
970df6478b fix(app): make metro blocklist regex windows-safe 2026-03-12 22:56:46 +07:00
Mohamed Boudra
3371f17606 fix(release): add manual linux appimage fallback 2026-03-12 22:55:18 +07:00
Mohamed Boudra
0f65fe894c fix(release): preload metro patch inline on windows 2026-03-12 22:52:15 +07:00
Mohamed Boudra
2ac27a3e3e fix(release): support iterative desktop recovery tags 2026-03-12 22:48:26 +07:00
Mohamed Boudra
defcc54af2 fix(release): harden desktop recovery workflows 2026-03-12 22:41:34 +07:00
Mohamed Boudra
fe84c5c9a3 Match explorer files pane sidebar background 2026-03-12 22:34:42 +07:00
Mohamed Boudra
2ff8041e15 Fix hidden sidebar workspace shortcuts 2026-03-12 22:33:27 +07:00
Mohamed Boudra
1d3e551e7f fix(release): recover desktop release builds 2026-03-12 22:26:44 +07:00
Mohamed Boudra
79795b6f49 Update outdated Paseo skill CLI options 2026-03-12 22:26:10 +07:00
Mohamed Boudra
16da3300e3 fix: prevent Codex replacement stream from being killed by stale turn_completed notification
The turn_completed notification handler called eventQueue.end(), which
could terminate a replacement stream's queue if the old interrupted
turn's completion arrived after the new stream had set up its queue.
The streamInternal loop already breaks on terminal events, making the
end() call redundant and unsafe during replaceAgentRun.
2026-03-12 22:24:41 +07:00
114 changed files with 5376 additions and 3653 deletions

View File

@@ -59,13 +59,11 @@ jobs:
run: |
set -euo pipefail
source_tag="${SOURCE_TAG}"
release_tag="$source_tag"
for prefix in desktop-windows-v desktop-linux-v desktop-macos-v desktop-v; do
if [[ "$source_tag" == ${prefix}* ]]; then
release_tag="v${source_tag#${prefix}}"
break
fi
done
if [[ "$source_tag" =~ ^(desktop-(windows|linux|macos)-|desktop-)?v([0-9]+\.[0-9]+\.[0-9]+) ]]; then
release_tag="v${BASH_REMATCH[3]}"
else
release_tag="$source_tag"
fi
echo "RELEASE_TAG=$release_tag" >> "$GITHUB_ENV"
version="${release_tag#v}"
@@ -253,13 +251,11 @@ jobs:
run: |
set -euo pipefail
source_tag="${SOURCE_TAG}"
release_tag="$source_tag"
for prefix in desktop-windows-v desktop-linux-v desktop-macos-v desktop-v; do
if [[ "$source_tag" == ${prefix}* ]]; then
release_tag="v${source_tag#${prefix}}"
break
fi
done
if [[ "$source_tag" =~ ^(desktop-(windows|linux|macos)-|desktop-)?v([0-9]+\.[0-9]+\.[0-9]+) ]]; then
release_tag="v${BASH_REMATCH[3]}"
else
release_tag="$source_tag"
fi
echo "RELEASE_TAG=$release_tag" >> "$GITHUB_ENV"
version="${release_tag#v}"
@@ -369,23 +365,87 @@ jobs:
fi
echo "RELEASE_DRAFT=$release_draft" >> "$GITHUB_ENV"
- name: Build and publish Linux Tauri release
- name: Build Linux Tauri release
if: env.IS_SMOKE_TAG != 'true'
uses: tauri-apps/tauri-action@v0
id: linux_tauri
continue-on-error: true
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }}
TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }}
NO_STRIP: "true"
NO_STRIP: "1"
APPIMAGE_EXTRACT_AND_RUN: "1"
with:
projectPath: packages/desktop
tagName: ${{ env.RELEASE_TAG }}
releaseName: Paseo ${{ env.RELEASE_TAG }}
releaseBody: See the assets to download and install this version.
releaseDraft: ${{ env.RELEASE_DRAFT }}
prerelease: false
args: --bundles appimage
shell: bash
run: |
set -euo pipefail
npm run tauri --workspace=@getpaseo/desktop build -- --bundles appimage
- name: Attempt manual Linux AppImage fallback
if: env.IS_SMOKE_TAG != 'true' && steps.linux_tauri.outcome == 'failure'
env:
TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }}
TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }}
shell: bash
run: |
set -euxo pipefail
appimage_dir="packages/desktop/src-tauri/target/release/bundle/appimage"
appdir_path="$appimage_dir/Paseo.AppDir"
canonical_appimage="$appimage_dir/Paseo_${DESKTOP_VERSION}_amd64.AppImage"
existing_appimage="$(find "$appimage_dir" -maxdepth 1 -type f -name '*.AppImage' | head -n 1)"
if [ -n "$existing_appimage" ]; then
if [ "$existing_appimage" != "$canonical_appimage" ]; then
mv "$existing_appimage" "$canonical_appimage"
fi
if [ ! -f "$canonical_appimage.sig" ]; then
npx tauri signer sign "$canonical_appimage"
fi
exit 0
fi
if [ ! -d "$appdir_path" ]; then
echo "::error::AppDir was not generated at $appdir_path"
exit 1
fi
cp --remove-destination "$appdir_path/usr/share/applications/Paseo.desktop" "$appdir_path/Paseo.desktop"
cp --remove-destination "$appdir_path/Paseo.png" "$appdir_path/.DirIcon"
env | sort | grep -E '^(APPIMAGE|DESKTOP_VERSION|NO_STRIP|RELEASE_TAG|SOURCE_TAG|TAURI_)' || true
tools_dir="$(mktemp -d)"
curl -fsSL https://github.com/AppImage/appimagetool/releases/download/continuous/appimagetool-x86_64.AppImage -o "$tools_dir/appimagetool-x86_64.AppImage"
chmod +x "$tools_dir/appimagetool-x86_64.AppImage"
ARCH=x86_64 APPIMAGE_EXTRACT_AND_RUN=1 "$tools_dir/appimagetool-x86_64.AppImage" "$appdir_path" "$canonical_appimage"
npx tauri signer sign "$canonical_appimage"
- name: Fail Linux release when AppImage bundling fails
if: env.IS_SMOKE_TAG != 'true' && steps.linux_tauri.outcome == 'failure'
shell: bash
run: |
set -euo pipefail
shopt -s nullglob
assets=(
packages/desktop/src-tauri/target/release/bundle/appimage/*.AppImage
packages/desktop/src-tauri/target/release/bundle/appimage/*.AppImage.sig
)
if [ "${#assets[@]}" -eq 0 ]; then
echo "::error::Linux AppImage assets are still missing after the manual fallback."
exit 1
fi
- name: Upload Linux release assets
if: env.IS_SMOKE_TAG != 'true'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
shell: bash
run: |
set -euo pipefail
shopt -s nullglob
assets=(
packages/desktop/src-tauri/target/release/bundle/appimage/*.AppImage
packages/desktop/src-tauri/target/release/bundle/appimage/*.AppImage.sig
)
if [ "${#assets[@]}" -eq 0 ]; then
echo "::error::No Linux AppImage assets were produced."
exit 1
fi
printf 'Uploading Linux assets:\n%s\n' "${assets[@]}"
gh release upload "$RELEASE_TAG" "${assets[@]}" --repo "${{ github.repository }}" --clobber
- name: Build Linux app (smoke only)
if: env.IS_SMOKE_TAG == 'true'
@@ -409,13 +469,11 @@ jobs:
run: |
set -euo pipefail
source_tag="${SOURCE_TAG}"
release_tag="$source_tag"
for prefix in desktop-windows-v desktop-linux-v desktop-macos-v desktop-v; do
if [[ "$source_tag" == ${prefix}* ]]; then
release_tag="v${source_tag#${prefix}}"
break
fi
done
if [[ "$source_tag" =~ ^(desktop-(windows|linux|macos)-|desktop-)?v([0-9]+\.[0-9]+\.[0-9]+) ]]; then
release_tag="v${BASH_REMATCH[3]}"
else
release_tag="$source_tag"
fi
echo "RELEASE_TAG=$release_tag" >> "$GITHUB_ENV"
version="${release_tag#v}"
@@ -485,7 +543,11 @@ jobs:
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Build web app for Tauri
run: npm run build:web --workspace=@getpaseo/app
shell: pwsh
run: |
$patchPath = (Get-Item "$env:GITHUB_WORKSPACE/scripts/metro-config-windows-loader-patch.cjs").FullName
$env:NODE_OPTIONS = "--require=$patchPath"
npm run build:web --workspace=@getpaseo/app
- name: Build managed runtime
run: npm run prepare:managed-runtime --workspace=@getpaseo/desktop

View File

@@ -1,5 +1,22 @@
# Changelog
## 0.1.27 - 2026-03-13
### Added
- Added voice runtime with new audio engine architecture for voice interactions.
- Added Grep tool support in Claude tool-call mapping.
- Added ability to open workspace files directly from agent chat messages.
- Added desktop notifications via a custom native bridge.
### Improved
- Improved image picker, markdown rendering, and UI interactions.
- Improved shell environment detection using shell-env.
### Fixed
- Fixed platform-specific markdown link rendering.
- Fixed Linux AppImage CLI resource paths.
- Fixed Codex replacement stream being killed by stale turn notifications.
## 0.1.26 - 2026-03-12
### Added

438
package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "paseo",
"version": "0.1.26",
"version": "0.1.27",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "paseo",
"version": "0.1.26",
"version": "0.1.27",
"hasInstallScript": true,
"license": "AGPL-3.0-or-later",
"workspaces": [
@@ -6402,128 +6402,6 @@
"node": ">=20.0"
}
},
"node_modules/@openai/codex": {
"version": "0.114.0",
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.114.0.tgz",
"integrity": "sha512-HMo8LRR6CtfKkaa28xvFK6eOarmBFTDfsrS9GJtEoaspGGemFok494CpafDspiTZaHZZGHfSUe5SWTUxYq7OaA==",
"license": "Apache-2.0",
"bin": {
"codex": "bin/codex.js"
},
"engines": {
"node": ">=16"
},
"optionalDependencies": {
"@openai/codex-darwin-arm64": "npm:@openai/codex@0.114.0-darwin-arm64",
"@openai/codex-darwin-x64": "npm:@openai/codex@0.114.0-darwin-x64",
"@openai/codex-linux-arm64": "npm:@openai/codex@0.114.0-linux-arm64",
"@openai/codex-linux-x64": "npm:@openai/codex@0.114.0-linux-x64",
"@openai/codex-win32-arm64": "npm:@openai/codex@0.114.0-win32-arm64",
"@openai/codex-win32-x64": "npm:@openai/codex@0.114.0-win32-x64"
}
},
"node_modules/@openai/codex-darwin-arm64": {
"name": "@openai/codex",
"version": "0.114.0-darwin-arm64",
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.114.0-darwin-arm64.tgz",
"integrity": "sha512-c9dlgo9O+66alr3s3G36fKE3KaO2+xrLZ/QcU3oujDruV86pqgVSEK2njHi+WIyyOa6m2AyWxdaHLEbKxPCNRg==",
"cpu": [
"arm64"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">=16"
}
},
"node_modules/@openai/codex-darwin-x64": {
"name": "@openai/codex",
"version": "0.114.0-darwin-x64",
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.114.0-darwin-x64.tgz",
"integrity": "sha512-oPAPeZSaJZ1AQneFPSJu44uqbZr63Bxut9FOTWZwuSqYx4YEees7i0HJmJcqQc0XnCbYtHJdqo/i07Uv374NLw==",
"cpu": [
"x64"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">=16"
}
},
"node_modules/@openai/codex-linux-arm64": {
"name": "@openai/codex",
"version": "0.114.0-linux-arm64",
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.114.0-linux-arm64.tgz",
"integrity": "sha512-oUFZGZgMiEqmo85C+dfhckpVApH25alLWfwBgfKr2IRgdx/tovy8vN101f6kj01E6nDDe4LfJdNSZGtZht4fRA==",
"cpu": [
"arm64"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=16"
}
},
"node_modules/@openai/codex-linux-x64": {
"name": "@openai/codex",
"version": "0.114.0-linux-x64",
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.114.0-linux-x64.tgz",
"integrity": "sha512-MF6RhxfBoccccd5CYII2C7TL2TvYzombBQhanDZfQAajr6n7IuoazCmoHDlrFHS4AcSw9bYA4MRlrwEjbEjgsw==",
"cpu": [
"x64"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=16"
}
},
"node_modules/@openai/codex-win32-arm64": {
"name": "@openai/codex",
"version": "0.114.0-win32-arm64",
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.114.0-win32-arm64.tgz",
"integrity": "sha512-CnRMHopj3en9aqQ2UaDW7EgpEGkxHdZVLLRq2cOy5D0HyuzF6Qb6595ADoFVJHEmPeN5Iz/KUbiGs5GLDjUwOA==",
"cpu": [
"arm64"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=16"
}
},
"node_modules/@openai/codex-win32-x64": {
"name": "@openai/codex",
"version": "0.114.0-win32-x64",
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.114.0-win32-x64.tgz",
"integrity": "sha512-ztRsH5Z+gPVZ5ZInx6HzXsTFFkuEdjk3Ay0UGVOhRRCWvevXQi/SL4eU8hwysCYrs8K/WRq3mo4c95w9zQ2wLw==",
"cpu": [
"x64"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=16"
}
},
"node_modules/@opencode-ai/sdk": {
"version": "1.2.6",
"resolved": "https://registry.npmjs.org/@opencode-ai/sdk/-/sdk-1.2.6.tgz",
@@ -12835,6 +12713,18 @@
"node": ">=0.10.0"
}
},
"node_modules/default-shell": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/default-shell/-/default-shell-2.2.0.tgz",
"integrity": "sha512-sPpMZcVhRQ0nEMDtuMJ+RtCxt7iHPAMBU+I4tAlo5dU1sjRpNax0crj6nR3qKpvVnckaQ9U38enXcwW9nZJeCw==",
"license": "MIT",
"engines": {
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/defaults": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.4.tgz",
@@ -14462,6 +14352,35 @@
"dev": true,
"license": "MIT"
},
"node_modules/execa": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz",
"integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==",
"license": "MIT",
"dependencies": {
"cross-spawn": "^7.0.3",
"get-stream": "^6.0.0",
"human-signals": "^2.1.0",
"is-stream": "^2.0.0",
"merge-stream": "^2.0.0",
"npm-run-path": "^4.0.1",
"onetime": "^5.1.2",
"signal-exit": "^3.0.3",
"strip-final-newline": "^2.0.0"
},
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sindresorhus/execa?sponsor=1"
}
},
"node_modules/execa/node_modules/signal-exit": {
"version": "3.0.7",
"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",
"integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==",
"license": "ISC"
},
"node_modules/expect-type": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz",
@@ -17102,6 +17021,18 @@
"node": ">= 0.4"
}
},
"node_modules/get-stream": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
"integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==",
"license": "MIT",
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/get-symbol-description": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.1.0.tgz",
@@ -17686,6 +17617,15 @@
"node": ">= 6"
}
},
"node_modules/human-signals": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz",
"integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==",
"license": "Apache-2.0",
"engines": {
"node": ">=10.17.0"
}
},
"node_modules/humanize-ms": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
@@ -18346,7 +18286,6 @@
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz",
"integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=8"
@@ -20787,7 +20726,6 @@
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz",
"integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=6"
@@ -21368,6 +21306,18 @@
"integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==",
"license": "ISC"
},
"node_modules/npm-run-path": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz",
"integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==",
"license": "MIT",
"dependencies": {
"path-key": "^3.0.0"
},
"engines": {
"node": ">=8"
}
},
"node_modules/nth-check": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz",
@@ -21586,7 +21536,6 @@
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz",
"integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==",
"dev": true,
"license": "MIT",
"dependencies": {
"mimic-fn": "^2.1.0"
@@ -21682,161 +21631,6 @@
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
"license": "MIT"
},
"node_modules/opencode-ai": {
"version": "1.2.24",
"resolved": "https://registry.npmjs.org/opencode-ai/-/opencode-ai-1.2.24.tgz",
"integrity": "sha512-LaSoATkVEF6jyXNnAPrkqYpmHsZfuf2uPLDSEGJkL4hT6HYFq8LkoknHDjCGIiE5Be3MmId4+l17eSIKDfmddw==",
"hasInstallScript": true,
"license": "MIT",
"bin": {
"opencode": "bin/opencode"
},
"optionalDependencies": {
"opencode-darwin-arm64": "1.2.24",
"opencode-darwin-x64": "1.2.24",
"opencode-darwin-x64-baseline": "1.2.24",
"opencode-linux-arm64": "1.2.24",
"opencode-linux-arm64-musl": "1.2.24",
"opencode-linux-x64": "1.2.24",
"opencode-linux-x64-baseline": "1.2.24",
"opencode-linux-x64-baseline-musl": "1.2.24",
"opencode-linux-x64-musl": "1.2.24",
"opencode-windows-x64": "1.2.24",
"opencode-windows-x64-baseline": "1.2.24"
}
},
"node_modules/opencode-darwin-arm64": {
"version": "1.2.24",
"resolved": "https://registry.npmjs.org/opencode-darwin-arm64/-/opencode-darwin-arm64-1.2.24.tgz",
"integrity": "sha512-mk5AkNbmATLwjnczd09Fu9dLTPcW4GSlRlIGWJBbLuGSPe7DWfbuhA5tj00VuaGeC09wDKVnW06BxWp/gRzgMQ==",
"cpu": [
"arm64"
],
"optional": true,
"os": [
"darwin"
]
},
"node_modules/opencode-darwin-x64": {
"version": "1.2.24",
"resolved": "https://registry.npmjs.org/opencode-darwin-x64/-/opencode-darwin-x64-1.2.24.tgz",
"integrity": "sha512-SuG7ItUejx42z9Bz46t0R3LLFIM34vZUcwJAkqqpS6MRd1KV9G4OQIJl9BR+Cwwvk70iqCnoDxj5NgFZjeeNOw==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"darwin"
]
},
"node_modules/opencode-darwin-x64-baseline": {
"version": "1.2.24",
"resolved": "https://registry.npmjs.org/opencode-darwin-x64-baseline/-/opencode-darwin-x64-baseline-1.2.24.tgz",
"integrity": "sha512-+MT/ZkgXxeID/kOcT04SBn8Vg+4tk8CytKldaNDLgUnsi/78vTxDXWs0Cc21Ohc894r45fEtOOb5npSNIMOVfQ==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"darwin"
]
},
"node_modules/opencode-linux-arm64": {
"version": "1.2.24",
"resolved": "https://registry.npmjs.org/opencode-linux-arm64/-/opencode-linux-arm64-1.2.24.tgz",
"integrity": "sha512-zSpbVCwVfUQ/Bey1MEGkJg3A52EiIPgnpS44HwmSANkiLPVXRBVuwbFYN2qJ7eJVAH939K9+QPmDOW4X8DWKeQ==",
"cpu": [
"arm64"
],
"optional": true,
"os": [
"linux"
]
},
"node_modules/opencode-linux-arm64-musl": {
"version": "1.2.24",
"resolved": "https://registry.npmjs.org/opencode-linux-arm64-musl/-/opencode-linux-arm64-musl-1.2.24.tgz",
"integrity": "sha512-LY28mspJOJ2s32E0AbrfbdWH5Bw+5YQkuB3ACJiVxW1MafjhRgPOsXjnw38WMiFjZ+NbhSPBOKnL0guKseVd4Q==",
"cpu": [
"arm64"
],
"optional": true,
"os": [
"linux"
]
},
"node_modules/opencode-linux-x64": {
"version": "1.2.24",
"resolved": "https://registry.npmjs.org/opencode-linux-x64/-/opencode-linux-x64-1.2.24.tgz",
"integrity": "sha512-OM3umRJ2ZpDe4YtDOGp32cQCqvMPuFJBKKngfvWL4zz4EWDzLPttb0sNL7xdC1xR/XidtADRYcvtzc2rr7/4Rw==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"linux"
]
},
"node_modules/opencode-linux-x64-baseline": {
"version": "1.2.24",
"resolved": "https://registry.npmjs.org/opencode-linux-x64-baseline/-/opencode-linux-x64-baseline-1.2.24.tgz",
"integrity": "sha512-yakykM+GxCMZ5WK2G0CQk5cGJkoKt09A/8Y0bhKy7l+Jc/QiTV2ZIlBwT5FO15dnuGlNH6zvukLyOQvOZinM+Q==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"linux"
]
},
"node_modules/opencode-linux-x64-baseline-musl": {
"version": "1.2.24",
"resolved": "https://registry.npmjs.org/opencode-linux-x64-baseline-musl/-/opencode-linux-x64-baseline-musl-1.2.24.tgz",
"integrity": "sha512-XHPuf9Rj9nWkVX7JJZ5NtYx9UWT0hPiem/kAhZzYITseRGmQMMfFAhIOBYYvvlzJzgdhcrLi198kJHaKCdXjag==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"linux"
]
},
"node_modules/opencode-linux-x64-musl": {
"version": "1.2.24",
"resolved": "https://registry.npmjs.org/opencode-linux-x64-musl/-/opencode-linux-x64-musl-1.2.24.tgz",
"integrity": "sha512-nFT4J4m4wKiZh4RsfgQJlW6W2w/VtMNnXnMiQk93FnncSf0nUXEB5Nam3P+vCjdR3i4cs355JiOvbshnMiJGSA==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"linux"
]
},
"node_modules/opencode-windows-x64": {
"version": "1.2.24",
"resolved": "https://registry.npmjs.org/opencode-windows-x64/-/opencode-windows-x64-1.2.24.tgz",
"integrity": "sha512-7nkwFgS8Jn35kgw+6JIWRU6PmbggSuTDiQ+RnFGoySFgYW/Sz/R30Z4KmnbfJLElzEef0kX+nZQasfVl0pSf6A==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"win32"
]
},
"node_modules/opencode-windows-x64-baseline": {
"version": "1.2.24",
"resolved": "https://registry.npmjs.org/opencode-windows-x64-baseline/-/opencode-windows-x64-baseline-1.2.24.tgz",
"integrity": "sha512-hXp1nPnkrHgM6cid1AogRJvjz931UbZ/G165B6AAR1gOb5EqtIh2gZcT0IRUyjooEpwH7+oGFRhT+3Kql/M0WA==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"win32"
]
},
"node_modules/optionator": {
"version": "0.9.4",
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz",
@@ -24688,6 +24482,50 @@
"node": ">=8"
}
},
"node_modules/shell-env": {
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/shell-env/-/shell-env-4.0.3.tgz",
"integrity": "sha512-Ioe5h+hCDZ7pKL5+JGzbtPvZ5ESMHePZ8nLxohlDL+twmlcmutttMhRkrQOed8DeLT8mkYBgbwZfohe8pqaA3g==",
"license": "MIT",
"dependencies": {
"default-shell": "^2.0.0",
"execa": "^5.1.1",
"strip-ansi": "^7.0.1"
},
"engines": {
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/shell-env/node_modules/ansi-regex": {
"version": "6.2.2",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz",
"integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==",
"license": "MIT",
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/chalk/ansi-regex?sponsor=1"
}
},
"node_modules/shell-env/node_modules/strip-ansi": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz",
"integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==",
"license": "MIT",
"dependencies": {
"ansi-regex": "^6.2.2"
},
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/chalk/strip-ansi?sponsor=1"
}
},
"node_modules/shell-quote": {
"version": "1.8.3",
"resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.3.tgz",
@@ -25427,6 +25265,15 @@
"node": ">=4"
}
},
"node_modules/strip-final-newline": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz",
"integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==",
"license": "MIT",
"engines": {
"node": ">=6"
}
},
"node_modules/strip-indent": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-4.1.1.tgz",
@@ -27847,7 +27694,7 @@
},
"packages/app": {
"name": "@getpaseo/app",
"version": "0.1.26",
"version": "0.1.27",
"dependencies": {
"@boudra/expo-two-way-audio": "^0.1.3",
"@dnd-kit/core": "^6.3.1",
@@ -27855,7 +27702,7 @@
"@dnd-kit/utilities": "^3.2.2",
"@expo/vector-icons": "^15.0.2",
"@floating-ui/react-native": "^0.10.7",
"@getpaseo/server": "0.1.26",
"@getpaseo/server": "0.1.27",
"@gorhom/bottom-sheet": "^5.2.6",
"@gorhom/portal": "^1.0.14",
"@lezer/common": "^1.5.0",
@@ -27994,11 +27841,11 @@
},
"packages/cli": {
"name": "@getpaseo/cli",
"version": "0.1.26",
"version": "0.1.27",
"dependencies": {
"@clack/prompts": "^1.0.0",
"@getpaseo/relay": "0.1.26",
"@getpaseo/server": "0.1.26",
"@getpaseo/relay": "0.1.27",
"@getpaseo/server": "0.1.27",
"chalk": "^5.3.0",
"commander": "^12.0.0",
"mime-types": "^2.1.35",
@@ -28035,14 +27882,14 @@
},
"packages/desktop": {
"name": "@getpaseo/desktop",
"version": "0.1.26",
"version": "0.1.27",
"devDependencies": {
"@tauri-apps/cli": "^2.9.6"
}
},
"packages/relay": {
"name": "@getpaseo/relay",
"version": "0.1.26",
"version": "0.1.27",
"dependencies": {
"base64-js": "^1.5.1",
"tweetnacl": "^1.0.3",
@@ -28058,12 +27905,12 @@
},
"packages/server": {
"name": "@getpaseo/server",
"version": "0.1.26",
"version": "0.1.27",
"dependencies": {
"@ai-sdk/openai": "2.0.52",
"@anthropic-ai/claude-agent-sdk": "^0.2.11",
"@deepgram/sdk": "^3.4.0",
"@getpaseo/relay": "0.1.26",
"@getpaseo/relay": "0.1.27",
"@lezer/common": "^1.5.0",
"@lezer/css": "^1.3.0",
"@lezer/highlight": "^1.2.3",
@@ -28073,7 +27920,6 @@
"@lezer/markdown": "^1.6.2",
"@lezer/python": "^1.1.18",
"@modelcontextprotocol/sdk": "^1.20.1",
"@openai/codex": "^0.114.0",
"@opencode-ai/sdk": "1.2.6",
"@sctg/sentencepiece-js": "^1.1.0",
"@xterm/headless": "^6.0.0",
@@ -28088,13 +27934,13 @@
"node-pty": "1.2.0-beta.11",
"onnxruntime-node": "^1.23.0",
"openai": "^4.20.0",
"opencode-ai": "^1.2.24",
"pino": "^10.2.0",
"pino-pretty": "^13.1.3",
"qrcode": "^1.5.4",
"rotating-file-stream": "^3.2.9",
"sherpa-onnx": "^1.12.23",
"sherpa-onnx-node": "^1.12.23",
"shell-env": "^4.0.3",
"sherpa-onnx": "1.12.28",
"sherpa-onnx-node": "1.12.28",
"strip-ansi": "^7.1.2",
"tiny-invariant": "^1.3.3",
"uuid": "^9.0.1",
@@ -28421,7 +28267,7 @@
},
"packages/website": {
"name": "@getpaseo/website",
"version": "0.1.26",
"version": "0.1.27",
"dependencies": {
"@cloudflare/vite-plugin": "^1.20.3",
"@cloudflare/workers-types": "^4.20260114.0",

View File

@@ -1,6 +1,6 @@
{
"name": "paseo",
"version": "0.1.26",
"version": "0.1.27",
"private": true,
"workspaces": [
"packages/server",

Binary file not shown.

View File

@@ -1,6 +1,4 @@
const { getDefaultConfig } = require("expo/metro-config");
const exclusionList =
require("@expo/metro/metro-config/defaults/exclusionList").default;
const { resolve } = require("metro-resolver");
const fs = require("fs");
const path = require("path");
@@ -18,8 +16,10 @@ const customWebPlatform = (process.env.PASEO_WEB_PLATFORM ?? "")
const config = getDefaultConfig(projectRoot);
const defaultResolveRequest = config.resolver.resolveRequest ?? resolve;
const escapedAppSrcRoot = appSrcRoot
.replace(/[|\\{}()[\]^$+*?.]/g, "\\$&")
.replace(/\//g, "[\\\\/]");
.split(path.sep)
.map((segment) => segment.replace(/[|\\{}()[\]^$+*?.]/g, "\\$&"))
.join("[\\\\/]");
const pathSeparatorPattern = "[\\\\/]";
config.resolver.extraNodeModules = {
...(config.resolver.extraNodeModules ?? {}),
@@ -28,9 +28,9 @@ config.resolver.extraNodeModules = {
"react/jsx-runtime": path.join(appNodeModulesRoot, "react/jsx-runtime"),
"react/jsx-dev-runtime": path.join(appNodeModulesRoot, "react/jsx-dev-runtime"),
};
config.resolver.blockList = exclusionList([
new RegExp(`^${escapedAppSrcRoot}[\\\\/].*\\.(test|spec)\\.(ts|tsx)$`),
]);
config.resolver.blockList = new RegExp(
`(^${escapedAppSrcRoot}${pathSeparatorPattern}.*\\.(test|spec)\\.(ts|tsx)$|${pathSeparatorPattern}__tests__${pathSeparatorPattern}.*)$`,
);
function isLocalModuleImport(moduleName) {
return (

View File

@@ -1,7 +1,7 @@
{
"name": "@getpaseo/app",
"main": "index.ts",
"version": "0.1.26",
"version": "0.1.27",
"private": true,
"scripts": {
"start": "expo start",
@@ -33,7 +33,7 @@
"@dnd-kit/utilities": "^3.2.2",
"@expo/vector-icons": "^15.0.2",
"@floating-ui/react-native": "^0.10.7",
"@getpaseo/server": "0.1.26",
"@getpaseo/server": "0.1.27",
"@gorhom/bottom-sheet": "^5.2.6",
"@gorhom/portal": "^1.0.14",
"@lezer/common": "^1.5.0",

View File

@@ -50,6 +50,7 @@ import { getIsTauri } from "@/constants/layout";
import { CommandCenter } from "@/components/command-center";
import { ProjectPickerModal } from "@/components/project-picker-modal";
import { KeyboardShortcutsDialog } from "@/components/keyboard-shortcuts-dialog";
import { listenToDesktopNotificationClicks } from "@/desktop/notifications/desktop-notifications";
import { useKeyboardShortcuts } from "@/hooks/use-keyboard-shortcuts";
import { queryClient } from "@/query/query-client";
import {
@@ -70,18 +71,6 @@ polyfillCrypto();
attachConsole();
const HostRuntimeBootstrapContext = createContext(false);
const IS_DEV = Boolean((globalThis as { __DEV__?: boolean }).__DEV__);
function logLeftSidebarOpenGesture(
event: string,
details: Record<string, unknown>
): void {
if (!IS_DEV) {
return;
}
console.log(`[LeftSidebarOpenGesture] ${event}`, details);
}
function PushNotificationRouter() {
const router = useRouter();
const lastHandledIdRef = useRef<string | null>(null);
@@ -89,20 +78,39 @@ function PushNotificationRouter() {
useEffect(() => {
if (Platform.OS === "web") {
if (getTauri()) {
void ensureOsNotificationPermission().then((granted) => {
console.log(
"[OSNotifications][Tauri] Startup permission preflight result:",
granted ? "granted" : "not-granted"
);
});
void ensureOsNotificationPermission();
let disposed = false;
let unlisten: (() => void) | null = null;
void listenToDesktopNotificationClicks((payload) => {
router.push(buildNotificationRoute(payload.data) as any);
})
.then((cleanup) => {
if (disposed) {
cleanup();
return;
}
unlisten = cleanup;
})
.catch((error) => {
console.error(
"[OSNotifications][Desktop] Failed to register notification click listener",
error
);
});
return () => {
disposed = true;
unlisten?.();
};
}
const target = globalThis as unknown as EventTarget;
const openFromWebClick = (event: Event) => {
const customEvent = event as CustomEvent<WebNotificationClickDetail>;
const route = buildNotificationRoute(customEvent.detail?.data);
event.preventDefault();
router.push(route as any);
router.push(buildNotificationRoute(customEvent.detail?.data) as any);
};
target.addEventListener(
@@ -319,10 +327,6 @@ function AppContainer({
})
.onStart(() => {
isGesturing.value = true;
runOnJS(logLeftSidebarOpenGesture)("start", {
mobileView,
openGestureEnabled,
});
})
.onUpdate((event) => {
// Start from closed position (-windowWidth) and move towards 0
@@ -339,13 +343,6 @@ function AppContainer({
isGesturing.value = false;
// Open if dragged more than 1/3 of sidebar or fast swipe
const shouldOpen = event.translationX > windowWidth / 3 || event.velocityX > 500;
runOnJS(logLeftSidebarOpenGesture)("end", {
translationX: event.translationX,
velocityX: event.velocityX,
shouldOpen,
mobileView,
openGestureEnabled,
});
if (shouldOpen) {
animateToOpen();
runOnJS(openAgentList)();
@@ -422,6 +419,7 @@ function ProvidersWrapper({ children }: { children: ReactNode }) {
return (
<VoiceProvider>
<OfferLinkListener upsertDaemonFromOfferUrl={upsertConnectionFromOfferUrl} />
<HostSessionManager />
{children}
</VoiceProvider>
);
@@ -562,7 +560,6 @@ export default function RootLayout() {
<QueryProvider>
<HostRuntimeBootstrapProvider>
<PushNotificationRouter />
<HostSessionManager />
<ProvidersWrapper>
<SidebarAnimationProvider>
<HorizontalScrollProvider>

View File

@@ -31,6 +31,7 @@ import { useAgentAutocomplete } from '@/hooks/use-agent-autocomplete'
import { useHostRuntimeSession } from '@/runtime/host-runtime'
import {
deleteAttachments,
persistAttachmentFromBlob,
persistAttachmentFromFileUri,
} from '@/attachments/service'
import { resolveStatusControlMode } from '@/components/agent-input-area.status-controls'
@@ -390,16 +391,24 @@ export function AgentInputArea({
async function handlePickImage() {
const result = await pickImages()
if (!result?.assets?.length) {
if (!result?.length) {
return
}
const newImages = await Promise.all(
result.assets.map(async (asset) => {
result.map(async (pickedImage) => {
if (pickedImage.source.kind === 'blob') {
return await persistAttachmentFromBlob({
blob: pickedImage.source.blob,
mimeType: pickedImage.mimeType || 'image/jpeg',
fileName: pickedImage.fileName ?? null,
})
}
return await persistAttachmentFromFileUri({
uri: asset.uri,
mimeType: asset.mimeType || 'image/jpeg',
fileName: asset.fileName ?? null,
uri: pickedImage.source.uri,
mimeType: pickedImage.mimeType || 'image/jpeg',
fileName: pickedImage.fileName ?? null,
})
})
)

View File

@@ -86,6 +86,7 @@ export interface AgentStreamViewProps {
pendingPermissions: Map<string, PendingPermission>;
routeBottomAnchorRequest?: BottomAnchorRouteRequest | null;
isAuthoritativeHistoryReady?: boolean;
onOpenWorkspaceFile?: (input: { filePath: string }) => void;
}
export const AgentStreamView = forwardRef<AgentStreamViewHandle, AgentStreamViewProps>(function AgentStreamView({
@@ -96,6 +97,7 @@ export const AgentStreamView = forwardRef<AgentStreamViewHandle, AgentStreamView
pendingPermissions,
routeBottomAnchorRequest = null,
isAuthoritativeHistoryReady = true,
onOpenWorkspaceFile,
}, ref) {
const viewportRef = useRef<StreamViewportHandle | null>(null);
const { theme } = useUnistyles();
@@ -159,6 +161,11 @@ export const AgentStreamView = forwardRef<AgentStreamViewHandle, AgentStreamView
}
if (normalized.file) {
if (onOpenWorkspaceFile) {
onOpenWorkspaceFile({ filePath: normalized.file });
return;
}
const route = buildHostWorkspaceFileRoute(
resolvedServerId,
workspaceId,
@@ -188,6 +195,7 @@ export const AgentStreamView = forwardRef<AgentStreamViewHandle, AgentStreamView
resolvedServerId,
router,
setExplorerTabForCheckout,
onOpenWorkspaceFile,
workspaceId,
]
);

View File

@@ -6,7 +6,6 @@ import {
Modal,
} from "react-native";
import { SafeAreaView } from "react-native-safe-area-context";
import { useEffect } from "react";
import { StyleSheet } from "react-native-unistyles";
import { Fonts } from "@/constants/theme";
@@ -154,16 +153,6 @@ const styles = StyleSheet.create((theme) => ({
}));
export function ArtifactDrawer({ artifact, onClose }: ArtifactDrawerProps) {
useEffect(() => {
if (!artifact) return;
console.log(
"[ArtifactDrawer] Showing artifact:",
artifact.id,
artifact.type,
artifact.title
);
}, [artifact]);
if (!artifact) {
return null;
}

View File

@@ -23,13 +23,7 @@ import { FileExplorerPane } from "./file-explorer-pane";
import { useKeyboardShiftStyle } from "@/hooks/use-keyboard-shift-style";
const MIN_CHAT_WIDTH = 400;
const IS_DEV = Boolean((globalThis as { __DEV__?: boolean }).__DEV__);
function logExplorerSidebar(event: string, details: Record<string, unknown>): void {
if (!IS_DEV) {
return;
}
console.log(`[ExplorerSidebar] ${event}`, details);
function logExplorerSidebar(_event: string, _details: Record<string, unknown>): void {
}
interface ExplorerSidebarProps {

View File

@@ -838,7 +838,7 @@ function getErrorRecoveryPath(state: AgentFileExplorerState | undefined): string
const styles = StyleSheet.create((theme) => ({
container: {
flex: 1,
backgroundColor: theme.colors.surface0,
backgroundColor: theme.colors.surfaceSidebar,
},
desktopSplit: {
flex: 1,
@@ -879,7 +879,7 @@ const styles = StyleSheet.create((theme) => ({
paddingHorizontal: theme.spacing[3],
borderBottomWidth: 1,
borderBottomColor: theme.colors.border,
backgroundColor: theme.colors.surface0,
backgroundColor: theme.colors.surfaceSidebar,
},
paneHeaderLeft: {
flex: 1,

View File

@@ -15,6 +15,7 @@ import { router, usePathname } from 'expo-router'
import { usePanelStore } from '@/stores/panel-store'
import { SidebarWorkspaceList } from './sidebar-workspace-list'
import { SidebarAgentListSkeleton } from './sidebar-agent-list-skeleton'
import { useSidebarShortcutModel } from '@/hooks/use-sidebar-shortcut-model'
import { useSidebarWorkspacesList } from '@/hooks/use-sidebar-workspaces-list'
import { useSidebarAnimation } from '@/contexts/sidebar-animation-context'
import { useTauriDragHandlers, useTrafficLightPadding } from '@/utils/tauri-window'
@@ -31,14 +32,6 @@ import {
import { useKeyboardShortcutsStore } from '@/stores/keyboard-shortcuts-store'
const DESKTOP_SIDEBAR_WIDTH = 320
const IS_DEV = Boolean((globalThis as { __DEV__?: boolean }).__DEV__)
function logLeftSidebarCloseGesture(event: string, details: Record<string, unknown>): void {
if (!IS_DEV) {
return
}
console.log(`[LeftSidebarCloseGesture] ${event}`, details)
}
interface LeftSidebarProps {
selectedAgentId?: string
@@ -124,6 +117,8 @@ export function LeftSidebar({ selectedAgentId: _selectedAgentId }: LeftSidebarPr
serverId: activeServerId,
enabled: isOpen,
})
const { collapsedProjectKeys, shortcutIndexByWorkspaceKey, toggleProjectCollapsed } =
useSidebarShortcutModel(projects)
const {
translateX,
backdropOpacity,
@@ -253,7 +248,6 @@ export function LeftSidebar({ selectedAgentId: _selectedAgentId }: LeftSidebarPr
})
.onStart(() => {
isGesturing.value = true
runOnJS(logLeftSidebarCloseGesture)('start', { isOpen, isMobile })
})
.onUpdate((event) => {
if (!isMobile) return
@@ -271,11 +265,6 @@ export function LeftSidebar({ selectedAgentId: _selectedAgentId }: LeftSidebarPr
isGesturing.value = false
if (!isMobile) return
const shouldClose = event.translationX < -windowWidth / 3 || event.velocityX < -500
runOnJS(logLeftSidebarCloseGesture)('end', {
translationX: event.translationX,
velocityX: event.velocityX,
shouldClose,
})
if (shouldClose) {
animateToClose()
runOnJS(handleClose)()
@@ -351,8 +340,10 @@ export function LeftSidebar({ selectedAgentId: _selectedAgentId }: LeftSidebarPr
<SidebarAgentListSkeleton />
) : (
<SidebarWorkspaceList
isOpen={isOpen}
serverId={activeServerId}
collapsedProjectKeys={collapsedProjectKeys}
onToggleProjectCollapsed={toggleProjectCollapsed}
shortcutIndexByWorkspaceKey={shortcutIndexByWorkspaceKey}
projects={projects}
isRefreshing={isManualRefresh && isRevalidating}
onRefresh={handleRefresh}
@@ -475,8 +466,10 @@ export function LeftSidebar({ selectedAgentId: _selectedAgentId }: LeftSidebarPr
<SidebarAgentListSkeleton />
) : (
<SidebarWorkspaceList
isOpen={isOpen}
serverId={activeServerId}
collapsedProjectKeys={collapsedProjectKeys}
onToggleProjectCollapsed={toggleProjectCollapsed}
shortcutIndexByWorkspaceKey={shortcutIndexByWorkspaceKey}
projects={projects}
isRefreshing={isManualRefresh && isRevalidating}
onRefresh={handleRefresh}

View File

@@ -101,7 +101,6 @@ export interface MessageInputRef {
const MIN_INPUT_HEIGHT = 30
const MAX_INPUT_HEIGHT = 160
const IS_WEB = Platform.OS === 'web'
const IS_DEV = Boolean((globalThis as { __DEV__?: boolean }).__DEV__)
type WebTextInputKeyPressEvent = NativeSyntheticEvent<
TextInputKeyPressEventData & {
@@ -125,13 +124,10 @@ type TextAreaHandle = {
}
function logWebStickyBottom(
event: string,
details: Record<string, unknown>
_event: string,
_details: Record<string, unknown>
): void {
if (!IS_DEV || !IS_WEB) {
return
}
console.log('[WebStickyBottom]', event, details)
// Intentionally disabled: this path is too noisy during voice debugging.
}
function getDebugNow(): number | null {
@@ -407,6 +403,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(funct
const showRealtimeOverlay = isRealtimeVoiceForCurrentAgent
const showOverlay = showDictationOverlay || showRealtimeOverlay
useEffect(() => {
if (isDictating || isDictationProcessing) {
return
@@ -1070,10 +1067,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(funct
/>
) : showRealtimeOverlay && voice ? (
<RealtimeVoiceOverlay
volume={voice.volume}
isMuted={voice.isMuted}
isDetecting={voice.isDetecting}
isSpeaking={voice.isSpeaking}
isSwitching={voice.isVoiceSwitching}
onToggleMute={voice.toggleMute}
onStop={() => {

View File

@@ -18,6 +18,9 @@ import {
useCallback,
createContext,
useContext,
isValidElement,
Children,
cloneElement,
} from "react";
import type { ReactNode, ComponentType } from "react";
import Markdown, { MarkdownIt } from "react-native-markdown-display";
@@ -68,7 +71,11 @@ import {
buildToolCallDisplayModel,
} from "@/utils/tool-call-display";
import { resolveToolCallIcon } from "@/utils/tool-call-icon";
import { parseInlinePathToken, type InlinePathTarget } from "@/utils/inline-path";
import {
parseFileProtocolUrl,
parseInlinePathToken,
type InlinePathTarget,
} from "@/utils/inline-path";
import { getMarkdownListMarker } from "@/utils/markdown-list";
import { openExternalUrl } from "@/utils/open-external-url";
import { markScrollInvestigationEvent } from "@/utils/scroll-jank-investigation";
@@ -433,20 +440,6 @@ export const assistantMessageStylesheet = StyleSheet.create((theme) => ({
containerSpacing: {
marginBottom: theme.spacing[4],
},
// Used in custom markdownRules for inline code styling
markdownCodeInline: {
backgroundColor: theme.colors.surface2,
color: theme.colors.foreground,
paddingHorizontal: theme.spacing[2],
paddingVertical: 2,
borderRadius: theme.borderRadius.sm,
fontFamily: Fonts.mono,
fontSize: 13,
},
markdownCodeInlineLink: {
color: theme.colors.primary,
textDecorationLine: "underline",
},
// Used in custom markdownRules for path chip styling
pathChip: {
backgroundColor: theme.colors.surface2,
@@ -463,6 +456,44 @@ export const assistantMessageStylesheet = StyleSheet.create((theme) => ({
},
}));
function MarkdownLink({
href,
style,
onPress,
children,
}: {
href: string;
style: any;
onPress: (url: string) => void;
children: ReactNode;
}) {
const [hovered, setHovered] = useState(false);
if (Platform.OS !== "web") {
return (
<Text
accessibilityRole="link"
onPress={() => onPress(href)}
style={style}
>
{children}
</Text>
);
}
return (
<Pressable
accessibilityRole="link"
onPress={() => onPress(href)}
onHoverIn={() => setHovered(true)}
onHoverOut={() => setHovered(false)}
>
<Text style={[style, hovered && { textDecorationLine: "underline" }]}>
{children}
</Text>
</Pressable>
);
}
function getInlineCodeAutoLinkUrl(
markdownParser: ReturnType<typeof MarkdownIt>,
content: string
@@ -487,6 +518,19 @@ function getInlineCodeAutoLinkUrl(
return match.url;
}
function nodeHasParentType(parent: unknown, type: string): boolean {
if (Array.isArray(parent)) {
return parent.some((entry) => entry?.type === type);
}
return (
typeof parent === "object" &&
parent !== null &&
"type" in parent &&
(parent as { type?: string }).type === type
);
}
const turnCopyButtonStylesheet = StyleSheet.create((theme) => ({
container: {
alignSelf: "flex-start",
@@ -709,16 +753,35 @@ export const AssistantMessage = memo(function AssistantMessage({
const markdownStyles = useMemo(() => createMarkdownStyles(theme), [theme]);
const markdownParser = useMemo(
() => MarkdownIt({ typographer: true, linkify: true }),
() => {
const parser = MarkdownIt({ typographer: true, linkify: true });
const defaultValidateLink = parser.validateLink.bind(parser);
parser.validateLink = (url: string) => {
if (url.trim().toLowerCase().startsWith("file://")) {
return true;
}
return defaultValidateLink(url);
};
return parser;
},
[]
);
const handleLinkPress = useCallback((url: string) => {
const fileTarget = onInlinePathPress
? parseFileProtocolUrl(url)
: null;
if (fileTarget) {
onInlinePathPress?.(fileTarget);
return false;
}
void openExternalUrl(url);
// react-native-markdown-display opens the link itself when this returns true.
// We already handled it above, so return false to avoid duplicate opens.
return false;
}, []);
}, [onInlinePathPress]);
const markdownRules = useMemo(() => {
return {
@@ -775,12 +838,16 @@ export const AssistantMessage = memo(function AssistantMessage({
code_inline: (
node: any,
_children: ReactNode[],
_parent: any,
_styles: any,
parent: any,
styles: any,
inheritedStyles: any = {}
) => {
const content = node.content ?? "";
const parsed = onInlinePathPress
const isLinkedInlineCode =
nodeHasParentType(parent, "link") ||
(!Array.isArray(parent) &&
typeof parent?.attributes?.href === "string");
const parsed = onInlinePathPress && !isLinkedInlineCode
? parseInlinePathToken(content)
: null;
@@ -803,30 +870,21 @@ export const AssistantMessage = memo(function AssistantMessage({
const inlineCodeLinkUrl = getInlineCodeAutoLinkUrl(markdownParser, content);
if (inlineCodeLinkUrl) {
return (
<Text
<MarkdownLink
key={node.key}
accessibilityRole="link"
onPress={() => {
handleLinkPress(inlineCodeLinkUrl);
}}
style={[
inheritedStyles,
assistantMessageStylesheet.markdownCodeInline,
assistantMessageStylesheet.markdownCodeInlineLink,
]}
href={inlineCodeLinkUrl}
style={[inheritedStyles, styles.code_inline, styles.link]}
onPress={handleLinkPress}
>
{content}
</Text>
</MarkdownLink>
);
}
return (
<Text
key={node.key}
style={[
inheritedStyles,
assistantMessageStylesheet.markdownCodeInline,
]}
style={[inheritedStyles, styles.code_inline]}
>
{content}
</Text>
@@ -877,6 +935,25 @@ export const AssistantMessage = memo(function AssistantMessage({
</View>
);
},
link: (
node: any,
children: ReactNode[],
_parent: any,
styles: any,
) => (
<MarkdownLink
key={node.key}
href={node.attributes?.href ?? ""}
style={styles.link}
onPress={handleLinkPress}
>
{Children.map(children, (child) =>
isValidElement(child)
? cloneElement(child, { style: [(child.props as any).style, { color: styles.link.color }] } as any)
: child
)}
</MarkdownLink>
),
};
}, [handleLinkPress, markdownParser, onInlinePathPress]);
@@ -1849,9 +1926,11 @@ export const ToolCall = memo(function ToolCall({
const hasDetails =
Boolean(error) ||
(effectiveDetail
? effectiveDetail.type !== "unknown" ||
effectiveDetail.input !== null ||
effectiveDetail.output !== null
? effectiveDetail.type === "plain_text"
? Boolean(effectiveDetail.text)
: effectiveDetail.type !== "unknown" ||
effectiveDetail.input !== null ||
effectiveDetail.output !== null
: false);
const handleToggle = useCallback(() => {

View File

@@ -2,13 +2,11 @@ import { ActivityIndicator, Pressable, View } from "react-native";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
import { Mic, MicOff, Square } from "lucide-react-native";
import { FOOTER_HEIGHT } from "@/constants/layout";
import { useVoiceTelemetry } from "@/contexts/voice-context";
import { VolumeMeter } from "./volume-meter";
interface RealtimeVoiceOverlayProps {
volume: number;
isMuted: boolean;
isDetecting: boolean;
isSpeaking: boolean;
isSwitching: boolean;
onToggleMute: () => void;
onStop: () => void;
@@ -18,16 +16,13 @@ const OVERLAY_BUTTON_SIZE = 44;
const OVERLAY_VERTICAL_PADDING = (FOOTER_HEIGHT - OVERLAY_BUTTON_SIZE) / 2;
export function RealtimeVoiceOverlay({
volume,
isMuted,
isDetecting,
isSpeaking,
isSwitching,
onToggleMute,
onStop,
}: RealtimeVoiceOverlayProps) {
const { theme } = useUnistyles();
const { volume, isDetecting, isSpeaking } = useVoiceTelemetry();
return (
<View style={styles.container}>
<View style={styles.meterContainer}>

View File

@@ -59,7 +59,6 @@ import {
import { SyncedLoader } from '@/components/synced-loader'
import { useToast } from '@/contexts/toast-context'
import { useCheckoutGitActionsStore } from '@/stores/checkout-git-actions-store'
import { buildSidebarShortcutModel } from '@/utils/sidebar-shortcuts'
import { hasVisibleOrderChanged, mergeWithRemainder } from '@/utils/sidebar-reorder'
import { decideLongPressMove } from '@/utils/sidebar-gesture-arbitration'
import { confirmDialog } from '@/utils/confirm-dialog'
@@ -79,10 +78,17 @@ function toProjectIconDataUri(icon: { mimeType: string; data: string } | null):
return `data:${icon.mimeType};base64,${icon.data}`
}
const workspaceKeyExtractor = (workspace: SidebarWorkspaceEntry) =>
workspace.workspaceKey
const projectKeyExtractor = (project: SidebarProjectEntry) => project.projectKey
interface SidebarWorkspaceListProps {
isOpen?: boolean
projects: SidebarProjectEntry[]
serverId: string | null
collapsedProjectKeys: ReadonlySet<string>
onToggleProjectCollapsed: (projectKey: string) => void
shortcutIndexByWorkspaceKey: Map<string, number>
isRefreshing?: boolean
onRefresh?: () => void
onWorkspacePress?: () => void
@@ -122,6 +128,7 @@ interface WorkspaceRowInnerProps {
archiveStatus?: 'idle' | 'pending' | 'success'
archivePendingLabel?: string
onArchive?: () => void
onCopyBranchName?: () => void
onCopyPath?: () => void
}
@@ -250,7 +257,6 @@ function NewWorktreeButton({
function useLongPressDragInteraction(input: {
drag: () => void
menuController: ReturnType<typeof useContextMenu> | null
debugId: string
}) {
const didLongPressRef = useRef(false)
const dragArmedRef = useRef(false)
@@ -288,15 +294,11 @@ function useLongPressDragInteraction(input: {
input.menuController.setOpen(true)
menuOpenedRef.current = true
didLongPressRef.current = true
console.log('[sidebar-dnd-debug] context menu opened', { id: input.debugId })
}, [input.debugId, input.menuController])
}, [input.menuController])
const handleLongPress = useCallback(() => {
// Manual timers own long-press behavior on mobile.
console.log('[sidebar-dnd-debug] native onLongPress ignored (manual state machine active)', {
id: input.debugId,
})
}, [input.debugId])
}, [])
useEffect(() => {
return () => {
@@ -325,16 +327,11 @@ function useLongPressDragInteraction(input: {
const dy = current.y - start.y
const distance = Math.sqrt(dx * dx + dy * dy)
if (distance > DRAG_ARM_STATIONARY_SLOP_PX) {
console.log('[sidebar-dnd-debug] drag arm cancelled (movement)', {
id: input.debugId,
distance,
})
return
}
dragArmedRef.current = true
dragActivatedRef.current = true
didLongPressRef.current = true
console.log('[sidebar-dnd-debug] drag armed', { id: input.debugId })
void Haptics.selectionAsync().catch(() => {})
input.drag()
}, DRAG_ARM_DELAY_MS)
@@ -356,17 +353,12 @@ function useLongPressDragInteraction(input: {
const dy = current.y - start.y
const distance = Math.sqrt(dx * dx + dy * dy)
if (distance > CONTEXT_MENU_STATIONARY_SLOP_PX) {
console.log('[sidebar-dnd-debug] context menu cancelled (movement)', {
id: input.debugId,
distance,
})
return
}
console.log('[sidebar-dnd-debug] long-press armed', { id: input.debugId })
void Haptics.selectionAsync().catch(() => {})
openContextMenuAtStartPoint()
}, CONTEXT_MENU_DELAY_MS)
}, [clearTimers, input.debugId, input.menuController, openContextMenuAtStartPoint])
}, [clearTimers, input.menuController, openContextMenuAtStartPoint])
const handleDragIntent = useCallback(
(details: { dx: number; dy: number; distance: number }) => {
@@ -376,10 +368,9 @@ function useLongPressDragInteraction(input: {
didStartDragRef.current = true
didLongPressRef.current = true
clearTimers()
console.log('[sidebar-dnd-debug] drag movement detected', { id: input.debugId, ...details })
void Haptics.impactAsync(Haptics.ImpactFeedbackStyle.Medium).catch(() => {})
},
[clearTimers, input.debugId]
[clearTimers]
)
const handleScrollIntent = useCallback(
@@ -387,18 +378,16 @@ function useLongPressDragInteraction(input: {
scrollIntentRef.current = true
didLongPressRef.current = true
clearTimers()
console.log('[sidebar-dnd-debug] scroll intent detected', { id: input.debugId, ...details })
},
[clearTimers, input.debugId]
[clearTimers]
)
const handleSwipeIntent = useCallback(
(details: { dx: number; dy: number; distance: number }) => {
didLongPressRef.current = true
clearTimers()
console.log('[sidebar-dnd-debug] swipe intent detected', { id: input.debugId, ...details })
},
[clearTimers, input.debugId]
[clearTimers]
)
const handlePressIn = useCallback((event: GestureResponderEvent) => {
@@ -416,13 +405,8 @@ function useLongPressDragInteraction(input: {
x: event.nativeEvent.pageX,
y: event.nativeEvent.pageY,
}
console.log('[sidebar-dnd-debug] press-in', {
id: input.debugId,
x: event.nativeEvent.pageX,
y: event.nativeEvent.pageY,
})
armTimers()
}, [armTimers, input.debugId])
}, [armTimers])
const handleTouchMove = useCallback(
(event: any) => {
@@ -469,20 +453,11 @@ function useLongPressDragInteraction(input: {
const handlePressOut = useCallback(() => {
clearTimers()
console.log('[sidebar-dnd-debug] press-out no context-menu', {
id: input.debugId,
didLongPress: didLongPressRef.current,
didStartDrag: didStartDragRef.current,
dragActivated: dragActivatedRef.current,
scrollIntent: scrollIntentRef.current,
dragArmed: dragArmedRef.current,
menuOpened: menuOpenedRef.current,
})
dragArmedRef.current = false
dragActivatedRef.current = false
touchStartRef.current = null
touchCurrentRef.current = null
}, [clearTimers, input.debugId])
}, [clearTimers])
return {
didLongPressRef,
@@ -511,7 +486,6 @@ function ProjectHeaderRow({
const interaction = useLongPressDragInteraction({
drag,
menuController,
debugId: `project:${project.projectKey}`,
})
const handlePress = useCallback(() => {
@@ -610,6 +584,7 @@ function WorkspaceRowInner({
archiveStatus = 'idle',
archivePendingLabel,
onArchive,
onCopyBranchName,
onCopyPath,
}: WorkspaceRowInnerProps) {
const { theme } = useUnistyles()
@@ -618,7 +593,6 @@ function WorkspaceRowInner({
const interaction = useLongPressDragInteraction({
drag,
menuController,
debugId: `workspace:${workspace.workspaceKey}`,
})
const handlePress = useCallback(() => {
@@ -694,6 +668,15 @@ function WorkspaceRowInner({
Copy path
</DropdownMenuItem>
) : null}
{onCopyBranchName ? (
<DropdownMenuItem
testID={`sidebar-workspace-menu-copy-branch-name-${workspace.workspaceKey}`}
leading={<Copy size={14} color={theme.colors.foregroundMuted} />}
onSelect={onCopyBranchName}
>
Copy branch name
</DropdownMenuItem>
) : null}
<DropdownMenuItem
testID={`sidebar-workspace-menu-archive-${workspace.workspaceKey}`}
leading={<Archive size={14} color={theme.colors.foregroundMuted} />}
@@ -731,6 +714,7 @@ function WorkspaceRowWithMenu({
drag,
isDragging,
dragHandleProps,
canCopyBranchName,
}: {
workspace: SidebarWorkspaceEntry
selected: boolean
@@ -740,6 +724,7 @@ function WorkspaceRowWithMenu({
drag: () => void
isDragging: boolean
dragHandleProps?: DraggableListDragHandleProps
canCopyBranchName: boolean
}) {
const toast = useToast()
const archiveWorktree = useCheckoutGitActionsStore((state) => state.archiveWorktree)
@@ -825,6 +810,11 @@ function WorkspaceRowWithMenu({
toast.copied('Path copied')
}, [toast, workspace.workspaceId])
const handleCopyBranchName = useCallback(() => {
void Clipboard.setStringAsync(workspace.name)
toast.copied('Branch name copied')
}, [toast, workspace.name])
return (
<WorkspaceRowInner
workspace={workspace}
@@ -841,6 +831,7 @@ function WorkspaceRowWithMenu({
archiveStatus={isWorktree ? archiveStatus : isArchivingWorkspace ? 'pending' : 'idle'}
archivePendingLabel={isWorktree ? 'Archiving...' : 'Hiding...'}
onArchive={isWorktree ? handleArchiveWorktree : handleArchiveWorkspace}
onCopyBranchName={canCopyBranchName ? handleCopyBranchName : undefined}
onCopyPath={handleCopyPath}
/>
)
@@ -1006,6 +997,7 @@ function WorkspaceRow({
drag,
isDragging,
dragHandleProps,
canCopyBranchName,
}: {
workspace: SidebarWorkspaceEntry
selected: boolean
@@ -1015,6 +1007,7 @@ function WorkspaceRow({
drag: () => void
isDragging: boolean
dragHandleProps?: DraggableListDragHandleProps
canCopyBranchName: boolean
}) {
return (
<WorkspaceRowWithMenu
@@ -1026,6 +1019,7 @@ function WorkspaceRow({
drag={drag}
isDragging={isDragging}
dragHandleProps={dragHandleProps}
canCopyBranchName={canCopyBranchName}
/>
)
}
@@ -1093,6 +1087,7 @@ function ProjectBlock({
selected={isSelected}
shortcutNumber={shortcutIndexByWorkspaceKey.get(item.workspaceKey) ?? null}
showShortcutBadge={showShortcutBadges}
canCopyBranchName={project.projectKind === 'git'}
onPress={() => {
if (!serverId) {
return
@@ -1131,6 +1126,13 @@ function ProjectBlock({
[renderWorkspaceRow]
)
const handleWorkspaceDragEnd = useCallback(
(workspaces: SidebarWorkspaceEntry[]) => {
onWorkspaceReorder(project.projectKey, workspaces)
},
[onWorkspaceReorder, project.projectKey]
)
return (
<View style={styles.projectBlock}>
{flattenedWorkspace ? (
@@ -1176,9 +1178,9 @@ function ProjectBlock({
<DraggableList
testID={`sidebar-workspace-list-${project.projectKey}`}
data={project.workspaces}
keyExtractor={(workspace) => workspace.workspaceKey}
keyExtractor={workspaceKeyExtractor}
renderItem={renderWorkspace}
onDragEnd={(workspaces) => onWorkspaceReorder(project.projectKey, workspaces)}
onDragEnd={handleWorkspaceDragEnd}
scrollEnabled={false}
useDragHandle
nestable={useNestable}
@@ -1193,9 +1195,11 @@ function ProjectBlock({
}
export function SidebarWorkspaceList({
isOpen = true,
projects,
serverId,
collapsedProjectKeys,
onToggleProjectCollapsed,
shortcutIndexByWorkspaceKey,
isRefreshing = false,
onRefresh,
onWorkspacePress,
@@ -1205,16 +1209,9 @@ export function SidebarWorkspaceList({
const isMobile = UnistylesRuntime.breakpoint === 'xs' || UnistylesRuntime.breakpoint === 'sm'
const isNative = Platform.OS !== 'web'
const pathname = usePathname()
const [collapsedProjectKeys, setCollapsedProjectKeys] = useState<Set<string>>(new Set())
const isTauri = getIsTauri()
const altDown = useKeyboardShortcutsStore((state) => state.altDown)
const cmdOrCtrlDown = useKeyboardShortcutsStore((state) => state.cmdOrCtrlDown)
const setSidebarShortcutWorkspaceTargets = useKeyboardShortcutsStore(
(state) => state.setSidebarShortcutWorkspaceTargets
)
const setVisibleWorkspaceTargets = useKeyboardShortcutsStore(
(state) => state.setVisibleWorkspaceTargets
)
const showShortcutBadges = altDown || (isTauri && cmdOrCtrlDown)
const getProjectOrder = useSidebarOrderStore((state) => state.getProjectOrder)
@@ -1236,21 +1233,8 @@ export function SidebarWorkspaceList({
}
}, [pathname])
useEffect(() => {
setCollapsedProjectKeys((prev) => {
const validProjectKeys = new Set(projects.map((project) => project.projectKey))
const next = new Set<string>()
for (const key of prev) {
if (validProjectKeys.has(key)) {
next.add(key)
}
}
return next
})
}, [projects])
const projectIconRequests = useMemo(() => {
if (!isOpen || !serverId) {
if (!serverId) {
return []
}
const unique = new Map<string, { serverId: string; cwd: string }>()
@@ -1262,7 +1246,7 @@ export function SidebarWorkspaceList({
unique.set(`${serverId}:${cwd}`, { serverId, cwd })
}
return Array.from(unique.values())
}, [isOpen, projects, serverId])
}, [projects, serverId])
const projectIconQueries = useQueries({
queries: projectIconRequests.map((request) => ({
@@ -1277,7 +1261,6 @@ export function SidebarWorkspaceList({
},
select: toProjectIconDataUri,
enabled: Boolean(
isOpen &&
getHostRuntimeStore().getClient(request.serverId) &&
isHostRuntimeConnected(getHostRuntimeStore().getSnapshot(request.serverId)) &&
request.cwd
@@ -1316,44 +1299,6 @@ export function SidebarWorkspaceList({
return byProject
}, [projectIconQueries, projectIconRequests, projects, serverId])
const shortcutModel = useMemo(
() =>
buildSidebarShortcutModel({
projects,
collapsedProjectKeys,
}),
[collapsedProjectKeys, projects]
)
useEffect(() => {
setSidebarShortcutWorkspaceTargets(shortcutModel.shortcutTargets)
setVisibleWorkspaceTargets(shortcutModel.visibleTargets)
}, [
setSidebarShortcutWorkspaceTargets,
setVisibleWorkspaceTargets,
shortcutModel.shortcutTargets,
shortcutModel.visibleTargets,
])
useEffect(() => {
return () => {
setSidebarShortcutWorkspaceTargets([])
setVisibleWorkspaceTargets([])
}
}, [setSidebarShortcutWorkspaceTargets, setVisibleWorkspaceTargets])
const toggleProjectCollapsed = useCallback((projectKey: string) => {
setCollapsedProjectKeys((prev) => {
const next = new Set(prev)
if (next.has(projectKey)) {
next.delete(projectKey)
} else {
next.add(projectKey)
}
return next
})
}, [])
const handleProjectDragEnd = useCallback(
(reorderedProjects: SidebarProjectEntry[]) => {
if (!serverId) {
@@ -1437,9 +1382,9 @@ export function SidebarWorkspaceList({
serverId={serverId}
activeWorkspaceSelection={activeWorkspaceSelection}
showShortcutBadges={showShortcutBadges}
shortcutIndexByWorkspaceKey={shortcutModel.shortcutIndexByWorkspaceKey}
shortcutIndexByWorkspaceKey={shortcutIndexByWorkspaceKey}
parentGestureRef={parentGestureRef}
onToggleCollapsed={() => toggleProjectCollapsed(item.projectKey)}
onToggleCollapsed={() => onToggleProjectCollapsed(item.projectKey)}
onWorkspacePress={onWorkspacePress}
onWorkspaceReorder={handleWorkspaceReorder}
onCreateWorktree={handleCreateWorktree}
@@ -1456,12 +1401,12 @@ export function SidebarWorkspaceList({
handleCreateWorktree,
handleWorkspaceReorder,
onWorkspacePress,
onToggleProjectCollapsed,
parentGestureRef,
projectIconByProjectKey,
serverId,
shortcutModel.shortcutIndexByWorkspaceKey,
shortcutIndexByWorkspaceKey,
showShortcutBadges,
toggleProjectCollapsed,
isNative,
]
)
@@ -1474,7 +1419,7 @@ export function SidebarWorkspaceList({
<DraggableList
testID="sidebar-project-list"
data={projects}
keyExtractor={(project) => project.projectKey}
keyExtractor={projectKeyExtractor}
renderItem={renderProject}
onDragEnd={handleProjectDragEnd}
scrollEnabled={false}
@@ -1495,9 +1440,7 @@ export function SidebarWorkspaceList({
style={styles.list}
contentContainerStyle={styles.listContent}
showsVerticalScrollIndicator={false}
onScrollBeginDrag={() =>
console.log('[sidebar-dnd-debug] outer scroll begin')
}
testID="sidebar-project-workspace-list-scroll"
>
{content}
@@ -1507,9 +1450,7 @@ export function SidebarWorkspaceList({
style={styles.list}
contentContainerStyle={styles.listContent}
showsVerticalScrollIndicator={false}
onScrollBeginDrag={() =>
console.log('[sidebar-dnd-debug] outer scroll begin')
}
testID="sidebar-project-workspace-list-scroll"
>
{content}

View File

@@ -24,7 +24,6 @@ const USER_SCROLL_DELTA_EPSILON = 1
const AUTO_SCROLL_BOTTOM_THRESHOLD_PX = 64
const AUTO_SCROLL_RESUME_THRESHOLD_PX = 1
const WEB_STREAM_SCROLLBAR_STYLE_ID = 'web-stream-viewport-scrollbar-style'
const IS_DEV = Boolean((globalThis as { __DEV__?: boolean }).__DEV__)
const WEB_STREAM_SCROLLBAR_STYLE = `
#agent-chat-scroll-web-dom-scroll,
#agent-chat-scroll-web-dom-virtualized {
@@ -40,11 +39,8 @@ const WEB_STREAM_SCROLLBAR_STYLE = `
}
`
function logWebStickyBottom(event: string, details: Record<string, unknown>): void {
if (!IS_DEV) {
return
}
console.log('[WebStickyBottom]', event, details)
function logWebStickyBottom(_event: string, _details: Record<string, unknown>): void {
// Intentionally disabled: this path is too noisy during voice debugging.
}
function getDebugNow(): number | null {

View File

@@ -64,6 +64,7 @@ export interface ComboboxProps {
title?: string
open?: boolean
onOpenChange?: (open: boolean) => void
enableDismissOnClose?: boolean
desktopPlacement?: 'top-start' | 'bottom-start'
/**
* Prevents an initial frame at 0,0 by hiding desktop content until floating
@@ -224,6 +225,7 @@ export function Combobox({
title = 'Select',
open,
onOpenChange,
enableDismissOnClose,
desktopPlacement = 'top-start',
desktopPreventInitialFlash = true,
anchorRef,
@@ -235,6 +237,7 @@ export function Combobox({
!isMobile && Platform.OS === 'web' && effectiveOptionsPosition === 'above-search'
const { height: windowHeight } = useWindowDimensions()
const bottomSheetRef = useRef<BottomSheetModal>(null)
const hasPresentedBottomSheetRef = useRef(false)
const snapPoints = useMemo(() => ['60%', '90%'], [])
const [availableSize, setAvailableSize] = useState<{ width?: number; height?: number } | null>(
null
@@ -379,11 +382,20 @@ export function Combobox({
useEffect(() => {
if (!isMobile) return
if (isOpen) {
bottomSheetRef.current?.present()
if (enableDismissOnClose === false && hasPresentedBottomSheetRef.current) {
bottomSheetRef.current?.snapToIndex(0)
} else {
hasPresentedBottomSheetRef.current = true
bottomSheetRef.current?.present()
}
} else {
bottomSheetRef.current?.dismiss()
if (enableDismissOnClose === false) {
bottomSheetRef.current?.close()
} else {
bottomSheetRef.current?.dismiss()
}
}
}, [isOpen, isMobile])
}, [enableDismissOnClose, isOpen, isMobile])
const handleSheetChange = useCallback(
(index: number) => {
@@ -622,6 +634,7 @@ export function Combobox({
onChange={handleSheetChange}
backdropComponent={renderBackdrop}
enablePanDownToClose
enableDismissOnClose={enableDismissOnClose}
backgroundComponent={ComboboxSheetBackground}
handleIndicatorStyle={styles.bottomSheetHandle}
keyboardBehavior="extend"

View File

@@ -9,6 +9,8 @@ import {
useState,
type PropsWithChildren,
type ReactElement,
type Ref,
type MutableRefObject,
} from "react";
import {
ActivityIndicator,
@@ -182,6 +184,16 @@ function coerceEventPoint(event: unknown): { pageX: number; pageY: number } | nu
return null;
}
function assignRef<T>(ref: Ref<T> | undefined, value: T): void {
if (typeof ref === "function") {
ref(value);
return;
}
if (ref && typeof ref === "object") {
(ref as MutableRefObject<T>).current = value;
}
}
export function ContextMenu({
open,
defaultOpen,
@@ -231,6 +243,7 @@ export function ContextMenuTrigger({
enabledOnMobile = false,
enabledOnWeb = true,
longPressDelayMs,
triggerRef,
...props
}: PropsWithChildren<
Omit<PressableProps, "style"> & {
@@ -239,6 +252,7 @@ export function ContextMenuTrigger({
enabledOnMobile?: boolean;
enabledOnWeb?: boolean;
longPressDelayMs?: number;
triggerRef?: Ref<View | null>;
}
>): ReactElement {
const ctx = useContextMenuContext("ContextMenuTrigger");
@@ -268,10 +282,18 @@ export function ContextMenuTrigger({
[ctx, disabled, shouldEnableOnThisPlatform]
);
const handleRef = useCallback(
(node: View | null) => {
assignRef(ctx.triggerRef, node);
assignRef(triggerRef, node);
},
[ctx.triggerRef, triggerRef]
);
return (
<Pressable
{...props}
ref={ctx.triggerRef}
ref={handleRef}
collapsable={false}
disabled={disabled}
delayLongPress={longPressDelayMs}

View File

@@ -170,10 +170,6 @@ export function useDropdownFloating({
}, []);
const update = useCallback(async () => {
console.log("[useDropdownFloating] update called", {
hasReferenceEl: !!referenceEl,
hasFloatingEl: !!floatingElRef.current,
});
if (!referenceEl || !floatingElRef.current) {
return;
@@ -185,12 +181,6 @@ export function useDropdownFloating({
measureElement(floatingElRef.current),
]);
console.log("[useDropdownFloating] measured", {
fromRect,
contentRect,
displayArea,
});
const result = computeGeometry({
fromRect,
contentSize: { width: contentRect.width, height: contentRect.height },
@@ -201,8 +191,6 @@ export function useDropdownFloating({
padding,
});
console.log("[useDropdownFloating] geometry result", result);
setGeometry(result);
} catch (e) {
console.warn("[useDropdownFloating] measure failed:", e);
@@ -210,7 +198,6 @@ export function useDropdownFloating({
}, [referenceEl, displayArea, placement, alignment, offset, padding]);
const floatingRef = useCallback((el: View | null) => {
console.log("[useDropdownFloating] floatingRef called", { hasEl: !!el });
floatingElRef.current = el;
}, []);
@@ -218,17 +205,10 @@ export function useDropdownFloating({
const [floatingReady, setFloatingReady] = useState(false);
const handleFloatingLayout = useCallback(() => {
console.log("[useDropdownFloating] handleFloatingLayout called");
setFloatingReady(true);
}, []);
useEffect(() => {
console.log("[useDropdownFloating] useEffect", {
open,
floatingReady,
hasReferenceEl: !!referenceEl,
hasFloatingEl: !!floatingElRef.current,
});
if (!open) {
setGeometry(null);
@@ -237,7 +217,6 @@ export function useDropdownFloating({
}
if (floatingReady && referenceEl && floatingElRef.current) {
console.log("[useDropdownFloating] calling update from useEffect");
update();
}
}, [open, floatingReady, referenceEl, update]);

View File

@@ -1,5 +1,8 @@
import {
Children,
cloneElement,
createContext,
isValidElement,
useCallback,
useContext,
useEffect,
@@ -8,6 +11,7 @@ import {
useState,
type PropsWithChildren,
type ReactElement,
type Ref,
} from "react";
import {
Dimensions,
@@ -53,6 +57,26 @@ function useTooltipContext(componentName: string): TooltipContextValue {
return ctx;
}
function composeEventHandlers<E>(
original?: (event: E) => void,
injected?: (event: E) => void
): (event: E) => void {
return (event: E) => {
original?.(event);
injected?.(event);
};
}
function assignRef<T>(ref: Ref<T> | undefined, value: T): void {
if (typeof ref === "function") {
ref(value);
return;
}
if (ref && typeof ref === "object") {
(ref as { current: T }).current = value;
}
}
function useControllableOpenState({
open,
defaultOpen,
@@ -224,8 +248,15 @@ export function TooltipTrigger({
onFocus,
onBlur,
onPress,
asChild = false,
triggerRefProp = "ref",
...props
}: PropsWithChildren<PressableProps>): ReactElement {
}: PropsWithChildren<
PressableProps & {
asChild?: boolean;
triggerRefProp?: string;
}
>): ReactElement {
const ctx = useTooltipContext("TooltipTrigger");
const openTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
@@ -276,37 +307,86 @@ export function TooltipTrigger({
[onHoverOut, close]
);
const handleFocus = useCallback(
(e: any) => {
onFocus?.(e);
if (!ctx.enabled || disabled) return;
clearOpenTimer();
ctx.setOpen(true);
},
[clearOpenTimer, ctx, disabled, onFocus]
);
const handleBlur = useCallback(
(e: any) => {
onBlur?.(e);
close();
},
[close, onBlur]
);
const handlePress = useCallback(
(e: any) => {
onPress?.(e);
close();
},
[close, onPress]
);
const triggerProps = {
...props,
disabled,
onHoverIn: handleHoverIn,
onHoverOut: handleHoverOut,
onFocus: handleFocus,
onBlur: handleBlur,
onPress: handlePress,
...(Platform.OS === "web"
? ({
// RN Web's hover handling can vary across environments; pointer events are the most reliable.
onPointerEnter: handleHoverIn,
onPointerLeave: handleHoverOut,
onMouseEnter: handleHoverIn,
onMouseLeave: handleHoverOut,
} as any)
: null),
};
if (asChild) {
const child = Children.only(children);
if (!isValidElement(child)) {
throw new Error("TooltipTrigger with asChild expects a single React element child");
}
const childProps = child.props as Record<string, any>;
const mergedProps = {
...childProps,
...triggerProps,
onHoverIn: composeEventHandlers(childProps.onHoverIn, handleHoverIn),
onHoverOut: composeEventHandlers(childProps.onHoverOut, handleHoverOut),
onFocus: composeEventHandlers(childProps.onFocus, handleFocus),
onBlur: composeEventHandlers(childProps.onBlur, handleBlur),
onPress: composeEventHandlers(childProps.onPress, handlePress),
onPointerEnter: composeEventHandlers(childProps.onPointerEnter, handleHoverIn),
onPointerLeave: composeEventHandlers(childProps.onPointerLeave, handleHoverOut),
onMouseEnter: composeEventHandlers(childProps.onMouseEnter, handleHoverIn),
onMouseLeave: composeEventHandlers(childProps.onMouseLeave, handleHoverOut),
} as Record<string, any>;
const existingRefProp = childProps[triggerRefProp] as Ref<View | null> | undefined;
mergedProps[triggerRefProp] = (node: View | null) => {
assignRef(existingRefProp, node);
assignRef(ctx.triggerRef, node);
};
return cloneElement(child, mergedProps);
}
return (
<Pressable
{...props}
{...triggerProps}
ref={ctx.triggerRef}
collapsable={false}
disabled={disabled}
onHoverIn={handleHoverIn}
onHoverOut={handleHoverOut}
onFocus={(e) => {
onFocus?.(e);
if (!ctx.enabled || disabled) return;
clearOpenTimer();
ctx.setOpen(true);
}}
onBlur={(e) => {
onBlur?.(e);
close();
}}
onPress={(e) => {
onPress?.(e);
close();
}}
{...(Platform.OS === "web"
? ({
// RN Web's hover handling can vary across environments; pointer events are the most reliable.
onPointerEnter: handleHoverIn,
onPointerLeave: handleHoverOut,
onMouseEnter: handleHoverIn,
onMouseLeave: handleHoverOut,
} as any)
: null)}
>
{children}
</Pressable>

View File

@@ -132,8 +132,6 @@ function createDriverHarness(input?: {
context.measurementState.offsetY = 720;
});
const modeChanges: BottomAnchorMode[] = [];
const warnings: Array<{ agentId: string; reason: string }> = [];
const logs: Array<{ event: string; details: Record<string, unknown> }> = [];
const driver = __private__.createBottomAnchorControllerDriver({
getAgentId: () => context.agentId,
getIsAuthoritativeHistoryReady: () => context.authoritativeReady,
@@ -145,10 +143,6 @@ function createDriverHarness(input?: {
onModeChange: (mode) => {
modeChanges.push(mode);
},
log: (event, details) => {
logs.push({ event, details });
},
warn: (details) => warnings.push(details),
scheduleFrame: (params) => scheduler.schedule(params),
cancelFrame: (handle) => scheduler.cancel(handle),
});
@@ -159,8 +153,6 @@ function createDriverHarness(input?: {
scheduler,
scrollToBottom,
modeChanges,
logs,
warnings,
};
}
@@ -367,7 +359,6 @@ describe("bottom anchor controller driver", () => {
harness.scheduler.flushAll();
expect(harness.scrollToBottom).toHaveBeenCalledTimes(1);
expect(harness.warnings).toEqual([]);
expect(harness.driver.getSnapshot().pendingRequest).toBeNull();
});

View File

@@ -110,8 +110,6 @@ type CreateBottomAnchorControllerDriverInput = {
isNearBottom: () => boolean;
scrollToBottom: (animated: boolean) => void;
onModeChange: (mode: BottomAnchorMode) => void;
log: (event: BottomAnchorEvent, details: Record<string, unknown>) => void;
warn: (details: { agentId: string; reason: BottomAnchorRequestReason }) => void;
scheduleFrame: (params: {
kind: "attempt" | "verification";
callback: () => void;
@@ -123,17 +121,6 @@ type CreateBottomAnchorControllerDriverInput = {
const MAX_VERIFICATION_RETRIES = 3;
const WEB_PARTIAL_VIRTUALIZED_CONFIRMATION_DELAY_FRAMES = 1;
const USER_SCROLL_AWAY_DELTA_PX = 24;
const IS_DEV = Boolean((globalThis as { __DEV__?: boolean }).__DEV__);
function logBottomAnchorEvent(
event: BottomAnchorEvent,
details: Record<string, unknown>
): void {
if (!IS_DEV) {
return;
}
console.debug("[BottomAnchor]", event, details);
}
function scheduleAnimationFrameWithDelay(input: {
callback: () => void;
@@ -323,10 +310,6 @@ function createBottomAnchorControllerDriver(
return;
}
blockedReason = nextBlockedReason;
input.log(
"blocked_reason_changed",
getLogContext({ nextBlockedReason })
);
};
const setModeInternal = (nextMode: BottomAnchorMode) => {
@@ -367,13 +350,6 @@ function createBottomAnchorControllerDriver(
setBlockedReason(null);
return;
}
input.log(
"request_cancelled",
getLogContext({
cancelledRequestReason: currentRequest.reason,
cancelReason: reason,
})
);
pendingRequest = null;
cancelPendingAttempt();
setBlockedReason(null);
@@ -398,19 +374,6 @@ function createBottomAnchorControllerDriver(
if (verificationHandle) {
input.cancelFrame(verificationHandle);
}
input.log(
"verification_scheduled",
getLogContext({
retries: attemptContext.retries,
startedContentHeight: attemptContext.startedContentHeight ?? null,
startedOffsetY: attemptContext.startedOffsetY ?? null,
startedViewportHeight: attemptContext.startedViewportHeight ?? null,
scheduledMeasurementState:
getDetailedMeasurementState(scheduledMeasurementState),
verificationDelayFrames:
delayFramesOverride ?? input.getTransportBehavior().verificationDelayFrames,
})
);
verificationHandle = input.scheduleFrame({
kind: "verification",
delayFrames:
@@ -427,15 +390,6 @@ function createBottomAnchorControllerDriver(
});
if (verificationBlockedReason) {
input.log(
"attempt_verified",
getLogContext({
verificationPhase: "blocked",
verificationBlockedReason,
retries: attemptContext.retries,
measurementState: getDetailedMeasurementState(measurementState),
})
);
pendingVerification = attemptContext;
setBlockedReason(verificationBlockedReason);
return;
@@ -451,26 +405,6 @@ function createBottomAnchorControllerDriver(
input.getTransportBehavior().verificationRetryMode,
});
input.log(
"attempt_verified",
getLogContext({
verifiedNearBottom,
retries: attemptContext.retries,
retryDisposition,
contentHeightDeltaSinceAttempt:
measurementState.contentHeight -
(attemptContext.startedContentHeight ?? measurementState.contentHeight),
offsetDeltaSinceAttempt:
measurementState.offsetY -
(attemptContext.startedOffsetY ?? measurementState.offsetY),
viewportHeightDeltaSinceAttempt:
measurementState.viewportHeight -
(attemptContext.startedViewportHeight ??
measurementState.viewportHeight),
measurementState: getDetailedMeasurementState(measurementState),
})
);
if (verifiedNearBottom) {
if (
isRequestAttempt &&
@@ -494,7 +428,6 @@ function createBottomAnchorControllerDriver(
pendingVerification = null;
markStickyMeasurementVerified();
if (isRequestAttempt) {
input.log("request_fulfilled", getLogContext());
pendingRequest = null;
}
setBlockedReason(null);
@@ -520,21 +453,7 @@ function createBottomAnchorControllerDriver(
return;
}
input.log(
"attempt_failed",
getLogContext({
retries: attemptContext.retries,
retryDisposition,
measurementState: getDetailedMeasurementState(measurementState),
})
);
pendingVerification = null;
if (isRequestAttempt && currentRequest) {
input.warn({
agentId: input.getAgentId(),
reason: currentRequest.reason,
});
}
setBlockedReason(
isRequestAttempt ? "waiting_for_post_layout_verification" : null
);
@@ -552,14 +471,6 @@ function createBottomAnchorControllerDriver(
startedViewportHeight: measurementState.viewportHeight,
};
pendingVerification = attemptContext;
input.log(
"attempt_started",
getLogContext({
animated,
retries: attemptContext.retries,
measurementState: getDetailedMeasurementState(measurementState),
})
);
input.scrollToBottom(animated);
scheduleVerification(attemptContext);
setBlockedReason(deriveDriverBlockedReason(input.getMeasurementState()));
@@ -576,18 +487,6 @@ function createBottomAnchorControllerDriver(
| "manual_reevaluate"
| "retry_scroll"
) => {
input.log(
"evaluate_called",
getLogContext({
evaluateReason: reason,
animated,
hasAttemptHandle: attemptHandle !== null,
hasVerificationHandle: verificationHandle !== null,
pendingVerificationRequestId: pendingVerification?.requestId ?? null,
pendingVerificationRetries: pendingVerification?.retries ?? null,
measurementState: getDetailedMeasurementState(input.getMeasurementState()),
})
);
if (attemptHandle) {
return;
}
@@ -610,17 +509,6 @@ function createBottomAnchorControllerDriver(
!shouldAttemptForPendingRequest &&
!shouldAttemptForStickyVerification
) {
input.log(
"attempt_started",
getLogContext({
attemptPhase: "skipped",
evaluateReason: reason,
nextBlockedReason,
shouldAttemptForPendingRequest,
shouldAttemptForStickyVerification,
measurementState: getDetailedMeasurementState(measurementState),
})
);
return;
}
@@ -630,17 +518,6 @@ function createBottomAnchorControllerDriver(
};
const createRequest = (request: BottomAnchorRouteRequest | BottomAnchorLocalRequest) => {
const existing = pendingRequest;
if (existing) {
input.log(
"request_cancelled",
getLogContext({
cancelledRequestReason: existing.reason,
cancelReason: "replaced_by_new_request",
})
);
}
cancelPendingAttempt();
const nextRequest: BottomAnchorRequest = {
id: requestSequence + 1,
@@ -659,10 +536,6 @@ function createBottomAnchorControllerDriver(
? "sticky-bottom"
: __private__.deriveModeForLocalRequest({ reason: request.reason })
);
input.log(
"request_created",
getLogContext({ requestReason: request.reason })
);
evaluate(request.reason === "jump-to-bottom", "request_created");
};
@@ -707,7 +580,6 @@ function createBottomAnchorControllerDriver(
}
cancelPendingRequest("user_scrolled_away");
setModeInternal("detached");
input.log("detached_by_user", getLogContext());
},
handleViewportMetricsChange(params) {
if (
@@ -905,10 +777,6 @@ export function useBottomAnchorController(input: {
isNearBottom: () => isNearBottomRef.current(),
scrollToBottom: (animated) => scrollToBottomRef.current(animated),
onModeChange: (nextMode) => setMode(nextMode),
log: (event, details) => logBottomAnchorEvent(event, details),
warn: (details) => {
console.warn("[BottomAnchor] request could not be fulfilled", details);
},
scheduleFrame: ({ callback, delayFrames }) =>
scheduleAnimationFrameWithDelay({ callback, delayFrames }),
cancelFrame: (handle) => cancelScheduledAnimationFrame(handle),

View File

@@ -2,21 +2,18 @@ import { ActivityIndicator, Alert, Pressable, View } from "react-native";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
import { Mic, MicOff, Square } from "lucide-react-native";
import { VolumeMeter } from "@/components/volume-meter";
import { useVoice } from "@/contexts/voice-context";
import { useVoice, useVoiceTelemetry } from "@/contexts/voice-context";
export function VoiceCompactIndicator() {
const { theme } = useUnistyles();
const { volume, isDetecting, isSpeaking } = useVoiceTelemetry();
const {
isVoiceMode,
isVoiceSwitching,
volume,
isMuted,
isDetecting,
isSpeaking,
toggleMute,
stopVoice,
} = useVoice();
if (!isVoiceMode) {
return null;
}

View File

@@ -2,22 +2,19 @@ import { View, Pressable } from "react-native";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
import { MicOff, Square } from "lucide-react-native";
import { VolumeMeter } from "./volume-meter";
import { useVoice } from "@/contexts/voice-context";
import { useVoice, useVoiceTelemetry } from "@/contexts/voice-context";
import { useHosts } from "@/runtime/host-runtime";
export function VoicePanel() {
const { theme } = useUnistyles();
const daemons = useHosts();
const { volume, isDetecting, isSpeaking } = useVoiceTelemetry();
const {
volume,
isMuted,
isDetecting,
isSpeaking,
stopVoice,
toggleMute,
activeServerId,
} = useVoice();
const hostLabel = activeServerId
? daemons.find((daemon) => daemon.serverId === activeServerId)?.label ?? null
: null;

View File

@@ -3,10 +3,9 @@ import { View } from "react-native";
import ReanimatedAnimated, {
useSharedValue,
useAnimatedStyle,
withSpring,
withTiming,
withRepeat,
withSequence,
withTiming,
Easing,
} from "react-native-reanimated";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
@@ -41,9 +40,7 @@ export function VolumeMeter({
orientation === "horizontal" ? (isCompact ? 8 : 12) : isCompact ? 14 : 20;
// Create shared values for 3 dots unconditionally
const line1Height = useSharedValue(MIN_HEIGHT);
const line2Height = useSharedValue(MIN_HEIGHT);
const line3Height = useSharedValue(MIN_HEIGHT);
const animatedVolume = useSharedValue(0);
const line1Pulse = useSharedValue(1);
const line2Pulse = useSharedValue(1);
const line3Pulse = useSharedValue(1);
@@ -90,54 +87,19 @@ export function VolumeMeter({
);
}, [isMuted]);
// Update heights based on volume with different responsiveness for all dots
// Drive a single animated volume value and derive the individual bar heights
// on the UI thread instead of scheduling three independent springs per sample.
useEffect(() => {
if (isMuted) {
// When muted, keep all lines at minimum height without animation
line1Height.value = MIN_HEIGHT;
line2Height.value = MIN_HEIGHT;
line3Height.value = MIN_HEIGHT;
animatedVolume.value = 0;
return;
}
if (volume > 0.001) {
// Active volume - animate heights based on volume
const target1 = MIN_HEIGHT + (MAX_HEIGHT * volume * 1.2);
const target2 = MIN_HEIGHT + (MAX_HEIGHT * volume * 1.05);
const target3 = MIN_HEIGHT + (MAX_HEIGHT * volume * 0.9);
line1Height.value = withSpring(target1, {
damping: 10,
stiffness: 200,
});
line2Height.value = withSpring(target2, {
damping: 12.5,
stiffness: 175,
});
line3Height.value = withSpring(target3, {
damping: 15,
stiffness: 150,
});
} else {
// No volume - return to minimum
line1Height.value = withSpring(MIN_HEIGHT, {
damping: 20,
stiffness: 150,
});
line2Height.value = withSpring(MIN_HEIGHT, {
damping: 20,
stiffness: 150,
});
line3Height.value = withSpring(MIN_HEIGHT, {
damping: 20,
stiffness: 150,
});
}
}, [volume, isMuted]);
animatedVolume.value = withTiming(volume, {
duration: volume > animatedVolume.value ? 70 : 140,
easing: Easing.out(Easing.cubic),
});
}, [animatedVolume, isMuted, volume]);
const lineColor = color ?? theme.colors.foreground;
const containerHeight =
@@ -147,9 +109,12 @@ export function VolumeMeter({
const line1Style = useAnimatedStyle(() => {
const isActive = isSpeaking;
const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
const volumeBoost = isMuted || !isActive ? 0 : volume * 0.3;
const currentVolume = isMuted ? 0 : animatedVolume.value;
const currentHeight = MIN_HEIGHT + (MAX_HEIGHT * currentVolume * 1.2);
const volumeBoost = isMuted || !isActive ? 0 : currentVolume * 0.3;
return {
height: line1Height.value * (isMuted || volume > 0.001 ? 1 : line1Pulse.value),
height:
currentHeight * (isMuted || currentVolume > 0.001 ? 1 : line1Pulse.value),
opacity: baseOpacity + volumeBoost,
};
});
@@ -157,9 +122,12 @@ export function VolumeMeter({
const line2Style = useAnimatedStyle(() => {
const isActive = isSpeaking;
const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
const volumeBoost = isMuted || !isActive ? 0 : volume * 0.3;
const currentVolume = isMuted ? 0 : animatedVolume.value;
const currentHeight = MIN_HEIGHT + (MAX_HEIGHT * currentVolume * 1.05);
const volumeBoost = isMuted || !isActive ? 0 : currentVolume * 0.3;
return {
height: line2Height.value * (isMuted || volume > 0.001 ? 1 : line2Pulse.value),
height:
currentHeight * (isMuted || currentVolume > 0.001 ? 1 : line2Pulse.value),
opacity: baseOpacity + volumeBoost,
};
});
@@ -167,9 +135,12 @@ export function VolumeMeter({
const line3Style = useAnimatedStyle(() => {
const isActive = isSpeaking;
const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
const volumeBoost = isMuted || !isActive ? 0 : volume * 0.3;
const currentVolume = isMuted ? 0 : animatedVolume.value;
const currentHeight = MIN_HEIGHT + (MAX_HEIGHT * currentVolume * 0.9);
const volumeBoost = isMuted || !isActive ? 0 : currentVolume * 0.3;
return {
height: line3Height.value * (isMuted || volume > 0.001 ? 1 : line3Pulse.value),
height:
currentHeight * (isMuted || currentVolume > 0.001 ? 1 : line3Pulse.value),
opacity: baseOpacity + volumeBoost,
};
});

View File

@@ -12,18 +12,6 @@ import { usePanelStore } from "@/stores/panel-store";
const ANIMATION_DURATION = 220;
const ANIMATION_EASING = Easing.bezier(0.25, 0.1, 0.25, 1);
const IS_DEV = Boolean((globalThis as { __DEV__?: boolean }).__DEV__);
function logExplorerAnimation(
event: string,
details: Record<string, unknown>
): void {
if (!IS_DEV) {
return;
}
console.log(`[ExplorerAnimation] ${event}`, details);
}
interface ExplorerSidebarAnimationContextValue {
translateX: SharedValue<number>;
backdropOpacity: SharedValue<number>;
@@ -66,23 +54,9 @@ export function ExplorerSidebarAnimationProvider({ children }: { children: React
// Don't animate if we're in the middle of a gesture - the gesture handler will handle it
if (isGesturing.value) {
logExplorerAnimation("sync-skipped-during-gesture", {
previousIsOpen,
nextIsOpen: isOpen,
mobileView,
desktopFileExplorerOpen,
});
return;
}
logExplorerAnimation("sync-state-change", {
previousIsOpen,
nextIsOpen: isOpen,
mobileView,
desktopFileExplorerOpen,
windowWidth,
});
if (isOpen) {
translateX.value = withTiming(0, {
duration: ANIMATION_DURATION,

View File

@@ -1,7 +1,7 @@
import { useRef, ReactNode, useCallback, useEffect, useMemo } from "react";
import { Buffer } from "buffer";
import { AppState, Platform } from "react-native";
import { useQueryClient } from "@tanstack/react-query";
import { useAudioPlayer } from "@/hooks/use-audio-player";
import { useClientActivity } from "@/hooks/use-client-activity";
import { usePushTokenRegistration } from "@/hooks/use-push-token-registration";
import { clearArchiveAgentPending } from "@/hooks/use-archive-agent";
@@ -29,8 +29,13 @@ import type { DaemonClient } from "@server/client/daemon-client";
import { File } from "expo-file-system";
import {
getHostRuntimeStore,
useHostRuntimeSession,
useHostRuntimeIsConnected,
} from "@/runtime/host-runtime";
import {
useVoiceAudioEngineOptional,
useVoiceRuntimeOptional,
} from "@/contexts/voice-context";
import type { AudioPlaybackSource } from "@/voice/audio-engine-types";
import {
useSessionStore,
type Agent,
@@ -41,6 +46,7 @@ import {
import { useDraftStore } from "@/stores/draft-store";
import type { AgentDirectoryEntry } from "@/types/agent-directory";
import { sendOsNotification } from "@/utils/os-notifications";
import { getIsAppActivelyVisible } from "@/utils/app-visibility";
import {
getInitKey,
getInitDeferred,
@@ -72,6 +78,55 @@ export type {
const HISTORY_STALE_AFTER_MS = 60_000;
const AUTHORITATIVE_REVALIDATION_DEBOUNCE_MS = 300;
type AudioOutputPayload = Extract<
SessionOutboundMessage,
{ type: "audio_output" }
>["payload"];
interface BufferedAudioChunk {
chunkIndex: number;
audio: string;
format: string;
id: string;
}
function decodeBase64Chunk(base64: string): Uint8Array {
return Buffer.from(base64, "base64");
}
function buildAudioPlaybackSource(
chunks: BufferedAudioChunk[]
): AudioPlaybackSource {
const decodedChunks = chunks.map((chunk) => decodeBase64Chunk(chunk.audio));
const totalSize = decodedChunks.reduce((sum, chunk) => sum + chunk.length, 0);
const output = new Uint8Array(totalSize);
let offset = 0;
for (const chunk of decodedChunks) {
output.set(chunk, offset);
offset += chunk.length;
}
const format = chunks[0]?.format ?? "pcm";
const mimeType =
format === "pcm"
? "audio/pcm;rate=24000;bits=16"
: format === "mp3"
? "audio/mpeg"
: `audio/${format}`;
const bytes = output.slice();
return {
size: bytes.byteLength,
type: mimeType,
async arrayBuffer() {
return bytes.buffer.slice(
bytes.byteOffset,
bytes.byteOffset + bytes.byteLength
);
},
};
}
const findLatestAssistantMessageText = (items: StreamItem[]): string | null => {
for (let i = items.length - 1; i >= 0; i -= 1) {
const item = items[i];
@@ -205,8 +260,10 @@ function SessionProviderInternal({
serverId,
client,
}: SessionProviderClientProps) {
const voiceRuntime = useVoiceRuntimeOptional();
const voiceAudioEngine = useVoiceAudioEngineOptional();
const queryClient = useQueryClient();
const { isConnected } = useHostRuntimeSession(serverId);
const isConnected = useHostRuntimeIsConnected(serverId);
// Zustand store actions
const initializeSession = useSessionStore((state) => state.initializeSession);
@@ -275,16 +332,6 @@ function SessionProviderInternal({
(state) => state.sessions[serverId]?.agents
);
// State for voice detection flags (will be set by RealtimeContext)
const isDetectingRef = useRef(false);
const isSpeakingRef = useRef(false);
const audioPlayer = useAudioPlayer({
isDetecting: () => isDetectingRef.current,
isSpeaking: () => isSpeakingRef.current,
});
const activeAudioGroupsRef = useRef<Set<string>>(new Set());
const previousAgentStatusRef = useRef<Map<string, AgentLifecycleStatus>>(
new Map()
);
@@ -307,6 +354,10 @@ function SessionProviderInternal({
const revalidationInFlightRef = useRef<Promise<void> | null>(null);
const revalidationQueuedRef = useRef(false);
const wasConnectedRef = useRef(isConnected);
const audioOutputBuffersRef = useRef<Map<string, BufferedAudioChunk[]>>(
new Map()
);
const activeAudioGroupsRef = useRef<Set<string>>(new Set());
useEffect(() => {
const subscription = AppState.addEventListener("change", (nextState) => {
@@ -452,11 +503,6 @@ function SessionProviderInternal({
const queue = session?.queuedMessages.get(agent.id);
if (queue && queue.length > 0) {
const [next, ...rest] = queue;
console.log(
"[Session] Flushing queued message for agent:",
agent.id,
next.text
);
if (sendAgentMessageRef.current) {
void sendAgentMessageRef.current(agent.id, next.text, next.images);
}
@@ -570,18 +616,10 @@ function SessionProviderInternal({
if (params.reason === "error") {
return;
}
const isActive = appState ? appState === "active" : true;
const isAwayFromAgent = !isActive || focusedAgentId !== params.agentId;
const isActivelyVisible = getIsAppActivelyVisible(appState);
const isAwayFromAgent =
!isActivelyVisible || focusedAgentId !== params.agentId;
if (!isAwayFromAgent) {
console.log(
"[OSNotifications] Skipping attention notification: user already focused on agent",
{
agentId: params.agentId,
reason: params.reason,
appState,
focusedAgentId,
}
);
return;
}
@@ -624,24 +662,55 @@ function SessionProviderInternal({
[serverId]
);
// Buffer for streaming audio chunks
interface AudioChunk {
chunkIndex: number;
audio: string; // base64
format: string;
id: string;
}
const audioChunkBuffersRef = useRef<Map<string, AudioChunk[]>>(new Map());
// Initialize session in store
useEffect(() => {
initializeSession(serverId, client, audioPlayer);
}, [serverId, client, audioPlayer, initializeSession]);
initializeSession(serverId, client);
}, [serverId, client, initializeSession]);
useEffect(() => {
updateSessionClient(serverId, client);
}, [serverId, client, updateSessionClient]);
useEffect(() => {
if (!voiceRuntime) {
return;
}
return voiceRuntime.registerSession({
serverId,
setVoiceMode: async (enabled, agentId) => {
if (!client) {
throw new Error("Daemon unavailable");
}
await client.setVoiceMode(enabled, agentId);
},
sendVoiceAudioChunk: async (audioData, mimeType, isLast) => {
if (!client) {
throw new Error("Daemon unavailable");
}
await client.sendVoiceAudioChunk(audioData, mimeType, isLast);
},
abortRequest: async () => {
if (!client) {
throw new Error("Daemon unavailable");
}
await client.abortRequest();
},
setAssistantAudioPlaying: (isPlaying) => {
setIsPlayingAudio(serverId, isPlaying);
},
});
}, [
client,
serverId,
setIsPlayingAudio,
voiceRuntime,
]);
useEffect(() => {
voiceRuntime?.updateSessionConnection(serverId, isConnected);
}, [isConnected, serverId, voiceRuntime]);
// If the client drops mid-initialization, clear pending flags
useEffect(() => {
if (!isConnected) {
@@ -969,6 +1038,14 @@ function SessionProviderInternal({
const { agentId, event, timestamp, seq, epoch } = message.payload;
const parsedTimestamp = new Date(timestamp);
const streamEvent = event as AgentStreamEventPayload;
if (
event.type === "turn_started" ||
event.type === "turn_completed" ||
event.type === "turn_failed" ||
event.type === "turn_canceled"
) {
voiceRuntime?.onTurnEvent(serverId, agentId, event.type);
}
// Attention notification stays in React (not extractable to pure reducer)
if (event.type === "attention_required") {
@@ -1118,13 +1195,6 @@ function SessionProviderInternal({
if (message.type !== "agent_permission_request") return;
const { agentId, request } = message.payload;
console.log(
"[Session] Permission request:",
request.id,
"for agent:",
agentId
);
setPendingPermissions(serverId, (prev) => {
const next = new Map(prev);
const key = derivePendingPermissionKey(agentId, request);
@@ -1140,13 +1210,6 @@ function SessionProviderInternal({
if (message.type !== "agent_permission_resolved") return;
const { requestId, agentId } = message.payload;
console.log(
"[Session] Permission resolved:",
requestId,
"for agent:",
agentId
);
setPendingPermissions(serverId, (prev) => {
const next = new Map(prev);
const derivedKey = `${agentId}:${requestId}`;
@@ -1168,92 +1231,70 @@ function SessionProviderInternal({
const unsubAudioOutput = client.on("audio_output", async (message) => {
if (message.type !== "audio_output") return;
const data = message.payload;
const playbackGroupId = data.groupId ?? data.id;
const isFinalChunk = data.isLastChunk ?? true;
const chunkIndex = data.chunkIndex ?? 0;
if (!voiceAudioEngine) {
return;
}
const payload: AudioOutputPayload = message.payload;
const playbackGroupId = payload.groupId ?? payload.id;
const chunkIndex = payload.chunkIndex ?? 0;
const isFinalChunk = payload.isLastChunk ?? true;
if (!audioOutputBuffersRef.current.has(playbackGroupId)) {
audioOutputBuffersRef.current.set(playbackGroupId, []);
}
const bufferedChunks = audioOutputBuffersRef.current.get(playbackGroupId)!;
bufferedChunks.push({
chunkIndex,
audio: payload.audio,
format: payload.format,
id: payload.id,
});
activeAudioGroupsRef.current.add(playbackGroupId);
setIsPlayingAudio(serverId, true);
if (!audioChunkBuffersRef.current.has(playbackGroupId)) {
audioChunkBuffersRef.current.set(playbackGroupId, []);
}
const buffer = audioChunkBuffersRef.current.get(playbackGroupId)!;
buffer.push({
chunkIndex,
audio: data.audio,
format: data.format,
id: data.id,
});
if (!isFinalChunk) {
return;
}
buffer.sort((a, b) => a.chunkIndex - b.chunkIndex);
let playbackFailed = false;
const chunkIds = buffer.map((chunk) => chunk.id);
const confirmAudioPlayed = (ids: string[]) => {
if (!client) {
console.warn("[Session] audio_played skipped: daemon unavailable");
return;
}
ids.forEach((chunkId) => {
void client.audioPlayed(chunkId).catch((error) => {
console.warn("[Session] Failed to confirm audio playback:", error);
});
});
bufferedChunks.sort((left, right) => left.chunkIndex - right.chunkIndex);
const chunkIds = bufferedChunks.map((chunk) => chunk.id);
const shouldPlay =
!payload.isVoiceMode ||
(voiceRuntime?.shouldPlayVoiceAudio(serverId) ?? false);
const audioBlob = buildAudioPlaybackSource(bufferedChunks);
const confirmAudioPlayed = async () => {
await Promise.all(
chunkIds.map((chunkId) =>
client.audioPlayed(chunkId).catch((error) => {
console.warn("[Session] Failed to confirm audio playback:", error);
})
)
);
};
let startedVoicePlayback = false;
try {
const mimeType =
data.format === "mp3" ? "audio/mpeg" : `audio/${data.format}`;
const decodedChunks: Uint8Array[] = [];
let totalSize = 0;
for (const chunk of buffer) {
const binaryString = atob(chunk.audio);
const bytes = new Uint8Array(binaryString.length);
for (let i = 0; i < binaryString.length; i++) {
bytes[i] = binaryString.charCodeAt(i);
if (shouldPlay) {
if (payload.isVoiceMode) {
startedVoicePlayback = true;
voiceRuntime?.onAssistantAudioStarted(serverId);
}
decodedChunks.push(bytes);
totalSize += bytes.length;
await voiceAudioEngine.play(audioBlob);
}
const concatenatedBytes = new Uint8Array(totalSize);
let offset = 0;
for (const chunk of decodedChunks) {
concatenatedBytes.set(chunk, offset);
offset += chunk.length;
}
const audioBlob = {
type: mimeType,
size: totalSize,
arrayBuffer: async () => {
return concatenatedBytes.buffer;
},
} as Blob;
await audioPlayer.play(audioBlob);
confirmAudioPlayed(chunkIds);
} catch (error: any) {
playbackFailed = true;
await confirmAudioPlayed();
} catch (error) {
console.error("[Session] Audio playback error:", error);
confirmAudioPlayed(chunkIds);
await confirmAudioPlayed();
} finally {
audioChunkBuffersRef.current.delete(playbackGroupId);
audioOutputBuffersRef.current.delete(playbackGroupId);
activeAudioGroupsRef.current.delete(playbackGroupId);
setIsPlayingAudio(serverId, activeAudioGroupsRef.current.size > 0);
if (activeAudioGroupsRef.current.size === 0) {
setIsPlayingAudio(serverId, false);
if (startedVoicePlayback) {
voiceRuntime?.onAssistantAudioFinished(serverId);
}
}
});
@@ -1261,7 +1302,6 @@ function SessionProviderInternal({
const unsubActivity = client.on("activity_log", (message) => {
if (message.type !== "activity_log") return;
const data = message.payload;
if (data.type === "system" && data.content.includes("Transcribing")) {
return;
}
@@ -1381,13 +1421,12 @@ function SessionProviderInternal({
if (message.type !== "transcription_result") return;
const transcriptText = message.payload.text.trim();
if (!transcriptText) {
} else {
audioPlayer.stop();
setIsPlayingAudio(serverId, false);
setCurrentAssistantMessage(serverId, "");
voiceRuntime?.onTranscriptionResult(serverId, transcriptText);
return;
}
setCurrentAssistantMessage(serverId, "");
});
const unsubAgentDeleted = client.on("agent_deleted", (message) => {
@@ -1395,7 +1434,6 @@ function SessionProviderInternal({
return;
}
const { agentId } = message.payload;
console.log("[Session] Agent deleted:", agentId);
deletePendingAgentUpdate(serverId, agentId);
clearArchiveAgentPending({ queryClient, serverId, agentId });
@@ -1471,7 +1509,6 @@ function SessionProviderInternal({
return;
}
const { agentId, archivedAt } = message.payload;
console.log("[Session] Agent archived:", agentId);
clearArchiveAgentPending({ queryClient, serverId, agentId });
setAgents(serverId, (prev) => {
@@ -1505,7 +1542,6 @@ function SessionProviderInternal({
};
}, [
client,
audioPlayer,
queryClient,
serverId,
setIsPlayingAudio,
@@ -1528,6 +1564,8 @@ function SessionProviderInternal({
requestCanonicalCatchUp,
applyAgentUpdatePayload,
applyTimelineResponse,
voiceRuntime,
voiceAudioEngine,
]);
const sendAgentMessage = useCallback(
@@ -1654,11 +1692,6 @@ function SessionProviderInternal({
worktreeName?: string;
requestId?: string;
}) => {
console.log(
"[Session] createAgent called with images:",
images?.length ?? 0,
images
);
if (!client) {
console.warn("[Session] createAgent skipped: daemon unavailable");
return;
@@ -1667,14 +1700,6 @@ function SessionProviderInternal({
let imagesData: Array<{ data: string; mimeType: string }> | undefined;
try {
imagesData = await encodeImages(images);
console.log(
"[Session] encodeImages result:",
imagesData?.length ?? 0,
imagesData?.map((img) => ({
dataLength: img.data?.length ?? 0,
mimeType: img.mimeType,
}))
);
} catch (error) {
console.error(
"[Session] Failed to prepare images for agent creation:",
@@ -1749,20 +1774,12 @@ function SessionProviderInternal({
[client]
);
const setVoiceDetectionFlags = useCallback(
(isDetecting: boolean, isSpeaking: boolean) => {
isDetectingRef.current = isDetecting;
isSpeakingRef.current = isSpeaking;
},
[]
);
// Cleanup on unmount
useEffect(() => {
return () => {
clearSession(serverId);
};
}, [serverId, clearSession]);
}, [clearSession, serverId]);
return children;
}

View File

@@ -1,41 +1,106 @@
import { createContext, useContext, useState, ReactNode, useCallback, useEffect, useRef } from "react";
import { useSpeechmaticsAudio } from "@/hooks/use-speechmatics-audio";
import type { SessionState } from "@/stores/session-store";
import { useSessionStore } from "@/stores/session-store";
import { useHostRuntimeSession } from "@/runtime/host-runtime";
import { resolveVoiceUnavailableMessage } from "@/utils/server-info-capabilities";
import {
createContext,
useContext,
useEffect,
useRef,
useSyncExternalStore,
type ReactNode,
} from "react";
import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake";
import { REALTIME_VOICE_VAD_CONFIG } from "@/voice/realtime-voice-config";
import { useSessionStore } from "@/stores/session-store";
import { createAudioEngine } from "@/voice/audio-engine";
import type { AudioEngine } from "@/voice/audio-engine-types";
import {
createVoiceRuntime,
type VoiceRuntime,
type VoiceRuntimeSnapshot,
type VoiceRuntimeTelemetrySnapshot,
} from "@/voice/voice-runtime";
const KEEP_AWAKE_TAG = "paseo:voice";
interface VoiceContextValue {
isVoiceMode: boolean;
isVoiceSwitching: boolean;
volume: number;
isMuted: boolean;
isDetecting: boolean;
isSpeaking: boolean;
segmentDuration: number;
interface VoiceContextValue extends VoiceRuntimeSnapshot {
startVoice: (serverId: string, agentId: string) => Promise<void>;
stopVoice: () => Promise<void>;
isVoiceModeForAgent: (serverId: string, agentId: string) => boolean;
toggleMute: () => void;
activeServerId: string | null;
activeAgentId: string | null;
}
const VoiceContext = createContext<VoiceContextValue | null>(null);
const EMPTY_SNAPSHOT: VoiceRuntimeSnapshot = {
phase: "disabled",
isVoiceMode: false,
isVoiceSwitching: false,
isMuted: false,
activeServerId: null,
activeAgentId: null,
};
const EMPTY_TELEMETRY: VoiceRuntimeTelemetrySnapshot = {
volume: 0,
isDetecting: false,
isSpeaking: false,
segmentDuration: 0,
};
const VoiceRuntimeContext = createContext<VoiceRuntime | null>(null);
const VoiceAudioEngineContext = createContext<AudioEngine | null>(null);
const noopSubscribe = () => () => {};
const getEmptySnapshot = () => EMPTY_SNAPSHOT;
const getEmptyTelemetry = () => EMPTY_TELEMETRY;
export function useVoice() {
const context = useContext(VoiceContext);
if (!context) {
const value = useVoiceOptional();
if (!value) {
throw new Error("useVoice must be used within VoiceProvider");
}
return context;
return value;
}
export function useVoiceOptional(): VoiceContextValue | null {
return useContext(VoiceContext);
const runtime = useContext(VoiceRuntimeContext);
const snapshot = useSyncExternalStore(
runtime ? runtime.subscribe : noopSubscribe,
runtime ? runtime.getSnapshot : getEmptySnapshot,
runtime ? runtime.getSnapshot : getEmptySnapshot
);
if (!runtime) {
return null;
}
return {
...snapshot,
startVoice: runtime.startVoice,
stopVoice: runtime.stopVoice,
isVoiceModeForAgent: runtime.isVoiceModeForAgent,
toggleMute: runtime.toggleMute,
};
}
export function useVoiceTelemetry() {
const telemetry = useVoiceTelemetryOptional();
if (!telemetry) {
throw new Error("useVoiceTelemetry must be used within VoiceProvider");
}
return telemetry;
}
export function useVoiceTelemetryOptional(): VoiceRuntimeTelemetrySnapshot | null {
const runtime = useContext(VoiceRuntimeContext);
const snapshot = useSyncExternalStore(
runtime ? runtime.subscribeTelemetry : noopSubscribe,
runtime ? runtime.getTelemetrySnapshot : getEmptyTelemetry,
runtime ? runtime.getTelemetrySnapshot : getEmptyTelemetry
);
return runtime ? snapshot : null;
}
export function useVoiceRuntimeOptional(): VoiceRuntime | null {
return useContext(VoiceRuntimeContext);
}
export function useVoiceAudioEngineOptional(): AudioEngine | null {
return useContext(VoiceAudioEngineContext);
}
interface VoiceProviderProps {
@@ -43,405 +108,55 @@ interface VoiceProviderProps {
}
export function VoiceProvider({ children }: VoiceProviderProps) {
const getSession = useSessionStore((state) => state.getSession);
const [activeServerId, setActiveServerId] = useState<string | null>(null);
const [activeAgentId, setActiveAgentId] = useState<string | null>(null);
const { client: runtimeClient, isConnected: activeRuntimeConnected } = useHostRuntimeSession(
activeServerId ?? ""
);
const activeSession = useSessionStore(
useCallback(
(state: ReturnType<typeof useSessionStore.getState>) => {
if (!activeServerId) {
return null;
}
return state.sessions[activeServerId] ?? null;
const engineRef = useRef<AudioEngine | null>(null);
const runtimeRef = useRef<VoiceRuntime | null>(null);
if (!engineRef.current) {
let runtime: VoiceRuntime | null = null;
const engine = createAudioEngine({
onCaptureData: (pcm) => {
runtime?.handleCapturePcm(pcm);
},
[activeServerId]
)
);
const realtimeSessionRef = useRef<SessionState | null>(null);
const [isVoiceMode, setIsVoiceMode] = useState(false);
const [isVoiceSwitching, setIsVoiceSwitching] = useState(false);
const bargeInPlaybackStopRef = useRef<number | null>(null);
const wasVoiceSocketConnectedRef = useRef(false);
const lastVoiceModeSyncedClientRef = useRef<SessionState["client"] | null>(null);
const voiceTransportReadyRef = useRef(false);
const voiceResyncInFlightRef = useRef(false);
const silenceGraceStartMsRef = useRef<number | null>(null);
const speechInterruptTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const speechInterruptStartMsRef = useRef<number | null>(null);
const speechStartInterruptSentRef = useRef(false);
const isVoiceModeRef = useRef(false);
const vadStateRef = useRef<{ isDetecting: boolean; isSpeaking: boolean }>({
isDetecting: false,
isSpeaking: false,
});
onVolumeLevel: (level) => {
runtime?.handleCaptureVolume(level);
},
onError: (error) => {
console.error("[VoiceEngine] Capture error:", error);
},
});
const clearSpeechStartInterruptTimer = useCallback((reason: string) => {
const timer = speechInterruptTimerRef.current;
if (timer) {
clearTimeout(timer);
speechInterruptTimerRef.current = null;
const startedAt = speechInterruptStartMsRef.current;
speechInterruptStartMsRef.current = null;
if (startedAt !== null) {
console.log("[Voice] Cleared speech-start interrupt timer", {
reason,
elapsedMs: Date.now() - startedAt,
});
} else {
console.log("[Voice] Cleared speech-start interrupt timer", { reason });
}
return;
}
speechInterruptStartMsRef.current = null;
}, []);
runtime = createVoiceRuntime({
engine,
getServerInfo: (serverId) =>
useSessionStore.getState().getSession(serverId)?.serverInfo ?? null,
activateKeepAwake: async (tag) => {
await activateKeepAwakeAsync(tag);
},
deactivateKeepAwake: async (tag) => {
await deactivateKeepAwake(tag);
},
});
const interruptActiveVoiceTurn = useCallback((source: string) => {
const session = realtimeSessionRef.current;
const sessionAudioPlayer = session?.audioPlayer ?? null;
const sessionClient = session?.client ?? null;
const sessionIsPlayingAudio = session?.isPlayingAudio ?? false;
engineRef.current = engine;
runtimeRef.current = runtime;
}
if (sessionIsPlayingAudio && sessionAudioPlayer) {
if (bargeInPlaybackStopRef.current === null) {
bargeInPlaybackStopRef.current = Date.now();
}
sessionAudioPlayer.stop();
}
try {
if (sessionClient) {
void sessionClient.abortRequest().catch((error) => {
console.error("[Voice] Failed to send abort_request:", error);
});
}
console.log("[Voice] Sent abort_request before streaming audio", { source });
} catch (error) {
console.error("[Voice] Failed to send abort_request:", error);
}
}, []);
const realtimeAudio = useSpeechmaticsAudio({
onSpeechStart: () => {
console.log("[Voice] Segment started (speech confirmed)");
},
onSpeechEnd: () => {
const silenceMs =
silenceGraceStartMsRef.current === null
? null
: Date.now() - silenceGraceStartMsRef.current;
if (silenceMs === null) {
console.log("[Voice] Segment finalized");
} else {
console.log("[Voice] Segment finalized", { silenceMs });
}
silenceGraceStartMsRef.current = null;
clearSpeechStartInterruptTimer("speech ended");
speechStartInterruptSentRef.current = false;
},
onAudioSegment: ({ audioData, isLast }) => {
if (!voiceTransportReadyRef.current) {
console.log("[Voice] Skipping audio segment: voice transport not ready");
return;
}
console.log(
"[Voice] Sending audio segment, length:",
audioData.length,
"isLast:",
isLast
);
// Send audio segment to server (realtime always goes to orchestrator)
const session = realtimeSessionRef.current;
try {
if (session?.client) {
void session.client
.sendVoiceAudioChunk(
audioData,
"audio/pcm;rate=16000;bits=16",
isLast
)
.catch((error) => {
console.error("[Voice] Failed to send audio segment:", error);
});
}
} catch (error) {
console.error("[Voice] Failed to send audio segment:", error);
}
},
onError: (error) => {
console.error("[Voice] Audio error:", error);
const session = realtimeSessionRef.current;
if (session?.client) {
// Send error through websocket instead of directly manipulating messages
console.error("[Voice] Cannot handle error - setMessages not available from SessionState");
}
},
volumeThreshold: REALTIME_VOICE_VAD_CONFIG.volumeThreshold,
confirmedDropGracePeriod: REALTIME_VOICE_VAD_CONFIG.confirmedDropGracePeriodMs,
silenceDuration: REALTIME_VOICE_VAD_CONFIG.silenceDurationMs,
speechConfirmationDuration: REALTIME_VOICE_VAD_CONFIG.speechConfirmationMs,
detectionGracePeriod: REALTIME_VOICE_VAD_CONFIG.detectionGracePeriodMs,
});
useEffect(() => {
realtimeSessionRef.current = activeSession;
}, [activeSession]);
useEffect(() => {
isVoiceModeRef.current = isVoiceMode;
}, [isVoiceMode]);
useEffect(() => {
if (!isVoiceMode) {
clearSpeechStartInterruptTimer("voice mode disabled");
speechStartInterruptSentRef.current = false;
return;
}
if (realtimeAudio.isSpeaking) {
if (
speechStartInterruptSentRef.current ||
speechInterruptTimerRef.current !== null
) {
return;
}
speechInterruptStartMsRef.current = Date.now();
speechInterruptTimerRef.current = setTimeout(() => {
speechInterruptTimerRef.current = null;
speechInterruptStartMsRef.current = null;
if (
!isVoiceModeRef.current ||
!vadStateRef.current.isSpeaking ||
speechStartInterruptSentRef.current
) {
return;
}
speechStartInterruptSentRef.current = true;
console.log("[Voice] Speech persisted beyond grace; interrupting turn", {
graceMs: REALTIME_VOICE_VAD_CONFIG.interruptGracePeriodMs,
});
interruptActiveVoiceTurn("speech_start_grace_elapsed");
}, REALTIME_VOICE_VAD_CONFIG.interruptGracePeriodMs);
return;
}
clearSpeechStartInterruptTimer("speech stopped before grace");
}, [
clearSpeechStartInterruptTimer,
interruptActiveVoiceTurn,
isVoiceMode,
realtimeAudio.isSpeaking,
]);
const engine = engineRef.current!;
const runtime = runtimeRef.current!;
useEffect(() => {
return () => {
clearSpeechStartInterruptTimer("voice provider unmounted");
};
}, [clearSpeechStartInterruptTimer]);
useEffect(() => {
const next = {
isDetecting: realtimeAudio.isDetecting,
isSpeaking: realtimeAudio.isSpeaking,
};
const prev = vadStateRef.current;
if (!isVoiceMode) {
silenceGraceStartMsRef.current = null;
vadStateRef.current = next;
return;
}
// Confirmed speech dropped below threshold: grace window starts.
if (prev.isSpeaking && !next.isSpeaking && next.isDetecting) {
silenceGraceStartMsRef.current = Date.now();
console.log("[Voice] Grace started (speech dropped below threshold)");
}
// User resumed speaking before grace timeout elapsed.
if (!prev.isSpeaking && next.isSpeaking && silenceGraceStartMsRef.current !== null) {
const resumedAfterMs = Date.now() - silenceGraceStartMsRef.current;
console.log("[Voice] Speech resumed during grace", { resumedAfterMs });
silenceGraceStartMsRef.current = null;
}
// Fully idle (neither detecting nor speaking).
if (!next.isDetecting && !next.isSpeaking) {
silenceGraceStartMsRef.current = null;
speechStartInterruptSentRef.current = false;
}
vadStateRef.current = next;
}, [isVoiceMode, realtimeAudio.isDetecting, realtimeAudio.isSpeaking]);
useEffect(() => {
const connected = activeRuntimeConnected;
const client = runtimeClient;
if (!connected) {
voiceTransportReadyRef.current = false;
}
if (!isVoiceMode || !activeServerId || !activeAgentId || !client) {
wasVoiceSocketConnectedRef.current = connected;
if (!isVoiceMode) {
lastVoiceModeSyncedClientRef.current = null;
}
voiceTransportReadyRef.current = false;
return;
}
const connectionRecovered = connected && !wasVoiceSocketConnectedRef.current;
const clientChanged = lastVoiceModeSyncedClientRef.current !== client;
if (connected && (connectionRecovered || clientChanged)) {
if (!voiceResyncInFlightRef.current) {
voiceResyncInFlightRef.current = true;
voiceTransportReadyRef.current = false;
setIsVoiceSwitching(true);
void client.setVoiceMode(true, activeAgentId).then(
() => {
console.log("[Voice] Re-synced voice mode after reconnect");
lastVoiceModeSyncedClientRef.current = client;
voiceTransportReadyRef.current = true;
},
(error) => {
console.error("[Voice] Failed to re-sync voice mode:", error);
}
).finally(() => {
voiceResyncInFlightRef.current = false;
setIsVoiceSwitching(false);
});
}
}
wasVoiceSocketConnectedRef.current = connected;
}, [activeAgentId, activeRuntimeConnected, activeServerId, isVoiceMode, runtimeClient]);
const isPlayingAudio = activeSession?.isPlayingAudio ?? false;
useEffect(() => {
if (!isPlayingAudio && bargeInPlaybackStopRef.current !== null) {
const latencyMs = Date.now() - bargeInPlaybackStopRef.current;
console.log("[Telemetry] barge_in.playback_stop_latency", {
latencyMs,
startedAt: new Date(bargeInPlaybackStopRef.current).toISOString(),
completedAt: new Date().toISOString(),
void runtime.destroy().catch((error) => {
console.error("[VoiceProvider] Failed to destroy voice runtime", error);
});
bargeInPlaybackStopRef.current = null;
}
}, [isPlayingAudio]);
const startVoice = useCallback(
async (serverId: string, agentId: string) => {
const session = getSession(serverId) ?? null;
if (!session) {
throw new Error(`Host ${serverId} is not connected`);
}
const unavailableMessage = resolveVoiceUnavailableMessage({
serverInfo: session.serverInfo,
mode: "voice",
});
if (unavailableMessage) {
throw new Error(unavailableMessage);
}
setIsVoiceSwitching(true);
voiceTransportReadyRef.current = false;
try {
const previousSession = realtimeSessionRef.current;
if (
isVoiceMode &&
previousSession?.client &&
(activeServerId !== serverId || activeAgentId !== agentId)
) {
await previousSession.client.setVoiceMode(false);
}
realtimeSessionRef.current = session;
setActiveServerId(serverId);
setActiveAgentId(agentId);
await activateKeepAwakeAsync(KEEP_AWAKE_TAG).catch((error) => {
console.warn("[Voice] Failed to activate keep-awake:", error);
});
if (session?.client) {
await session.client.setVoiceMode(true, agentId);
} else {
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
}
await session.audioPlayer?.warmup?.();
await realtimeAudio.start();
voiceTransportReadyRef.current = true;
setIsVoiceMode(true);
lastVoiceModeSyncedClientRef.current = session.client;
console.log("[Voice] Mode enabled");
} catch (error: any) {
console.error("[Voice] Failed to start:", error);
await realtimeAudio.stop().catch(() => undefined);
setActiveServerId((current) => (current === serverId ? null : current));
setActiveAgentId((current) => (current === agentId ? null : current));
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
throw error;
} finally {
setIsVoiceSwitching(false);
}
},
[activeAgentId, activeServerId, getSession, isVoiceMode, realtimeAudio]
);
const stopVoice = useCallback(async () => {
setIsVoiceSwitching(true);
voiceTransportReadyRef.current = false;
try {
const session = realtimeSessionRef.current;
session?.audioPlayer?.stop();
if (session?.client) {
await session.client.setVoiceMode(false);
lastVoiceModeSyncedClientRef.current = session.client;
} else {
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
}
await realtimeAudio.stop();
setIsVoiceMode(false);
setActiveServerId(null);
setActiveAgentId(null);
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
console.log("[Voice] Mode disabled");
} catch (error: any) {
console.error("[Voice] Failed to stop:", error);
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
throw error;
} finally {
setIsVoiceSwitching(false);
}
}, [realtimeAudio]);
const isVoiceModeForAgent = useCallback(
(serverId: string, agentId: string) =>
isVoiceMode && activeServerId === serverId && activeAgentId === agentId,
[activeAgentId, activeServerId, isVoiceMode]
);
const value: VoiceContextValue = {
isVoiceMode,
isVoiceSwitching,
volume: realtimeAudio.volume,
isMuted: realtimeAudio.isMuted,
isDetecting: realtimeAudio.isDetecting,
isSpeaking: realtimeAudio.isSpeaking,
segmentDuration: realtimeAudio.segmentDuration,
startVoice,
stopVoice,
isVoiceModeForAgent,
toggleMute: realtimeAudio.toggleMute,
activeServerId,
activeAgentId,
};
};
}, [runtime]);
return (
<VoiceContext.Provider value={value}>
{children}
</VoiceContext.Provider>
<VoiceAudioEngineContext.Provider value={engine}>
<VoiceRuntimeContext.Provider value={runtime}>
{children}
</VoiceRuntimeContext.Provider>
</VoiceAudioEngineContext.Provider>
);
}

View File

@@ -0,0 +1,58 @@
import { invokeDesktopCommand } from "@/desktop/tauri/invoke-desktop-command";
import { getTauri } from "@/utils/tauri";
export const DESKTOP_NOTIFICATION_CLICK_EVENT = "desktop-notification-click";
export interface DesktopNotificationInput {
title: string;
body?: string;
data?: Record<string, unknown>;
}
export interface DesktopNotificationClickPayload {
data?: Record<string, unknown>;
}
export type DesktopNotificationClickHandler = (
payload: DesktopNotificationClickPayload
) => void;
export type DesktopNotificationClickUnlisten = () => void;
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
export async function sendDesktopNotification(
input: DesktopNotificationInput
): Promise<boolean> {
try {
await invokeDesktopCommand("send_desktop_notification", { input });
return true;
} catch (error) {
console.warn("[OSNotifications][Desktop] Failed to send desktop notification", error);
return false;
}
}
export async function listenToDesktopNotificationClicks(
handler: DesktopNotificationClickHandler
): Promise<DesktopNotificationClickUnlisten> {
const listen = getTauri()?.event?.listen;
if (typeof listen !== "function") {
throw new Error("Tauri event API is unavailable.");
}
const unlisten = await listen(DESKTOP_NOTIFICATION_CLICK_EVENT, (event: unknown) => {
const payload = isRecord(event) && isRecord(event.payload) ? event.payload : null;
if (!payload) {
return;
}
handler({
data: isRecord(payload.data) ? payload.data : undefined,
});
});
return typeof unlisten === "function" ? unlisten : () => {};
}

View File

@@ -0,0 +1,107 @@
import { describe, expect, it, vi, beforeEach } from "vitest";
const tauriState = vi.hoisted(() => ({
api: null as any,
}));
vi.mock("@/utils/tauri", () => ({
getTauri: () => tauriState.api,
}));
import {
normalizePickedImageAssets,
openImagePathsWithTauriDialog,
} from "./image-attachment-picker";
describe("image-attachment-picker", () => {
beforeEach(() => {
tauriState.api = null;
});
it("normalizes a picked File into a blob source", async () => {
const file = new File(["hello"], "picked.png", { type: "image/png" });
const result = await normalizePickedImageAssets([
{
uri: "blob:test",
mimeType: "image/png",
fileName: null,
file,
},
]);
expect(result).toHaveLength(1);
expect(result[0]?.source.kind).toBe("blob");
expect(result[0]?.fileName).toBe("picked.png");
expect(result[0]?.mimeType).toBe("image/png");
});
it("keeps filesystem picker results as file uris", async () => {
const result = await normalizePickedImageAssets([
{
uri: "file:///tmp/picked.png",
mimeType: "image/png",
fileName: "picked.png",
},
]);
expect(result).toEqual([
{
source: { kind: "file_uri", uri: "file:///tmp/picked.png" },
mimeType: "image/png",
fileName: "picked.png",
},
]);
});
it("converts data urls into blob sources when no file path exists", async () => {
const result = await normalizePickedImageAssets([
{
uri: "data:image/png;base64,AAEC",
mimeType: "image/png",
fileName: "inline.png",
},
]);
expect(result).toHaveLength(1);
expect(result[0]?.source.kind).toBe("blob");
expect(result[0]?.fileName).toBe("inline.png");
expect(result[0]?.mimeType).toBe("image/png");
});
it("uses the tauri dialog api when available", async () => {
const open = vi.fn().mockResolvedValue(["/tmp/one.png", "/tmp/two.jpg"]);
tauriState.api = {
dialog: { open },
};
const result = await openImagePathsWithTauriDialog();
expect(open).toHaveBeenCalledWith(
expect.objectContaining({
multiple: true,
directory: false,
title: "Attach images",
})
);
expect(result).toEqual(["/tmp/one.png", "/tmp/two.jpg"]);
});
it("falls back to core invoke for the tauri dialog plugin", async () => {
const invoke = vi.fn().mockResolvedValue("/tmp/one.png");
tauriState.api = {
core: { invoke },
};
const result = await openImagePathsWithTauriDialog();
expect(invoke).toHaveBeenCalledWith("plugin:dialog|open", {
options: expect.objectContaining({
multiple: true,
directory: false,
title: "Attach images",
}),
});
expect(result).toEqual(["/tmp/one.png"]);
});
});

View File

@@ -0,0 +1,110 @@
import { getTauri } from "@/utils/tauri";
export type PickedImageSource =
| { kind: "file_uri"; uri: string }
| { kind: "blob"; blob: Blob };
export interface PickedImageAttachmentInput {
source: PickedImageSource;
mimeType?: string | null;
fileName?: string | null;
}
export interface ExpoImagePickerAssetLike {
uri: string;
mimeType?: string | null;
fileName?: string | null;
file?: File | null;
}
const IMAGE_FILE_EXTENSIONS = [
"png",
"jpg",
"jpeg",
"gif",
"webp",
"avif",
"heic",
"heif",
"tiff",
"bmp",
"svg",
];
function isAbsoluteWindowsPath(value: string): boolean {
return /^[a-zA-Z]:[\\/]/.test(value);
}
function shouldTreatAsFileUri(uri: string): boolean {
return uri.startsWith("file://") || uri.startsWith("/") || isAbsoluteWindowsPath(uri);
}
async function blobFromUri(uri: string): Promise<Blob> {
const response = await fetch(uri);
if (!response.ok) {
throw new Error(`Failed to read picked image from '${uri}'.`);
}
return await response.blob();
}
export async function normalizePickedImageAssets(
assets: readonly ExpoImagePickerAssetLike[]
): Promise<PickedImageAttachmentInput[]> {
return await Promise.all(
assets.map(async (asset) => {
if (asset.file instanceof Blob) {
return {
source: { kind: "blob", blob: asset.file },
mimeType: asset.mimeType ?? asset.file.type ?? null,
fileName: asset.fileName ?? asset.file.name ?? null,
};
}
if (shouldTreatAsFileUri(asset.uri)) {
return {
source: { kind: "file_uri", uri: asset.uri },
mimeType: asset.mimeType ?? null,
fileName: asset.fileName ?? null,
};
}
return {
source: { kind: "blob", blob: await blobFromUri(asset.uri) },
mimeType: asset.mimeType ?? null,
fileName: asset.fileName ?? null,
};
})
);
}
function normalizeTauriDialogSelection(selection: string | string[] | null): string[] {
if (!selection) {
return [];
}
return Array.isArray(selection) ? selection : [selection];
}
export async function openImagePathsWithTauriDialog(): Promise<string[]> {
const tauri = getTauri();
const options = {
directory: false,
multiple: true,
filters: [{ name: "Images", extensions: IMAGE_FILE_EXTENSIONS }],
title: "Attach images",
};
const dialogOpen = tauri?.dialog?.open;
if (typeof dialogOpen === "function") {
return normalizeTauriDialogSelection(await dialogOpen(options));
}
const invoke = tauri?.core?.invoke;
if (typeof invoke !== "function") {
throw new Error("Tauri dialog API is not available.");
}
const result = await invoke("plugin:dialog|open", { options });
return normalizeTauriDialogSelection(
Array.isArray(result) || typeof result === "string" || result === null ? result : null
);
}

View File

@@ -5,6 +5,7 @@ import {
shouldClearAgentAttention,
type AgentAttentionClearTrigger,
} from "@/utils/agent-attention";
import { getIsAppActivelyVisible } from "@/utils/app-visibility";
type AttentionReason = "finished" | "error" | "permission" | null | undefined;
@@ -23,20 +24,6 @@ interface AgentAttentionClearController {
clearOnAgentBlur: () => void;
}
function getIsAppVisible(): boolean {
const isAppStateActive = AppState.currentState === "active";
if (Platform.OS !== "web") {
return isAppStateActive;
}
const documentVisible =
typeof document === "undefined" || document.visibilityState === "visible";
const windowFocused =
typeof document === "undefined" ||
typeof document.hasFocus !== "function" ||
document.hasFocus();
return isAppStateActive && documentVisible && windowFocused;
}
export function useAgentAttentionClear({
agentId,
client,
@@ -45,12 +32,14 @@ export function useAgentAttentionClear({
attentionReason,
isScreenFocused,
}: UseAgentAttentionClearParams): AgentAttentionClearController {
const [isAppVisible, setIsAppVisible] = useState<boolean>(() => getIsAppVisible());
const [isAppVisible, setIsAppVisible] = useState<boolean>(() =>
getIsAppActivelyVisible()
);
const deferredFocusEntryClearRef = useRef(false);
const prevRequiresAttentionRef = useRef(Boolean(requiresAttention));
const prevActivelyViewedRef = useRef(isScreenFocused && getIsAppVisible());
const prevActivelyViewedRef = useRef(isScreenFocused && getIsAppActivelyVisible());
const prevScreenFocusedRef = useRef(false);
const prevAppVisibleRef = useRef(getIsAppVisible());
const prevAppVisibleRef = useRef(getIsAppActivelyVisible());
const clearAttention = useCallback(
(trigger: AgentAttentionClearTrigger) => {
@@ -78,7 +67,7 @@ export function useAgentAttentionClear({
useEffect(() => {
const updateVisibility = () => {
setIsAppVisible(getIsAppVisible());
setIsAppVisible(getIsAppActivelyVisible());
};
const appStateSubscription = AppState.addEventListener(

View File

@@ -1 +0,0 @@
export * from "./use-audio-player.native";

View File

@@ -1,366 +0,0 @@
import { useState, useRef } from "react";
import {
initialize,
playPCMData,
stopPlayback,
pausePlayback,
resumePlayback,
} from "@boudra/expo-two-way-audio";
interface QueuedAudio {
audioData: Blob;
resolve: (duration: number) => void;
reject: (error: Error) => void;
}
/**
* Resample PCM16 audio between sample rates.
* Speechmatics expects 16kHz.
*/
function resamplePcm16(pcm: Uint8Array, fromRate: number, toRate: number): Uint8Array {
if (fromRate === toRate) {
return pcm;
}
const inputSamples = Math.floor(pcm.length / 2);
const outputSamples = Math.floor((inputSamples * toRate) / fromRate);
const out = new Uint8Array(outputSamples * 2);
const ratio = fromRate / toRate;
const readInt16 = (sampleIndex: number): number => {
const i = sampleIndex * 2;
if (i + 1 >= pcm.length) {
return 0;
}
const lo = pcm[i]!;
const hi = pcm[i + 1]!;
let value = (hi << 8) | lo;
if (value & 0x8000) {
value = value - 0x10000;
}
return value;
};
const writeInt16 = (sampleIndex: number, value: number): void => {
const clamped = Math.max(-32768, Math.min(32767, Math.round(value)));
const i = sampleIndex * 2;
out[i] = clamped & 0xff;
out[i + 1] = (clamped >> 8) & 0xff;
};
for (let i = 0; i < outputSamples; i++) {
const srcPos = i * ratio;
const i0 = Math.floor(srcPos);
const frac = srcPos - i0;
const s0 = readInt16(i0);
const s1 = readInt16(Math.min(inputSamples - 1, i0 + 1));
writeInt16(i, s0 + (s1 - s0) * frac);
}
return out;
}
function parsePcmSampleRate(mimeType: string): number | null {
const match = /rate=(\d+)/i.exec(mimeType);
if (!match) {
return null;
}
const rate = Number(match[1]);
return Number.isFinite(rate) && rate > 0 ? rate : null;
}
export interface AudioPlayerOptions {
isDetecting?: () => boolean;
isSpeaking?: () => boolean;
}
/**
* Hook for audio playback using Speechmatics two-way audio with echo cancellation
*/
export function useAudioPlayer(options?: AudioPlayerOptions) {
const [isPlaying, setIsPlaying] = useState(false);
const [audioInitialized, setAudioInitialized] = useState(false);
const queueRef = useRef<QueuedAudio[]>([]);
const suppressedQueueRef = useRef<QueuedAudio[]>([]);
const isProcessingQueueRef = useRef(false);
const activePlaybackRef = useRef<{
resolve: (duration: number) => void;
reject: (error: Error) => void;
} | null>(null);
const playbackTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const checkIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
async function play(audioData: Blob): Promise<number> {
return new Promise((resolve, reject) => {
// Check if we should suppress playback due to voice detection/speaking
const shouldSuppress =
(options?.isDetecting && options.isDetecting()) ||
(options?.isSpeaking && options.isSpeaking());
if (shouldSuppress) {
console.log("[AudioPlayer] Suppressing playback - voice detection/speaking active");
// Add to suppressed queue instead
suppressedQueueRef.current.push({ audioData, resolve, reject });
// Start checking for when flags clear
startCheckingForClearFlags();
return;
}
// Add to queue with its promise handlers
queueRef.current.push({ audioData, resolve, reject });
// Start processing queue if not already processing
if (!isProcessingQueueRef.current) {
processQueue();
}
});
}
function startCheckingForClearFlags(): void {
// Already checking
if (checkIntervalRef.current !== null) {
return;
}
console.log("[AudioPlayer] Starting to check for clear flags");
checkIntervalRef.current = setInterval(() => {
const isStillBlocked =
(options?.isDetecting && options.isDetecting()) ||
(options?.isSpeaking && options.isSpeaking());
if (!isStillBlocked && suppressedQueueRef.current.length > 0) {
console.log("[AudioPlayer] Flags cleared - moving suppressed queue to main queue");
// Move all suppressed items to main queue
const suppressedItems = [...suppressedQueueRef.current];
suppressedQueueRef.current = [];
// Add to front of main queue (they were waiting)
queueRef.current = [...suppressedItems, ...queueRef.current];
// Stop checking
if (checkIntervalRef.current !== null) {
clearInterval(checkIntervalRef.current);
checkIntervalRef.current = null;
}
// Start processing if not already
if (!isProcessingQueueRef.current) {
processQueue();
}
} else if (!isStillBlocked && suppressedQueueRef.current.length === 0) {
// No more suppressed items and flags are clear - stop checking
if (checkIntervalRef.current !== null) {
clearInterval(checkIntervalRef.current);
checkIntervalRef.current = null;
}
}
}, 100); // Check every 100ms
}
async function processQueue(): Promise<void> {
if (isProcessingQueueRef.current || queueRef.current.length === 0) {
return;
}
isProcessingQueueRef.current = true;
while (queueRef.current.length > 0) {
// Before processing each item, check if flags became active
const shouldSuppress =
(options?.isDetecting && options.isDetecting()) ||
(options?.isSpeaking && options.isSpeaking());
if (shouldSuppress) {
console.log("[AudioPlayer] Flags became active during processing - moving remaining queue to suppressed");
// Move remaining queue to suppressed
suppressedQueueRef.current = [...queueRef.current, ...suppressedQueueRef.current];
queueRef.current = [];
startCheckingForClearFlags();
break;
}
const item = queueRef.current.shift()!;
try {
const duration = await playAudio(item.audioData);
item.resolve(duration);
} catch (error) {
item.reject(error as Error);
}
}
isProcessingQueueRef.current = false;
}
async function processNextInQueue(): Promise<void> {
if (queueRef.current.length > 0) {
const item = queueRef.current.shift()!;
try {
const duration = await playAudio(item.audioData);
item.resolve(duration);
} catch (error) {
item.reject(error as Error);
}
} else {
isProcessingQueueRef.current = false;
}
}
async function playAudio(audioData: Blob): Promise<number> {
return new Promise(async (resolve, reject) => {
activePlaybackRef.current = { resolve, reject };
try {
console.log(
`[AudioPlayer] Playing audio (${audioData.size} bytes, type: ${audioData.type})`
);
// Initialize audio if not already initialized
if (!audioInitialized) {
console.log("[AudioPlayer] Initializing audio...");
await initialize();
setAudioInitialized(true);
console.log(
"[AudioPlayer] ✅ Initialized (Speechmatics two-way audio)"
);
}
// Workaround: Resume playback before playing new audio to ensure the audio engine is ready
// This fixes the issue where playback doesn't work after calling stopPlayback()
console.log("[AudioPlayer] Resuming playback engine...");
resumePlayback();
// Get PCM data from blob (server sends PCM16)
const arrayBuffer = await audioData.arrayBuffer();
const pcm = new Uint8Array(arrayBuffer);
const inputRate = parsePcmSampleRate(audioData.type || "") ?? 24000;
const pcm16k = resamplePcm16(pcm, inputRate, 16000);
// Calculate total duration
const samples = pcm16k.length / 2; // 16-bit = 2 bytes per sample
const durationSec = samples / 16000; // 16kHz sample rate
const audioSizeKb = (pcm16k.length / 1024).toFixed(2);
console.log(
"[AudioPlayer] 🔊 Playing audio:",
audioSizeKb,
"KB, duration:",
durationSec.toFixed(2),
"s"
);
setIsPlaying(true);
// Play entire PCM data at once through Speechmatics
playPCMData(pcm16k);
// Clear any existing timeout
if (playbackTimeoutRef.current) {
clearTimeout(playbackTimeoutRef.current);
}
// Wait for playback to finish (estimate based on duration)
playbackTimeoutRef.current = setTimeout(() => {
console.log("[AudioPlayer] ✅ Playback finished");
setIsPlaying(false);
playbackTimeoutRef.current = null;
activePlaybackRef.current = null;
resolve(durationSec);
}, durationSec * 1000);
} catch (error) {
console.error("[AudioPlayer] Error playing audio:", error);
// Clear timeout on error
if (playbackTimeoutRef.current) {
clearTimeout(playbackTimeoutRef.current);
playbackTimeoutRef.current = null;
}
setIsPlaying(false);
activePlaybackRef.current = null;
reject(error);
}
});
}
function stop(): void {
if (isPlaying) {
console.log("[AudioPlayer] 🛑 Stopping playback (interrupted)");
// Stop native playback
stopPlayback();
// Clear playback timeout
if (playbackTimeoutRef.current) {
clearTimeout(playbackTimeoutRef.current);
playbackTimeoutRef.current = null;
}
}
setIsPlaying(false);
// Reject the currently playing promise, if any.
if (activePlaybackRef.current) {
activePlaybackRef.current.reject(new Error("Playback stopped"));
activePlaybackRef.current = null;
}
// Reject all pending promises in the main queue
while (queueRef.current.length > 0) {
const item = queueRef.current.shift()!;
item.reject(new Error("Playback stopped"));
}
// Reject all pending promises in the suppressed queue
while (suppressedQueueRef.current.length > 0) {
const item = suppressedQueueRef.current.shift()!;
item.reject(new Error("Playback stopped"));
}
// Clear check interval
if (checkIntervalRef.current !== null) {
clearInterval(checkIntervalRef.current);
checkIntervalRef.current = null;
}
isProcessingQueueRef.current = false;
}
function clearQueue(): void {
// Reject all pending promises in the main queue
while (queueRef.current.length > 0) {
const item = queueRef.current.shift()!;
item.reject(new Error("Queue cleared"));
}
// Reject all pending promises in the suppressed queue
while (suppressedQueueRef.current.length > 0) {
const item = suppressedQueueRef.current.shift()!;
item.reject(new Error("Queue cleared"));
}
// Clear check interval
if (checkIntervalRef.current !== null) {
clearInterval(checkIntervalRef.current);
checkIntervalRef.current = null;
}
}
return {
play,
stop,
isPlaying: () => isPlaying,
clearQueue,
warmup: async () => {
if (!audioInitialized) {
await initialize();
setAudioInitialized(true);
}
// Ensure playback engine isn't suspended after a previous stop.
resumePlayback();
},
};
}

View File

@@ -1,275 +0,0 @@
import { useMemo, useRef, useState } from "react";
export interface AudioPlayerOptions {
isDetecting?: () => boolean;
isSpeaking?: () => boolean;
}
/**
* Web audio player for server-provided audio chunks.
*
* Supports:
* - `audio/pcm` (assumed PCM16 LE, mono, default 24kHz unless `rate=` is present)
* - formats supported by `AudioContext.decodeAudioData` (e.g. mp3)
*/
export function useAudioPlayer(options?: AudioPlayerOptions) {
const [isPlayingState, setIsPlayingState] = useState(false);
type QueuedAudio = {
audioData: Blob;
resolve: (duration: number) => void;
reject: (error: Error) => void;
};
const audioContextRef = useRef<AudioContext | null>(null);
const queueRef = useRef<QueuedAudio[]>([]);
const suppressedQueueRef = useRef<QueuedAudio[]>([]);
const isProcessingQueueRef = useRef(false);
const checkIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
const activePlaybackRef = useRef<{
source: AudioBufferSourceNode;
resolve: (duration: number) => void;
reject: (error: Error) => void;
settled: boolean;
} | null>(null);
const decodeAudioData = async (context: AudioContext, buffer: ArrayBuffer): Promise<AudioBuffer> => {
const maybePromise = context.decodeAudioData(buffer);
if (maybePromise && typeof (maybePromise as Promise<AudioBuffer>).then === "function") {
return maybePromise as Promise<AudioBuffer>;
}
return await new Promise<AudioBuffer>((resolve, reject) => {
context.decodeAudioData(buffer, resolve, reject);
});
};
const parsePcmSampleRate = (mimeType: string): number | null => {
const match = /rate=(\\d+)/i.exec(mimeType);
if (!match) {
return null;
}
const rate = Number(match[1]);
return Number.isFinite(rate) && rate > 0 ? rate : null;
};
const pcm16LeToAudioBuffer = (context: AudioContext, bytes: Uint8Array, sampleRate: number): AudioBuffer => {
const sampleCount = Math.floor(bytes.length / 2);
const audioBuffer = context.createBuffer(1, sampleCount, sampleRate);
const channel = audioBuffer.getChannelData(0);
for (let i = 0; i < sampleCount; i++) {
const lo = bytes[i * 2]!;
const hi = bytes[i * 2 + 1]!;
let value = (hi << 8) | lo;
if (value & 0x8000) {
value = value - 0x10000;
}
channel[i] = value / 0x8000;
}
return audioBuffer;
};
const ensureContext = async (): Promise<AudioContext> => {
if (audioContextRef.current) {
if (audioContextRef.current.state === "suspended") {
try {
await audioContextRef.current.resume();
} catch {
// Best effort. If this fails due to autoplay policies, a later user gesture
// (or explicit warmup call) can unlock it.
}
}
return audioContextRef.current;
}
const context = new AudioContext();
if (context.state === "suspended") {
try {
await context.resume();
} catch {
// See note above.
}
}
audioContextRef.current = context;
return context;
};
const shouldSuppressPlayback = (): boolean => {
return Boolean(options?.isDetecting?.()) || Boolean(options?.isSpeaking?.());
};
const startCheckingForClearFlags = (): void => {
if (checkIntervalRef.current !== null) {
return;
}
checkIntervalRef.current = setInterval(() => {
const isStillBlocked = shouldSuppressPlayback();
if (!isStillBlocked && suppressedQueueRef.current.length > 0) {
const suppressedItems = [...suppressedQueueRef.current];
suppressedQueueRef.current = [];
queueRef.current = [...suppressedItems, ...queueRef.current];
if (checkIntervalRef.current !== null) {
clearInterval(checkIntervalRef.current);
checkIntervalRef.current = null;
}
if (!isProcessingQueueRef.current) {
void processQueue();
}
} else if (!isStillBlocked && suppressedQueueRef.current.length === 0) {
if (checkIntervalRef.current !== null) {
clearInterval(checkIntervalRef.current);
checkIntervalRef.current = null;
}
}
}, 100);
};
const playAudio = async (audioData: Blob): Promise<number> => {
const context = await ensureContext();
const arrayBuffer = await audioData.arrayBuffer();
let audioBuffer: AudioBuffer;
const type = (audioData.type || "").toLowerCase();
if (type.startsWith("audio/pcm")) {
const sampleRate = parsePcmSampleRate(type) ?? 24000;
audioBuffer = pcm16LeToAudioBuffer(context, new Uint8Array(arrayBuffer), sampleRate);
} else {
audioBuffer = await decodeAudioData(context, arrayBuffer);
}
const durationSec = audioBuffer.duration;
const source = context.createBufferSource();
source.buffer = audioBuffer;
source.connect(context.destination);
return await new Promise<number>((resolve, reject) => {
activePlaybackRef.current = { source, resolve, reject, settled: false };
setIsPlayingState(true);
const settleOnce = (fn: () => void) => {
const active = activePlaybackRef.current;
if (!active || active.source !== source || active.settled) {
return;
}
active.settled = true;
activePlaybackRef.current = null;
setIsPlayingState(false);
fn();
};
source.onended = () => {
settleOnce(() => resolve(durationSec));
};
try {
source.start();
} catch (e) {
settleOnce(() => reject(e instanceof Error ? e : new Error(String(e))));
}
});
};
const processQueue = async (): Promise<void> => {
if (isProcessingQueueRef.current || queueRef.current.length === 0) {
return;
}
isProcessingQueueRef.current = true;
while (queueRef.current.length > 0) {
if (shouldSuppressPlayback()) {
suppressedQueueRef.current = [...queueRef.current, ...suppressedQueueRef.current];
queueRef.current = [];
startCheckingForClearFlags();
break;
}
const item = queueRef.current.shift()!;
try {
const duration = await playAudio(item.audioData);
item.resolve(duration);
} catch (error) {
item.reject(error as Error);
}
}
isProcessingQueueRef.current = false;
};
const play = async (audioData: Blob): Promise<number> => {
return await new Promise<number>((resolve, reject) => {
if (shouldSuppressPlayback()) {
suppressedQueueRef.current.push({ audioData, resolve, reject });
startCheckingForClearFlags();
return;
}
queueRef.current.push({ audioData, resolve, reject });
if (!isProcessingQueueRef.current) {
void processQueue();
}
});
};
const stop = (): void => {
// Stop currently playing audio (and reject its promise)
if (activePlaybackRef.current) {
const active = activePlaybackRef.current;
activePlaybackRef.current = null;
try {
active.source.onended = null;
active.source.stop();
} catch {
// ignore
}
if (!active.settled) {
active.settled = true;
active.reject(new Error("Playback stopped"));
}
}
setIsPlayingState(false);
while (queueRef.current.length > 0) {
queueRef.current.shift()!.reject(new Error("Playback stopped"));
}
while (suppressedQueueRef.current.length > 0) {
suppressedQueueRef.current.shift()!.reject(new Error("Playback stopped"));
}
if (checkIntervalRef.current !== null) {
clearInterval(checkIntervalRef.current);
checkIntervalRef.current = null;
}
isProcessingQueueRef.current = false;
};
const clearQueue = (): void => {
while (queueRef.current.length > 0) {
queueRef.current.shift()!.reject(new Error("Queue cleared"));
}
while (suppressedQueueRef.current.length > 0) {
suppressedQueueRef.current.shift()!.reject(new Error("Queue cleared"));
}
if (checkIntervalRef.current !== null) {
clearInterval(checkIntervalRef.current);
checkIntervalRef.current = null;
}
};
return useMemo(
() => ({
play,
stop,
isPlaying: () => isPlayingState,
clearQueue,
warmup: async () => {
await ensureContext();
},
}),
[isPlayingState]
);
}

View File

@@ -18,41 +18,27 @@ export interface AudioCaptureConfig {
*/
async function getActualRecordingUri(createdAt: Date): Promise<string | null> {
try {
console.log('[AudioRecorder] Searching for recording file created at:', createdAt.toISOString());
const audioDir = new Directory(Paths.cache, 'Audio');
console.log('[AudioRecorder] Audio cache directory URI:', audioDir.uri);
console.log('[AudioRecorder] Directory exists:', audioDir.exists);
if (!audioDir.exists) {
console.log('[AudioRecorder] Audio cache directory does not exist');
return null;
}
const files = audioDir.list();
console.log('[AudioRecorder] Found files in Audio cache:', files.length);
if (!files.length) {
console.log('[AudioRecorder] No files found in Audio cache directory');
return null;
}
const validFiles = files
.map(file => {
const info = file.info();
console.log('[AudioRecorder] File info:', {
uri: info.uri,
size: info.size,
creationTime: info.creationTime ? new Date(info.creationTime).toISOString() : null,
});
return info;
})
.filter(f => f.size && f.size > 0);
console.log('[AudioRecorder] Valid files (size > 0):', validFiles.length);
if (validFiles.length === 0) {
console.log('[AudioRecorder] No valid files found (all are zero-byte)');
return null;
}
@@ -62,10 +48,6 @@ async function getActualRecordingUri(createdAt: Date): Promise<string | null> {
for (const file of validFiles) {
if (!file.creationTime || !file.uri) continue;
const diff = Math.abs(file.creationTime - createdAt.getTime());
console.log('[AudioRecorder] Time diff for file:', {
uri: file.uri,
diffMs: diff,
});
if (diff < minDiff) {
closest = file;
minDiff = diff;
@@ -74,15 +56,9 @@ async function getActualRecordingUri(createdAt: Date): Promise<string | null> {
if (closest) {
const resultUri = closest.uri?.slice(0, -1) ?? null;
console.log('[AudioRecorder] Found closest file:', {
uri: resultUri,
size: closest.size,
timeDiffMs: minDiff,
});
return resultUri;
}
console.log('[AudioRecorder] No closest file found');
return null;
} catch (e) {
console.error('[AudioRecorder] Error finding actual recording file:', e);
@@ -208,7 +184,6 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
attemptGuardRef.current.assertCurrent(attemptId);
// Request microphone permissions
console.log('[AudioRecorder] Requesting recording permissions...');
const permissionResponse = await requestRecordingPermissionsAsync();
attemptGuardRef.current.assertCurrent(attemptId);
@@ -217,38 +192,26 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
}
// Configure audio mode for recording
console.log('[AudioRecorder] Configuring audio mode...');
await setAudioModeAsync({
playsInSilentMode: true,
allowsRecording: true,
});
attemptGuardRef.current.assertCurrent(attemptId);
console.log('[AudioRecorder] Starting recording with options:', {
sampleRate: recordingOptions.sampleRate,
numberOfChannels: recordingOptions.numberOfChannels,
bitRate: recordingOptions.bitRate,
});
const startTime = new Date();
setRecordingStartTime(startTime);
attemptGuardRef.current.assertCurrent(attemptId);
// Prepare the recorder before recording (required step)
console.log('[AudioRecorder] Preparing recorder...');
await recorder.prepareToRecordAsync();
attemptGuardRef.current.assertCurrent(attemptId);
console.log('[AudioRecorder] Starting recording...');
await recorder.record();
attemptGuardRef.current.assertCurrent(attemptId);
console.log('[AudioRecorder] Recording started at:', startTime.toISOString());
console.log('[AudioRecorder] Recorder isRecording:', recorder.isRecording);
} catch (error: any) {
setRecordingStartTime(null);
if (error instanceof AttemptCancelledError) {
console.log('[AudioRecorder] Recording start cancelled.');
return;
}
if (error?.message !== 'Recording cancelled') {
@@ -278,15 +241,12 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
// Get URI from recorder
let uri = recorder.uri;
console.log('[AudioRecorder] Initial URI from recorder:', uri);
// Workaround for Expo SDK 54 Android bug - find actual recording file
if (recordingStartTime && (!uri || uri === '')) {
console.log('[AudioRecorder] Using workaround to find actual recording file...');
const actualUri = await getActualRecordingUri(recordingStartTime);
if (actualUri) {
uri = actualUri;
console.log('[AudioRecorder] Found actual recording URI:', uri);
}
}
@@ -299,7 +259,6 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
// Get file info
const file = new File(uri);
const exists = file.exists;
console.log('[AudioRecorder] File exists:', exists);
if (!exists) {
setRecordingStartTime(null);
@@ -309,11 +268,6 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
// Convert URI to Blob
const audioBlob = await uriToBlob(uri);
console.log('[AudioRecorder] Recording converted to blob:', {
size: audioBlob.size,
type: audioBlob.type,
});
// Clean up the temporary file
file.delete();

View File

@@ -163,16 +163,6 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
throw new Error("Microphone capture is not supported in this environment");
}
console.log("[AudioRecorder][Web] Microphone preflight", {
secureContext,
currentOrigin,
isTauri,
hasMediaDevices:
typeof navigator !== "undefined" &&
!!navigator.mediaDevices &&
typeof navigator.mediaDevices.getUserMedia === "function",
});
if (!secureContext && !isTauri) {
throw new Error(
`Microphone access requires HTTPS or localhost. Current origin: ${currentOrigin}`

View File

@@ -1,44 +1,63 @@
import { useCallback, useEffect, useRef } from "react";
import { Buffer } from "buffer";
import { useState } from "react";
import { useSpeechmaticsAudio } from "@/hooks/use-speechmatics-audio";
import { createAudioEngine } from "@/voice/audio-engine";
import type { DictationAudioSource, DictationAudioSourceConfig } from "./use-dictation-audio-source.types";
export function useDictationAudioSource(config: DictationAudioSourceConfig): DictationAudioSource {
const onPcmSegmentRef = useRef(config.onPcmSegment);
const onErrorRef = useRef(config.onError);
const [volume, setVolume] = useState(0);
const engineRef = useRef<ReturnType<typeof createAudioEngine> | null>(null);
const getOrCreateEngine = useCallback(() => {
if (engineRef.current) {
return engineRef.current;
}
engineRef.current = createAudioEngine({
onCaptureData: (pcm) => {
onPcmSegmentRef.current(Buffer.from(pcm).toString("base64"));
},
onVolumeLevel: (level) => {
setVolume(level);
},
onError: (error) => {
onErrorRef.current?.(error);
},
});
return engineRef.current;
}, []);
useEffect(() => {
onPcmSegmentRef.current = config.onPcmSegment;
onErrorRef.current = config.onError;
}, [config.onPcmSegment, config.onError]);
const speechmatics = useSpeechmaticsAudio({
enableContinuousStreaming: true,
onAudioSegment: ({ audioData }) => {
onPcmSegmentRef.current(audioData);
},
onError: (err) => {
onErrorRef.current?.(err);
},
volumeThreshold: 0.3,
silenceDuration: 2000,
speechConfirmationDuration: 300,
detectionGracePeriod: 200,
});
const start = useCallback(async () => {
await speechmatics.start();
}, [speechmatics]);
const engine = getOrCreateEngine();
await engine.initialize();
await engine.startCapture();
}, [getOrCreateEngine]);
const stop = useCallback(async () => {
await speechmatics.stop();
}, [speechmatics]);
await engineRef.current?.stopCapture();
setVolume(0);
}, []);
useEffect(() => {
return () => {
const engine = engineRef.current;
engineRef.current = null;
void engine?.destroy().catch(() => undefined);
};
}, []);
return {
start,
stop,
volume: speechmatics.volume,
volume,
};
}

View File

@@ -1,4 +1,5 @@
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { parsePcm16Wav } from "@/utils/pcm16-wav";
import { getTauri } from "@/utils/tauri";
import type { DictationAudioSource, DictationAudioSourceConfig } from "./use-dictation-audio-source.types";
@@ -65,84 +66,6 @@ const int16ToFloat32 = (input: Int16Array): Float32Array => {
return out;
};
type Pcm16Wav = {
sampleRate: number;
samples: Int16Array;
};
const parsePcm16Wav = (buffer: ArrayBuffer): Pcm16Wav | null => {
if (buffer.byteLength < 44) {
return null;
}
const view = new DataView(buffer);
const readAscii = (offset: number, length: number): string => {
let out = "";
for (let i = 0; i < length; i += 1) {
out += String.fromCharCode(view.getUint8(offset + i));
}
return out;
};
if (readAscii(0, 4) !== "RIFF" || readAscii(8, 4) !== "WAVE") {
return null;
}
let offset = 12;
let channels = 0;
let sampleRate = 0;
let bitsPerSample = 0;
let dataOffset = 0;
let dataSize = 0;
while (offset + 8 <= buffer.byteLength) {
const chunkId = readAscii(offset, 4);
const chunkSize = view.getUint32(offset + 4, true);
const chunkDataOffset = offset + 8;
if (chunkDataOffset + chunkSize > buffer.byteLength) {
break;
}
if (chunkId === "fmt " && chunkSize >= 16) {
const audioFormat = view.getUint16(chunkDataOffset, true);
channels = view.getUint16(chunkDataOffset + 2, true);
sampleRate = view.getUint32(chunkDataOffset + 4, true);
bitsPerSample = view.getUint16(chunkDataOffset + 14, true);
if (audioFormat !== 1) {
return null;
}
} else if (chunkId === "data") {
dataOffset = chunkDataOffset;
dataSize = chunkSize;
break;
}
offset = chunkDataOffset + chunkSize + (chunkSize % 2);
}
if (!dataOffset || !dataSize || sampleRate <= 0 || bitsPerSample !== 16 || channels <= 0) {
return null;
}
const sampleCount = Math.floor(dataSize / 2);
const interleaved = new Int16Array(buffer, dataOffset, sampleCount);
if (channels === 1) {
return { sampleRate, samples: new Int16Array(interleaved) };
}
const frameCount = Math.floor(interleaved.length / channels);
const mono = new Int16Array(frameCount);
for (let frame = 0; frame < frameCount; frame += 1) {
let sum = 0;
for (let ch = 0; ch < channels; ch += 1) {
sum += interleaved[frame * channels + ch] ?? 0;
}
mono[frame] = Math.round(sum / channels);
}
return { sampleRate, samples: mono };
};
const int16ToBase64 = (pcm: Int16Array): string => {
const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
let binary = "";
@@ -240,15 +163,6 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
if (missingNavigator) {
throw new Error("Microphone capture is not supported in this environment");
}
console.log("[DictationAudio][Web] Microphone preflight", {
secureContext,
currentOrigin,
isTauri,
hasMediaDevices:
typeof navigator !== "undefined" &&
!!navigator.mediaDevices &&
typeof navigator.mediaDevices.getUserMedia === "function",
});
if (!secureContext && !isTauri) {
throw new Error(`Microphone access requires HTTPS or localhost. Current origin: ${currentOrigin}`);
}

View File

@@ -13,18 +13,6 @@ interface UseExplorerOpenGestureParams {
onOpen: () => void;
}
const IS_DEV = Boolean((globalThis as { __DEV__?: boolean }).__DEV__);
function logExplorerOpenGesture(
event: string,
details: Record<string, unknown>
): void {
if (!IS_DEV) {
return;
}
console.log(`[ExplorerOpenGesture] ${event}`, details);
}
export function useExplorerOpenGesture({
enabled,
onOpen,
@@ -82,7 +70,6 @@ export function useExplorerOpenGesture({
})
.onStart(() => {
isGesturing.value = true;
runOnJS(logExplorerOpenGesture)("start", { enabled });
})
.onUpdate((event) => {
// Right sidebar: start from closed position (+windowWidth) and move towards 0.
@@ -103,15 +90,6 @@ export function useExplorerOpenGesture({
const shouldOpenByPosition = translateX.value < (windowWidth * 2) / 3;
const shouldOpenByVelocity = event.velocityX < -500;
const shouldOpen = shouldOpenByPosition || shouldOpenByVelocity;
runOnJS(logExplorerOpenGesture)("end", {
translationX: event.translationX,
velocityX: event.velocityX,
panelTranslateX: translateX.value,
windowWidth,
shouldOpenByPosition,
shouldOpenByVelocity,
shouldOpen,
});
if (shouldOpen) {
animateToOpen();
runOnJS(onOpen)();

View File

@@ -1,11 +1,16 @@
import { useCallback, useRef } from "react";
import { Alert } from "react-native";
import { Platform } from "react-native";
import * as ImagePicker from "expo-image-picker";
type ImagePickerSuccessResult = ImagePicker.ImagePickerSuccessResult;
import { isTauriEnvironment } from "@/utils/tauri";
import {
normalizePickedImageAssets,
openImagePathsWithTauriDialog,
type PickedImageAttachmentInput,
} from "@/hooks/image-attachment-picker";
interface UseImageAttachmentPickerResult {
pickImages: () => Promise<ImagePickerSuccessResult | null>;
pickImages: () => Promise<PickedImageAttachmentInput[] | null>;
}
export function useImageAttachmentPicker(): UseImageAttachmentPickerResult {
@@ -37,6 +42,18 @@ export function useImageAttachmentPicker(): UseImageAttachmentPickerResult {
isPickingRef.current = true;
try {
if (Platform.OS === "web" && isTauriEnvironment()) {
const selectedPaths = await openImagePathsWithTauriDialog();
if (selectedPaths.length === 0) {
return null;
}
return selectedPaths.map((path) => ({
source: { kind: "file_uri" as const, uri: path },
mimeType: null,
fileName: null,
}));
}
const hasPermission = await ensurePermission();
if (!hasPermission) {
return null;
@@ -44,7 +61,7 @@ export function useImageAttachmentPicker(): UseImageAttachmentPickerResult {
const pendingResult = await ImagePicker.getPendingResultAsync();
if (pendingResult && "canceled" in pendingResult && !pendingResult.canceled) {
return pendingResult as ImagePickerSuccessResult;
return await normalizePickedImageAssets(pendingResult.assets);
}
const result = await ImagePicker.launchImageLibraryAsync({
@@ -57,7 +74,7 @@ export function useImageAttachmentPicker(): UseImageAttachmentPickerResult {
return null;
}
return result;
return await normalizePickedImageAssets(result.assets);
} catch (error) {
console.error("[ImageAttachmentPicker] Failed to pick image:", error);
Alert.alert("Error", "Failed to select image");

View File

@@ -0,0 +1,71 @@
import { useCallback, useEffect, useMemo, useState } from 'react'
import type { SidebarProjectEntry } from '@/hooks/use-sidebar-workspaces-list'
import { useKeyboardShortcutsStore } from '@/stores/keyboard-shortcuts-store'
import { buildSidebarShortcutModel } from '@/utils/sidebar-shortcuts'
export function useSidebarShortcutModel(projects: SidebarProjectEntry[]) {
const [collapsedProjectKeys, setCollapsedProjectKeys] = useState<Set<string>>(new Set())
const setSidebarShortcutWorkspaceTargets = useKeyboardShortcutsStore(
(state) => state.setSidebarShortcutWorkspaceTargets
)
const setVisibleWorkspaceTargets = useKeyboardShortcutsStore(
(state) => state.setVisibleWorkspaceTargets
)
const shortcutModel = useMemo(
() =>
buildSidebarShortcutModel({
projects,
collapsedProjectKeys,
}),
[collapsedProjectKeys, projects]
)
useEffect(() => {
setCollapsedProjectKeys((prev) => {
const validProjectKeys = new Set(projects.map((project) => project.projectKey))
const next = new Set<string>()
for (const key of prev) {
if (validProjectKeys.has(key)) {
next.add(key)
}
}
return next
})
}, [projects])
useEffect(() => {
setSidebarShortcutWorkspaceTargets(shortcutModel.shortcutTargets)
setVisibleWorkspaceTargets(shortcutModel.visibleTargets)
}, [
setSidebarShortcutWorkspaceTargets,
setVisibleWorkspaceTargets,
shortcutModel.shortcutTargets,
shortcutModel.visibleTargets,
])
useEffect(() => {
return () => {
setSidebarShortcutWorkspaceTargets([])
setVisibleWorkspaceTargets([])
}
}, [setSidebarShortcutWorkspaceTargets, setVisibleWorkspaceTargets])
const toggleProjectCollapsed = useCallback((projectKey: string) => {
setCollapsedProjectKeys((prev) => {
const next = new Set(prev)
if (next.has(projectKey)) {
next.delete(projectKey)
} else {
next.add(projectKey)
}
return next
})
}, [])
return {
collapsedProjectKeys,
shortcutIndexByWorkspaceKey: shortcutModel.shortcutIndexByWorkspaceKey,
toggleProjectCollapsed,
}
}

View File

@@ -267,8 +267,6 @@ export function useSidebarWorkspacesList(options?: {
const value = options?.serverId
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null
}, [options?.serverId])
const enabled = options?.enabled ?? true
const persistedProjectOrder = useSidebarOrderStore((state) =>
serverId ? (state.projectOrderByServerId[serverId] ?? EMPTY_ORDER) : EMPTY_ORDER
)
@@ -345,7 +343,7 @@ export function useSidebarWorkspacesList(options?: {
}, [persistedProjectOrder, persistedWorkspaceOrderByScope, projects, serverId])
const refreshAll = useCallback(() => {
if (!isActive || !serverId || connectionStatus !== 'online' || !enabled) {
if (!isActive || !serverId || connectionStatus !== 'online') {
return
}
const client = runtime.getClient(serverId)
@@ -377,7 +375,7 @@ export function useSidebarWorkspacesList(options?: {
// ignore explicit refresh failures; hook keeps existing data
}
})()
}, [connectionStatus, enabled, isActive, runtime, serverId])
}, [connectionStatus, isActive, runtime, serverId])
const isLoading = isActive && Boolean(serverId) && connectionStatus === 'online' && !hasHydratedWorkspaces
const isInitialLoad = isLoading && projects.length === 0

View File

@@ -1 +0,0 @@
export * from "./use-speechmatics-audio.native";

View File

@@ -1,274 +0,0 @@
import { useState, useEffect, useRef, useCallback } from "react";
import {
initialize,
useMicrophonePermissions,
toggleRecording,
tearDown,
useExpoTwoWayAudioEventListener,
type MicrophoneDataCallback,
type VolumeLevelCallback,
} from "@boudra/expo-two-way-audio";
import { SpeechSegmenter } from "@/voice/speech-segmenter";
export interface SpeechmaticsAudioConfig {
onAudioSegment?: (segment: { audioData: string; isLast: boolean }) => void;
onSpeechStart?: () => void;
onSpeechEnd?: () => void;
onError?: (error: Error) => void;
/** When true, stream microphone PCM continuously without VAD gating. */
enableContinuousStreaming?: boolean;
volumeThreshold: number; // Volume threshold for speech detection (0-1)
/** ms dip debounce before VAD transitions from confirmed speaking to non-speaking */
confirmedDropGracePeriod?: number;
silenceDuration: number; // ms of silence before ending segment
speechConfirmationDuration: number; // ms of sustained speech before confirming
detectionGracePeriod: number; // ms grace period for volume dips during detection
}
export interface SpeechmaticsAudio {
start: () => Promise<void>;
stop: () => Promise<void>;
toggleMute: () => void;
isActive: boolean;
isSpeaking: boolean;
isDetecting: boolean;
isMuted: boolean;
volume: number;
segmentDuration: number;
}
/**
* Hook for audio capture with echo cancellation using Speechmatics expo-two-way-audio
*/
export function useSpeechmaticsAudio(
config: SpeechmaticsAudioConfig
): SpeechmaticsAudio {
const [microphonePermission, requestMicrophonePermission] =
useMicrophonePermissions();
const [isActive, setIsActive] = useState(false);
const [isSpeaking, setIsSpeaking] = useState(false);
const [isDetecting, setIsDetecting] = useState(false);
const [audioInitialized, setAudioInitialized] = useState(false);
const [volume, setVolume] = useState(0);
const [isMuted, setIsMuted] = useState(false);
const [segmentDuration, setSegmentDuration] = useState(0);
const enableContinuousStreaming = config.enableContinuousStreaming === true;
const isActiveRef = useRef(isActive);
useEffect(() => {
isActiveRef.current = isActive;
}, [isActive]);
const isMutedRef = useRef(isMuted);
useEffect(() => {
isMutedRef.current = isMuted;
}, [isMuted]);
const callbacksRef = useRef({
onAudioSegment: config.onAudioSegment,
onSpeechStart: config.onSpeechStart,
onSpeechEnd: config.onSpeechEnd,
});
useEffect(() => {
callbacksRef.current = {
onAudioSegment: config.onAudioSegment,
onSpeechStart: config.onSpeechStart,
onSpeechEnd: config.onSpeechEnd,
};
}, [config.onAudioSegment, config.onSpeechStart, config.onSpeechEnd]);
const segmenterRef = useRef<SpeechSegmenter | null>(null);
if (segmenterRef.current === null) {
segmenterRef.current = new SpeechSegmenter(
{
enableContinuousStreaming,
volumeThreshold: config.volumeThreshold,
confirmedDropGracePeriodMs: config.confirmedDropGracePeriod,
silenceDurationMs: config.silenceDuration,
speechConfirmationMs: config.speechConfirmationDuration,
detectionGracePeriodMs: config.detectionGracePeriod,
},
{
onAudioSegment: (segment) => callbacksRef.current.onAudioSegment?.(segment),
onSpeechStart: () => callbacksRef.current.onSpeechStart?.(),
onSpeechEnd: () => callbacksRef.current.onSpeechEnd?.(),
onDetectingChange: (next) => setIsDetecting(next),
onSpeakingChange: (next) => setIsSpeaking(next),
}
);
}
useEffect(() => {
segmenterRef.current?.updateConfig({
enableContinuousStreaming,
volumeThreshold: config.volumeThreshold,
confirmedDropGracePeriodMs: config.confirmedDropGracePeriod,
silenceDurationMs: config.silenceDuration,
speechConfirmationMs: config.speechConfirmationDuration,
detectionGracePeriodMs: config.detectionGracePeriod,
});
}, [
enableContinuousStreaming,
config.volumeThreshold,
config.confirmedDropGracePeriod,
config.silenceDuration,
config.speechConfirmationDuration,
config.detectionGracePeriod,
]);
// Update segment duration timer
useEffect(() => {
if (!isDetecting && !isSpeaking) {
setSegmentDuration(0);
return;
}
const startTime = segmenterRef.current?.getSpeechDetectionStartMs() ?? Date.now();
const interval = setInterval(() => {
const elapsed = Date.now() - startTime;
setSegmentDuration(elapsed);
}, 100);
return () => clearInterval(interval);
}, [isDetecting, isSpeaking]);
// Listen to microphone data
useExpoTwoWayAudioEventListener(
"onMicrophoneData",
useCallback<MicrophoneDataCallback>(
(event) => {
if (!isActiveRef.current || isMutedRef.current) return;
const pcmData: Uint8Array = event.data;
segmenterRef.current?.pushPcmChunk(pcmData);
},
[]
)
);
// Listen to volume level for VAD
useExpoTwoWayAudioEventListener(
"onInputVolumeLevelData",
useCallback<VolumeLevelCallback>(
(event) => {
if (!isActiveRef.current) return;
const volumeLevel: number = event.data;
setVolume(volumeLevel);
if (isMutedRef.current) return;
segmenterRef.current?.pushVolumeLevel(volumeLevel, Date.now());
},
[]
)
);
const ensureMicrophonePermission = useCallback(async () => {
let permissionStatus = microphonePermission;
if (!permissionStatus?.granted) {
try {
permissionStatus = await requestMicrophonePermission();
} catch (err) {
throw new Error("Failed to request microphone permission");
}
}
if (!permissionStatus?.granted) {
throw new Error(
"Microphone permission is required to capture audio. Please enable microphone access in system settings."
);
}
}, [microphonePermission, requestMicrophonePermission]);
async function start(): Promise<void> {
if (isActive) {
console.log("[SpeechmaticsAudio] Already active");
return;
}
try {
await ensureMicrophonePermission();
// Initialize audio if not already initialized
if (!audioInitialized) {
console.log("[SpeechmaticsAudio] Initializing audio...");
await initialize();
setAudioInitialized(true);
console.log("[SpeechmaticsAudio] Audio initialized");
}
console.log("[SpeechmaticsAudio] Starting audio capture...");
// Start recording
toggleRecording(true);
setIsActive(true);
console.log("[SpeechmaticsAudio] Audio capture started successfully");
} catch (error) {
console.error("[SpeechmaticsAudio] Start error:", error);
const err = error instanceof Error ? error : new Error(String(error));
config.onError?.(err);
await stop();
throw error;
}
}
async function stop(): Promise<void> {
console.log("[SpeechmaticsAudio] Stopping audio capture...");
// Stop recording
if (isActive) {
toggleRecording(false);
}
segmenterRef.current?.stop(Date.now());
// Tear down audio session
if (audioInitialized) {
tearDown();
setAudioInitialized(false);
console.log("[SpeechmaticsAudio] Audio torn down");
}
// Reset state
segmenterRef.current?.reset();
setIsActive(false);
setIsSpeaking(false);
setIsDetecting(false);
setVolume(0);
setIsMuted(false);
console.log("[SpeechmaticsAudio] Audio capture stopped");
}
function toggleMute(): void {
setIsMuted((prev) => {
const newMuted = !prev;
console.log("[SpeechmaticsAudio] Mute toggled:", newMuted);
if (newMuted) {
// Clear any ongoing speech detection/speaking state
segmenterRef.current?.reset();
setIsSpeaking(false);
setIsDetecting(false);
}
return newMuted;
});
}
return {
start,
stop,
toggleMute,
isActive,
isSpeaking,
isDetecting,
isMuted,
volume,
segmentDuration,
};
}

View File

@@ -1,410 +0,0 @@
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { SpeechSegmenter } from "@/voice/speech-segmenter";
import { getTauri } from "@/utils/tauri";
export interface SpeechmaticsAudioConfig {
onAudioSegment?: (segment: { audioData: string; isLast: boolean }) => void;
onSpeechStart?: () => void;
onSpeechEnd?: () => void;
onError?: (error: Error) => void;
/** When true, stream microphone PCM continuously without VAD gating. */
enableContinuousStreaming?: boolean;
volumeThreshold: number; // 0-1
/** ms dip debounce before VAD transitions from confirmed speaking to non-speaking */
confirmedDropGracePeriod?: number;
silenceDuration: number; // ms of silence before ending segment
speechConfirmationDuration: number; // ms of sustained speech before confirming
detectionGracePeriod: number; // ms grace period for volume dips during detection
}
export interface SpeechmaticsAudio {
start: () => Promise<void>;
stop: () => Promise<void>;
toggleMute: () => void;
isActive: boolean;
isSpeaking: boolean;
isDetecting: boolean;
isMuted: boolean;
volume: number;
segmentDuration: number;
}
const getAudioContextCtor = (): (typeof AudioContext) | null => {
if (typeof window === "undefined") {
return null;
}
const ctor =
(window as typeof window & { webkitAudioContext?: typeof AudioContext }).AudioContext ||
(window as typeof window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext;
return ctor ?? null;
};
const floatToInt16 = (sample: number): number => {
const clamped = Math.max(-1, Math.min(1, sample));
return clamped < 0 ? Math.round(clamped * 0x8000) : Math.round(clamped * 0x7fff);
};
const resampleToPcm16 = (input: Float32Array, inputRate: number, outputRate: number): Int16Array => {
if (input.length === 0) {
return new Int16Array(0);
}
if (inputRate === outputRate) {
const out = new Int16Array(input.length);
for (let i = 0; i < input.length; i++) {
out[i] = floatToInt16(input[i]);
}
return out;
}
const ratio = inputRate / outputRate;
const outputLength = Math.max(1, Math.round(input.length / ratio));
const out = new Int16Array(outputLength);
for (let i = 0; i < outputLength; i++) {
const sourceIndex = i * ratio;
const i0 = Math.floor(sourceIndex);
const i1 = Math.min(input.length - 1, i0 + 1);
const frac = sourceIndex - i0;
const sample = input[i0] * (1 - frac) + input[i1] * frac;
out[i] = floatToInt16(sample);
}
return out;
};
export function useSpeechmaticsAudio(config: SpeechmaticsAudioConfig): SpeechmaticsAudio {
const [isActive, setIsActive] = useState(false);
const [isSpeaking, setIsSpeaking] = useState(false);
const [isDetecting, setIsDetecting] = useState(false);
const [volume, setVolume] = useState(0);
const [isMuted, setIsMuted] = useState(false);
const [segmentDuration, setSegmentDuration] = useState(0);
const enableContinuousStreaming = config.enableContinuousStreaming === true;
const callbacksRef = useRef({
onAudioSegment: config.onAudioSegment,
onSpeechStart: config.onSpeechStart,
onSpeechEnd: config.onSpeechEnd,
onError: config.onError,
});
useEffect(() => {
callbacksRef.current = {
onAudioSegment: config.onAudioSegment,
onSpeechStart: config.onSpeechStart,
onSpeechEnd: config.onSpeechEnd,
onError: config.onError,
};
}, [config.onAudioSegment, config.onSpeechStart, config.onSpeechEnd, config.onError]);
const segmenterRef = useRef<SpeechSegmenter | null>(null);
if (segmenterRef.current === null) {
segmenterRef.current = new SpeechSegmenter(
{
enableContinuousStreaming,
volumeThreshold: config.volumeThreshold,
confirmedDropGracePeriodMs: config.confirmedDropGracePeriod,
silenceDurationMs: config.silenceDuration,
speechConfirmationMs: config.speechConfirmationDuration,
detectionGracePeriodMs: config.detectionGracePeriod,
},
{
onAudioSegment: (segment) => callbacksRef.current.onAudioSegment?.(segment),
onSpeechStart: () => callbacksRef.current.onSpeechStart?.(),
onSpeechEnd: () => callbacksRef.current.onSpeechEnd?.(),
onDetectingChange: (next) => setIsDetecting(next),
onSpeakingChange: (next) => setIsSpeaking(next),
}
);
}
useEffect(() => {
segmenterRef.current?.updateConfig({
enableContinuousStreaming,
volumeThreshold: config.volumeThreshold,
confirmedDropGracePeriodMs: config.confirmedDropGracePeriod,
silenceDurationMs: config.silenceDuration,
speechConfirmationMs: config.speechConfirmationDuration,
detectionGracePeriodMs: config.detectionGracePeriod,
});
}, [
enableContinuousStreaming,
config.volumeThreshold,
config.confirmedDropGracePeriod,
config.silenceDuration,
config.speechConfirmationDuration,
config.detectionGracePeriod,
]);
const refs = useRef<{
started: boolean;
stream: MediaStream | null;
context: AudioContext | null;
source: MediaStreamAudioSourceNode | null;
processor: ScriptProcessorNode | null;
gain: GainNode | null;
}>({
started: false,
stream: null,
context: null,
source: null,
processor: null,
gain: null,
});
const isMutedRef = useRef(isMuted);
useEffect(() => {
isMutedRef.current = isMuted;
}, [isMuted]);
const vadConfigRef = useRef({
volumeThreshold: config.volumeThreshold,
});
useEffect(() => {
vadConfigRef.current = {
volumeThreshold: config.volumeThreshold,
};
}, [config.volumeThreshold]);
const vadLogRef = useRef({
lastLogMs: 0,
lastSpeaking: false,
lastDetecting: false,
});
// Update segment duration timer
useEffect(() => {
if (!isDetecting && !isSpeaking) {
setSegmentDuration(0);
return;
}
const startTime = segmenterRef.current?.getSpeechDetectionStartMs() ?? Date.now();
const interval = setInterval(() => {
setSegmentDuration(Date.now() - startTime);
}, 100);
return () => clearInterval(interval);
}, [isDetecting, isSpeaking]);
const stopInternal = useCallback(async () => {
refs.current.started = false;
try {
refs.current.processor?.disconnect();
refs.current.source?.disconnect();
refs.current.gain?.disconnect();
} catch {
// best-effort teardown
}
if (refs.current.stream) {
for (const track of refs.current.stream.getTracks()) {
try {
track.stop();
} catch {
// ignore
}
}
}
const context = refs.current.context;
if (context && context.state !== "closed") {
try {
await context.close();
} catch {
// ignore
}
}
refs.current.stream = null;
refs.current.context = null;
refs.current.source = null;
refs.current.processor = null;
refs.current.gain = null;
segmenterRef.current?.stop(Date.now());
setIsActive(false);
setIsSpeaking(false);
setIsDetecting(false);
setVolume(0);
setIsMuted(false);
}, []);
const start = useCallback(async () => {
if (refs.current.started) {
return;
}
const missingNavigator =
typeof navigator === "undefined" ||
!navigator.mediaDevices ||
typeof navigator.mediaDevices.getUserMedia !== "function";
const secureContext =
typeof window !== "undefined" && typeof window.isSecureContext === "boolean"
? window.isSecureContext
: true;
const currentOrigin = typeof window !== "undefined" && window.location ? window.location.origin : "unknown";
const isTauri = getTauri() !== null;
try {
if (missingNavigator) {
throw new Error("Microphone capture is not supported in this environment");
}
console.log("[Voice][Web] Microphone preflight", {
secureContext,
currentOrigin,
isTauri,
hasMediaDevices:
typeof navigator !== "undefined" &&
!!navigator.mediaDevices &&
typeof navigator.mediaDevices.getUserMedia === "function",
});
if (!secureContext && !isTauri) {
throw new Error(`Microphone access requires HTTPS or localhost. Current origin: ${currentOrigin}`);
}
if (!secureContext && isTauri) {
console.warn(
"[Voice][Web] Insecure context reported under Tauri; attempting getUserMedia anyway",
{ currentOrigin }
);
}
const AudioContextCtor = getAudioContextCtor();
if (!AudioContextCtor) {
throw new Error("AudioContext unavailable");
}
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
channelCount: 1,
noiseSuppression: true,
echoCancellation: true,
autoGainControl: true,
},
});
const context = new AudioContextCtor();
if (context.state === "suspended") {
await context.resume();
}
const source = context.createMediaStreamSource(stream);
const processor = context.createScriptProcessor(4096, 1, 1);
const gain = context.createGain();
gain.gain.value = 0;
refs.current = {
started: true,
stream,
context,
source,
processor,
gain,
};
processor.onaudioprocess = (event) => {
if (!refs.current.started) {
return;
}
const input = event.inputBuffer.getChannelData(0);
let sumSquares = 0;
for (let i = 0; i < input.length; i++) {
const sample = input[i];
sumSquares += sample * sample;
}
const rms = Math.sqrt(sumSquares / Math.max(1, input.length));
const normalized = Math.min(1, Math.max(0, rms * 2));
setVolume(normalized);
if (isMutedRef.current) {
return;
}
const nowMs = Date.now();
segmenterRef.current?.pushVolumeLevel(normalized, nowMs);
const segmenter = segmenterRef.current;
if (segmenter) {
const speakingNow = segmenter.getIsSpeaking();
const detectingNow = segmenter.getIsDetecting();
const shouldLog =
nowMs - vadLogRef.current.lastLogMs >= 150 ||
speakingNow !== vadLogRef.current.lastSpeaking ||
detectingNow !== vadLogRef.current.lastDetecting;
if (shouldLog) {
const threshold = vadConfigRef.current.volumeThreshold;
const releaseThreshold = segmenter.getVolumeReleaseThreshold();
console.log("[Voice][Web][VAD] level", {
volume: Number(normalized.toFixed(3)),
threshold: Number(threshold.toFixed(3)),
releaseThreshold: Number(releaseThreshold.toFixed(3)),
confirmedDropGraceMs: config.confirmedDropGracePeriod ?? 250,
isSpeaking: speakingNow,
isDetecting: detectingNow,
});
vadLogRef.current.lastLogMs = nowMs;
vadLogRef.current.lastSpeaking = speakingNow;
vadLogRef.current.lastDetecting = detectingNow;
}
}
const pcm16 = resampleToPcm16(input, context.sampleRate, 16000);
const pcmBytes = new Uint8Array(pcm16.buffer, pcm16.byteOffset, pcm16.byteLength);
segmenterRef.current?.pushPcmChunk(pcmBytes);
};
source.connect(processor);
processor.connect(gain);
gain.connect(context.destination);
setIsActive(true);
return;
} catch (error) {
const err = error instanceof Error ? error : new Error(String(error));
callbacksRef.current.onError?.(err);
await stopInternal();
throw err;
}
}, [stopInternal]);
const stop = useCallback(async () => {
await stopInternal();
}, [stopInternal]);
const toggleMute = useCallback(() => {
setIsMuted((prev) => {
const nextMuted = !prev;
if (nextMuted) {
segmenterRef.current?.reset();
setIsSpeaking(false);
setIsDetecting(false);
}
return nextMuted;
});
}, []);
useEffect(() => {
return () => {
if (refs.current.started) {
void stopInternal();
}
};
}, [stopInternal]);
return useMemo(
() => ({
start,
stop,
toggleMute,
isActive,
isSpeaking,
isDetecting,
isMuted,
volume,
segmentDuration,
}),
[start, stop, toggleMute, isActive, isSpeaking, isDetecting, isMuted, volume, segmentDuration]
);
}

View File

@@ -922,8 +922,7 @@ describe("HostRuntimeStore", () => {
useSessionStore.getState().initializeSession(
host.serverId,
fakeClient as unknown as DaemonClient,
null as any
fakeClient as unknown as DaemonClient
);
store.syncHosts([host]);
@@ -1046,8 +1045,7 @@ describe("HostRuntimeStore", () => {
useSessionStore.getState().initializeSession(
host.serverId,
fakeClient as unknown as DaemonClient,
null as any
fakeClient as unknown as DaemonClient
);
store.syncHosts([host]);
@@ -1122,8 +1120,7 @@ describe("HostRuntimeStore", () => {
useSessionStore.getState().initializeSession(
host.serverId,
fakeClient as unknown as DaemonClient,
null as any
fakeClient as unknown as DaemonClient
);
store.syncHosts([host]);

View File

@@ -103,29 +103,6 @@ function toErrorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
function toReasonCode(reason: string | null): string | null {
if (!reason) {
return null;
}
const normalized = reason.toLowerCase();
if (normalized.includes("timed out")) {
return "connect_timeout";
}
if (normalized.includes("disposed")) {
return "disposed";
}
if (normalized.includes("client closed") || normalized.includes("client_closed")) {
return "client_closed";
}
if (normalized.includes("transport")) {
return "transport_error";
}
if (normalized.includes("failed to connect")) {
return "connect_failed";
}
return "unknown";
}
function hashForLog(value: string): string {
let hash = 0;
for (let index = 0; index < value.length; index += 1) {
@@ -915,35 +892,12 @@ export class HostRuntimeController {
});
}
private logConnectionTransition(input: {
private logConnectionTransition(_input: {
from: HostRuntimeConnectionMachineState["tag"];
to: HostRuntimeConnectionMachineState["tag"];
event: HostRuntimeConnectionMachineEvent;
}): void {
const { event } = input;
const reason =
event.type === "connect_failed"
? event.message
: event.type === "client_state"
? event.state.status === "disconnected"
? event.state.reason ?? event.lastError ?? null
: null
: null;
const reasonCode =
event.type === "connect_failed"
? "connect_failed"
: toReasonCode(reason);
console.info("[HostRuntimeTransition]", {
serverId: this.host.serverId,
clientIdHash: this.clientIdHash,
from: input.from,
to: input.to,
event: event.type,
connectionPath: this.snapshot.activeConnection?.type ?? null,
generation: this.snapshot.clientGeneration,
reasonCode,
reason,
});
// Intentionally empty - logging removed.
}
private trackConnectionFirstSeen(): void {
@@ -1822,6 +1776,15 @@ export function useHostRuntimeSession(serverId: string): {
};
}
export function useHostRuntimeIsConnected(serverId: string): boolean {
const store = getHostRuntimeStore();
return useSyncExternalStore(
(onStoreChange) => store.subscribe(serverId, onStoreChange),
() => isHostRuntimeConnected(store.getSnapshot(serverId)),
() => isHostRuntimeConnected(store.getSnapshot(serverId))
);
}
export function useHosts(): HostProfile[] {
const store = getHostRuntimeStore();
return useSyncExternalStore(

View File

@@ -60,20 +60,11 @@ import {
} from "./agent-ready-screen-bottom-anchor";
const EMPTY_STREAM_ITEMS: StreamItem[] = [];
const IS_DEV = Boolean((globalThis as { __DEV__?: boolean }).__DEV__);
function logAgentExplorer(event: string, details: Record<string, unknown>): void {
if (!IS_DEV) {
return;
}
console.log(`[AgentExplorer] ${event}`, details);
}
function logWebStickyBottom(event: string, details: Record<string, unknown>): void {
if (!IS_DEV || Platform.OS !== "web") {
return;
}
console.log("[WebStickyBottom]", event, details);
function logWebStickyBottom(
_event: string,
_details: Record<string, unknown>
): void {
// Intentionally disabled: this path is too noisy during voice debugging.
}
export function AgentReadyScreen({
@@ -81,11 +72,13 @@ export function AgentReadyScreen({
agentId,
showExplorerSidebar = true,
wrapWithExplorerSidebarProvider = true,
onOpenWorkspaceFile,
}: {
serverId: string;
agentId: string;
showExplorerSidebar?: boolean;
wrapWithExplorerSidebarProvider?: boolean;
onOpenWorkspaceFile?: (input: { filePath: string }) => void;
}) {
const resolvedAgentId = agentId?.trim() || undefined;
const resolvedServerId = serverId?.trim() || undefined;
@@ -129,6 +122,7 @@ export function AgentReadyScreen({
isConnected={runtimeIsConnected}
connectionStatus={connectionStatus}
showExplorerSidebar={showExplorerSidebar}
onOpenWorkspaceFile={onOpenWorkspaceFile}
/>
);
@@ -146,6 +140,7 @@ type AgentScreenContentProps = {
isConnected: boolean;
connectionStatus: HostRuntimeConnectionStatus;
showExplorerSidebar: boolean;
onOpenWorkspaceFile?: (input: { filePath: string }) => void;
};
function toErrorMessage(error: unknown): string {
@@ -166,6 +161,7 @@ function AgentScreenContent({
isConnected,
connectionStatus,
showExplorerSidebar,
onOpenWorkspaceFile,
}: AgentScreenContentProps) {
const { theme } = useUnistyles();
const toast = useToast();
@@ -259,10 +255,6 @@ function AgentScreenContent({
}, [resolveCachedCheckoutIsGit, resolvedAgentId, checkout?.isGit, serverId]);
const openExplorerForActiveCheckout = useCallback(() => {
const checkoutContext = resolveCurrentExplorerCheckout();
logAgentExplorer("openExplorerForActiveCheckout", {
hasCheckoutContext: Boolean(checkoutContext),
checkoutContext,
});
if (checkoutContext) {
activateExplorerTabForCheckout(checkoutContext);
}
@@ -904,6 +896,7 @@ function AgentScreenContent({
pendingPermissions={pendingPermissions}
routeBottomAnchorRequest={routeBottomAnchorRequest}
isAuthoritativeHistoryReady={hasAppliedAuthoritativeHistory}
onOpenWorkspaceFile={onOpenWorkspaceFile}
/>
</ReanimatedAnimated.View>
</View>

View File

@@ -156,8 +156,6 @@ export function WorkspaceDesktopTabsRow({
const layoutItem = layout.items[index] ?? null;
const resolvedTabWidth = layoutItem?.width ?? 150;
const showLabel = layoutItem?.showLabel ?? true;
const labelCharCap = layoutItem?.labelCharCap ?? tab.label.length;
const renderedLabel = showLabel ? tab.label.slice(0, Math.max(1, labelCharCap)) : "";
const presentation = deriveWorkspaceTabPresentation({ tab, agent: tabAgent });
const tooltipLabel =
tab.kind === "agent" && tab.titleState === "loading"
@@ -172,11 +170,7 @@ export function WorkspaceDesktopTabsRow({
return (
<ContextMenu key={tab.key}>
<Tooltip delayDuration={400} enabledOnDesktop enabledOnMobile={false}>
<TooltipTrigger
testID={`workspace-tab-tooltip-${tab.key}`}
accessibilityRole="none"
style={styles.tabTooltipTrigger}
>
<TooltipTrigger asChild triggerRefProp="triggerRef">
<ContextMenuTrigger
testID={`workspace-tab-${tab.key}`}
enabledOnMobile={false}
@@ -228,9 +222,11 @@ export function WorkspaceDesktopTabsRow({
isActive && styles.tabLabelActive,
shouldShowCloseButton && styles.tabLabelWithCloseButton,
]}
selectable={false}
numberOfLines={1}
ellipsizeMode="tail"
>
{renderedLabel}
{tab.label}
</Text>
)
) : null}
@@ -411,9 +407,6 @@ const styles = StyleSheet.create((theme) => ({
paddingRight: theme.spacing[2],
paddingVertical: theme.spacing[1],
},
tabTooltipTrigger: {
flexShrink: 0,
},
tab: {
paddingHorizontal: theme.spacing[3],
paddingVertical: theme.spacing[2],
@@ -421,6 +414,7 @@ const styles = StyleSheet.create((theme) => ({
flexDirection: "row",
alignItems: "center",
gap: theme.spacing[1],
userSelect: "none",
},
tabHandle: {
flexDirection: "row",
@@ -428,6 +422,7 @@ const styles = StyleSheet.create((theme) => ({
gap: theme.spacing[1],
flex: 1,
minWidth: 0,
userSelect: "none",
},
tabIcon: {
flexShrink: 0,
@@ -444,6 +439,7 @@ const styles = StyleSheet.create((theme) => ({
color: theme.colors.foregroundMuted,
fontSize: theme.fontSize.sm,
fontWeight: theme.fontWeight.normal,
userSelect: "none",
},
tabLabelSkeleton: {
width: 96,

View File

@@ -29,6 +29,7 @@ type WorkspaceDraftAgentTabProps = {
tabId: string
draftId: string
onCreated: (snapshot: AgentSnapshotPayload) => void
onOpenWorkspaceFile: (input: { filePath: string }) => void
}
export function WorkspaceDraftAgentTab({
@@ -37,6 +38,7 @@ export function WorkspaceDraftAgentTab({
tabId,
draftId,
onCreated,
onOpenWorkspaceFile,
}: WorkspaceDraftAgentTabProps) {
const { client, isConnected } = useHostRuntimeSession(serverId)
const addImagesRef = useRef<((images: ImageAttachment[]) => void) | null>(null)
@@ -204,6 +206,7 @@ export function WorkspaceDraftAgentTab({
agent={draftAgent}
streamItems={optimisticStreamItems}
pendingPermissions={EMPTY_PENDING_PERMISSIONS}
onOpenWorkspaceFile={onOpenWorkspaceFile}
/>
</View>
) : (

View File

@@ -1,4 +1,4 @@
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { memo, useCallback, useEffect, useMemo, useRef, useState } from "react";
import {
ActivityIndicator,
BackHandler,
@@ -25,7 +25,7 @@ import { StyleSheet, UnistylesRuntime, useUnistyles } from "react-native-unistyl
import { SidebarMenuToggle } from "@/components/headers/menu-header";
import { HeaderToggleButton } from "@/components/headers/header-toggle-button";
import { ScreenHeader } from "@/components/headers/screen-header";
import { Combobox } from "@/components/ui/combobox";
import { Combobox, type ComboboxOption } from "@/components/ui/combobox";
import { Shortcut } from "@/components/ui/shortcut";
import {
DropdownMenu,
@@ -80,6 +80,7 @@ import { WorkspaceDraftAgentTab } from "@/screens/workspace/workspace-draft-agen
import { WorkspaceDesktopTabsRow } from "@/screens/workspace/workspace-desktop-tabs-row";
import {
deriveWorkspaceTabPresentation,
type WorkspaceTabPresentation,
WorkspaceTabIcon,
WorkspaceTabOptionRow,
} from "@/screens/workspace/workspace-tab-presentation";
@@ -141,6 +142,120 @@ function buildOpenIntentKey(input: {
return `${input.serverId}:${input.workspaceId}:${openParam}`;
}
type MobileWorkspaceTabSwitcherProps = {
activeTabKey: string;
activeTabLabel: string;
activeTabPresentation: WorkspaceTabPresentation | null;
tabSwitcherOptions: ComboboxOption[];
tabByKey: Map<string, WorkspaceTabDescriptor>;
tabPresentationsByKey: Map<string, WorkspaceTabPresentation>;
onSelectSwitcherTab: (key: string) => void;
onSelectNewTabOption: (key: typeof NEW_TAB_AGENT_OPTION_ID) => void;
};
const MobileWorkspaceTabSwitcher = memo(function MobileWorkspaceTabSwitcher({
activeTabKey,
activeTabLabel,
activeTabPresentation,
tabSwitcherOptions,
tabByKey,
tabPresentationsByKey,
onSelectSwitcherTab,
onSelectNewTabOption,
}: MobileWorkspaceTabSwitcherProps) {
const { theme } = useUnistyles();
const [isOpen, setIsOpen] = useState(false);
const anchorRef = useRef<View>(null);
return (
<View style={styles.mobileTabsRow} testID="workspace-tabs-row">
<Pressable
ref={anchorRef}
style={({ hovered, pressed }) => [
styles.switcherTrigger,
(hovered || pressed || isOpen) && styles.switcherTriggerActive,
{ borderWidth: 0, borderColor: "transparent" },
Platform.OS === "web"
? {
outlineStyle: "solid",
outlineWidth: 0,
outlineColor: "transparent",
}
: null,
]}
onPress={() => setIsOpen(true)}
>
<View style={styles.switcherTriggerLeft}>
<View style={styles.switcherTriggerIcon} testID="workspace-active-tab-icon">
{activeTabPresentation ? (
<WorkspaceTabIcon presentation={activeTabPresentation} active />
) : null}
</View>
<Text style={styles.switcherTriggerText} numberOfLines={1}>
{activeTabLabel}
</Text>
</View>
<ChevronDown size={theme.iconSize.sm} color={theme.colors.foregroundMuted} />
</Pressable>
<View style={styles.mobileTabsActions}>
<Tooltip delayDuration={0} enabledOnDesktop enabledOnMobile={false}>
<TooltipTrigger
testID="workspace-new-agent-tab"
onPress={() => onSelectNewTabOption(NEW_TAB_AGENT_OPTION_ID)}
accessibilityRole="button"
accessibilityLabel="New agent tab"
style={({ hovered, pressed }) => [
styles.newTabActionButton,
(hovered || pressed) && styles.newTabActionButtonHovered,
]}
>
<Plus size={theme.iconSize.sm} color={theme.colors.foregroundMuted} />
</TooltipTrigger>
<TooltipContent side="bottom" align="end" offset={8}>
<View style={styles.newTabTooltipRow}>
<Text style={styles.newTabTooltipText}>New agent tab</Text>
<Shortcut keys={["mod", "T"]} style={styles.newTabTooltipShortcut} />
</View>
</TooltipContent>
</Tooltip>
</View>
<Combobox
options={tabSwitcherOptions}
value={activeTabKey}
onSelect={onSelectSwitcherTab}
searchable={false}
title="Switch tab"
searchPlaceholder="Search tabs"
open={isOpen}
onOpenChange={setIsOpen}
enableDismissOnClose={false}
anchorRef={anchorRef}
renderOption={({ option, selected, active, onPress }) => {
const tab = tabByKey.get(option.id);
if (!tab) {
return <View />;
}
const presentation =
tabPresentationsByKey.get(option.id) ??
deriveWorkspaceTabPresentation({ tab });
return (
<WorkspaceTabOptionRow
presentation={presentation}
selected={selected}
active={active}
onPress={onPress}
/>
);
}}
/>
</View>
);
});
export function WorkspaceScreen({
serverId,
workspaceId,
@@ -344,6 +459,10 @@ function WorkspaceScreenContent({
const isWorkspaceHeaderLoading = workspaceHeader === null;
const isGitCheckout = checkoutQuery.data?.isGit ?? false;
const currentBranchName =
checkoutQuery.data?.isGit && checkoutQuery.data.currentBranch !== "HEAD"
? trimNonEmpty(checkoutQuery.data.currentBranch)
: null;
const mobileView = usePanelStore((state) => state.mobileView);
const desktopFileExplorerOpen = usePanelStore(
(state) => state.desktop.fileExplorerOpen
@@ -752,12 +871,21 @@ function WorkspaceScreenContent({
[closeToAgent, isMobile, navigateToTabId, normalizedServerId, normalizedWorkspaceId, openOrFocusTab]
);
const [isTabSwitcherOpen, setIsTabSwitcherOpen] = useState(false);
const handleOpenFileFromChat = useCallback(
({ filePath }: { filePath: string }) => {
const normalizedFilePath = filePath.trim();
if (!normalizedFilePath) {
return;
}
handleOpenFileFromExplorer(normalizedFilePath);
},
[handleOpenFileFromExplorer]
);
const [hoveredTabKey, setHoveredTabKey] = useState<string | null>(null);
const [hoveredCloseTabKey, setHoveredCloseTabKey] = useState<string | null>(
null
);
const tabSwitcherAnchorRef = useRef<View>(null);
const tabByKey = useMemo(() => {
const map = new Map<string, WorkspaceTabDescriptor>();
@@ -822,7 +950,6 @@ function WorkspaceScreenContent({
const handleSelectSwitcherTab = useCallback(
(key: string) => {
setIsTabSwitcherOpen(false);
navigateToTabId(key);
},
[navigateToTabId]
@@ -1025,6 +1152,20 @@ function WorkspaceScreenContent({
}
}, [normalizedWorkspaceId, toast]);
const handleCopyBranchName = useCallback(async () => {
if (!currentBranchName) {
toast.error("Branch name not available");
return;
}
try {
await Clipboard.setStringAsync(currentBranchName);
toast.copied("Branch name");
} catch {
toast.error("Copy failed");
}
}, [currentBranchName, toast]);
const handleBulkCloseTabs = useCallback(
async (input: { tabsToClose: WorkspaceTabDescriptor[]; title: string; logLabel: string }) => {
const { tabsToClose, title, logLabel } = input;
@@ -1240,6 +1381,7 @@ function WorkspaceScreenContent({
workspaceId={normalizedWorkspaceId}
tabId={activeTabId ?? target.draftId}
draftId={target.draftId}
onOpenWorkspaceFile={handleOpenFileFromChat}
onCreated={(agentSnapshot) => {
const tabId = activeTabId ?? target.draftId;
const normalized = normalizeAgentSnapshot(agentSnapshot, normalizedServerId);
@@ -1274,6 +1416,7 @@ function WorkspaceScreenContent({
agentId={target.agentId}
showExplorerSidebar={false}
wrapWithExplorerSidebarProvider={false}
onOpenWorkspaceFile={handleOpenFileFromChat}
/>
);
}
@@ -1389,6 +1532,20 @@ function WorkspaceScreenContent({
>
Copy workspace path
</DropdownMenuItem>
{currentBranchName ? (
<DropdownMenuItem
testID="workspace-header-copy-branch-name"
leading={
<Copy
size={16}
color={theme.colors.foregroundMuted}
/>
}
onSelect={handleCopyBranchName}
>
Copy branch name
</DropdownMenuItem>
) : null}
</DropdownMenuContent>
</DropdownMenu>
</View>
@@ -1477,91 +1634,16 @@ function WorkspaceScreenContent({
/>
{isMobile ? (
<View style={styles.mobileTabsRow} testID="workspace-tabs-row">
<Pressable
ref={tabSwitcherAnchorRef}
style={({ hovered, pressed }) => [
styles.switcherTrigger,
(hovered || pressed || isTabSwitcherOpen) && styles.switcherTriggerActive,
{ borderWidth: 0, borderColor: "transparent" },
Platform.OS === "web"
? {
outlineStyle: "solid",
outlineWidth: 0,
outlineColor: "transparent",
}
: null,
]}
onPress={() => setIsTabSwitcherOpen(true)}
>
<View style={styles.switcherTriggerLeft}>
<View style={styles.switcherTriggerIcon} testID="workspace-active-tab-icon">
{activeTabPresentation ? (
<WorkspaceTabIcon presentation={activeTabPresentation} active />
) : null}
</View>
<Text style={styles.switcherTriggerText} numberOfLines={1}>
{activeTabLabel}
</Text>
</View>
<ChevronDown size={theme.iconSize.sm} color={theme.colors.foregroundMuted} />
</Pressable>
<View style={styles.mobileTabsActions}>
<Tooltip delayDuration={0} enabledOnDesktop enabledOnMobile={false}>
<TooltipTrigger
testID="workspace-new-agent-tab"
onPress={() => handleSelectNewTabOption(NEW_TAB_AGENT_OPTION_ID)}
accessibilityRole="button"
accessibilityLabel="New agent tab"
style={({ hovered, pressed }) => [
styles.newTabActionButton,
(hovered || pressed) && styles.newTabActionButtonHovered,
]}
>
<Plus size={theme.iconSize.sm} color={theme.colors.foregroundMuted} />
</TooltipTrigger>
<TooltipContent side="bottom" align="end" offset={8}>
<View style={styles.newTabTooltipRow}>
<Text style={styles.newTabTooltipText}>New agent tab</Text>
<Shortcut keys={["mod", "T"]} style={styles.newTabTooltipShortcut} />
</View>
</TooltipContent>
</Tooltip>
</View>
<Combobox
options={tabSwitcherOptions}
value={activeTabKey}
onSelect={handleSelectSwitcherTab}
searchable={false}
title="Switch tab"
searchPlaceholder="Search tabs"
open={isTabSwitcherOpen}
onOpenChange={setIsTabSwitcherOpen}
anchorRef={tabSwitcherAnchorRef}
renderOption={({ option, selected, active, onPress }) => {
const tab = tabByKey.get(option.id);
if (!tab) {
return <View />;
}
const presentation =
tabPresentationsByKey.get(option.id) ??
deriveWorkspaceTabPresentation({ tab });
return (
<WorkspaceTabOptionRow
presentation={presentation}
selected={selected}
active={active}
onPress={onPress}
/>
);
}}
/>
</View>
<MobileWorkspaceTabSwitcher
activeTabKey={activeTabKey}
activeTabLabel={activeTabLabel}
activeTabPresentation={activeTabPresentation}
tabSwitcherOptions={tabSwitcherOptions}
tabByKey={tabByKey}
tabPresentationsByKey={tabPresentationsByKey}
onSelectSwitcherTab={handleSelectSwitcherTab}
onSelectNewTabOption={handleSelectNewTabOption}
/>
) : (
<WorkspaceDesktopTabsRow
tabs={tabs}

View File

@@ -51,23 +51,6 @@ export const DEFAULT_EXPLORER_FILES_SPLIT_RATIO = 0.38;
export const MIN_EXPLORER_FILES_SPLIT_RATIO = 0.2;
export const MAX_EXPLORER_FILES_SPLIT_RATIO = 0.8;
const IS_DEV = Boolean((globalThis as { __DEV__?: boolean }).__DEV__);
function logPanelTransition(
action: string,
details: Record<string, unknown>
): void {
if (!IS_DEV) {
return;
}
const stack =
new Error().stack
?.split("\n")
.slice(2, 8)
.join("\n") ?? "stack unavailable";
console.log(`[PanelStore] ${action}`, details, stack);
}
interface PanelState {
// Mobile: which panel is currently shown
mobileView: MobilePanelView;
@@ -151,103 +134,52 @@ export const usePanelStore = create<PanelState>()(
explorerFilesSplitRatio: DEFAULT_EXPLORER_FILES_SPLIT_RATIO,
openAgentList: () =>
set((state) => {
const nextState = {
mobileView: "agent-list" as const,
desktop: { ...state.desktop, agentListOpen: true },
};
logPanelTransition("openAgentList", {
fromMobileView: state.mobileView,
toMobileView: nextState.mobileView,
fromDesktopAgentListOpen: state.desktop.agentListOpen,
toDesktopAgentListOpen: nextState.desktop.agentListOpen,
});
return nextState;
}),
set((state) => ({
mobileView: "agent-list" as const,
desktop: { ...state.desktop, agentListOpen: true },
})),
openFileExplorer: () =>
set((state) => {
const resolvedTab = resolveExplorerTabFromActiveCheckout(state);
const nextMobileView: MobilePanelView = "file-explorer";
const nextDesktop = { ...state.desktop, fileExplorerOpen: true };
const nextState = {
mobileView: nextMobileView,
desktop: nextDesktop,
return {
mobileView: "file-explorer" as MobilePanelView,
desktop: { ...state.desktop, fileExplorerOpen: true },
...(resolvedTab ? { explorerTab: resolvedTab } : {}),
};
logPanelTransition("openFileExplorer", {
fromMobileView: state.mobileView,
toMobileView: nextMobileView,
fromDesktopFileExplorerOpen: state.desktop.fileExplorerOpen,
toDesktopFileExplorerOpen: nextDesktop.fileExplorerOpen,
resolvedTab: resolvedTab ?? null,
activeCheckout: state.activeExplorerCheckout,
});
return nextState;
}),
closeFileExplorer: () =>
set((state) => {
const nextState = {
mobileView:
state.mobileView === "file-explorer" ? ("agent" as const) : state.mobileView,
desktop: {
...state.desktop,
fileExplorerOpen: false,
},
};
logPanelTransition("closeFileExplorer", {
fromMobileView: state.mobileView,
toMobileView: nextState.mobileView,
fromDesktopFileExplorerOpen: state.desktop.fileExplorerOpen,
toDesktopFileExplorerOpen: nextState.desktop.fileExplorerOpen,
});
return nextState;
}),
set((state) => ({
mobileView:
state.mobileView === "file-explorer" ? ("agent" as const) : state.mobileView,
desktop: {
...state.desktop,
fileExplorerOpen: false,
},
})),
closeToAgent: () =>
set((state) => {
const nextState = {
mobileView: "agent" as const,
// On desktop, closing depends on which panel triggered it
// This is called when closing via gesture/backdrop, so we close the currently active mobile panel
desktop: {
agentListOpen:
state.mobileView === "agent-list" ? false : state.desktop.agentListOpen,
fileExplorerOpen:
state.mobileView === "file-explorer" ? false : state.desktop.fileExplorerOpen,
},
};
logPanelTransition("closeToAgent", {
fromMobileView: state.mobileView,
toMobileView: nextState.mobileView,
fromDesktopAgentListOpen: state.desktop.agentListOpen,
toDesktopAgentListOpen: nextState.desktop.agentListOpen,
fromDesktopFileExplorerOpen: state.desktop.fileExplorerOpen,
toDesktopFileExplorerOpen: nextState.desktop.fileExplorerOpen,
});
return nextState;
}),
set((state) => ({
mobileView: "agent" as const,
// On desktop, closing depends on which panel triggered it
// This is called when closing via gesture/backdrop, so we close the currently active mobile panel
desktop: {
agentListOpen:
state.mobileView === "agent-list" ? false : state.desktop.agentListOpen,
fileExplorerOpen:
state.mobileView === "file-explorer" ? false : state.desktop.fileExplorerOpen,
},
})),
toggleAgentList: () =>
set((state) => {
set((state) => ({
// Mobile: toggle between agent and agent-list
const newMobileView: MobilePanelView =
state.mobileView === "agent-list" ? "agent" : "agent-list";
const nextState = {
mobileView: newMobileView,
desktop: {
...state.desktop,
agentListOpen: !state.desktop.agentListOpen,
},
};
logPanelTransition("toggleAgentList", {
fromMobileView: state.mobileView,
toMobileView: nextState.mobileView,
fromDesktopAgentListOpen: state.desktop.agentListOpen,
toDesktopAgentListOpen: nextState.desktop.agentListOpen,
});
return nextState;
}),
mobileView: (state.mobileView === "agent-list" ? "agent" : "agent-list") as MobilePanelView,
desktop: {
...state.desktop,
agentListOpen: !state.desktop.agentListOpen,
},
})),
toggleFileExplorer: () =>
set((state) => {
@@ -266,23 +198,12 @@ export const usePanelStore = create<PanelState>()(
mobileView: nextMobileView,
desktop: nextDesktop,
};
let resolvedTab: ExplorerTab | null = null;
if (willOpenMobile || willOpenDesktop) {
resolvedTab = resolveExplorerTabFromActiveCheckout(state);
const resolvedTab = resolveExplorerTabFromActiveCheckout(state);
if (resolvedTab) {
nextState.explorerTab = resolvedTab;
}
}
logPanelTransition("toggleFileExplorer", {
fromMobileView: state.mobileView,
toMobileView: nextMobileView,
fromDesktopFileExplorerOpen: state.desktop.fileExplorerOpen,
toDesktopFileExplorerOpen: nextDesktop.fileExplorerOpen,
willOpenMobile,
willOpenDesktop,
resolvedTab: resolvedTab ?? null,
activeCheckout: state.activeExplorerCheckout,
});
return nextState;
}),

View File

@@ -1,7 +1,6 @@
import { create } from "zustand";
import { subscribeWithSelector } from "zustand/middleware";
import type { DaemonClient } from "@server/client/daemon-client";
import type { useAudioPlayer } from "@/hooks/use-audio-player";
import type { AgentDirectoryEntry } from "@/types/agent-directory";
import type { StreamItem } from "@/types/stream";
import type { PendingPermission } from "@/types/shared";
@@ -208,9 +207,6 @@ export interface SessionState {
// Server metadata (from server_info handshake)
serverInfo: DaemonServerInfo | null;
// Audio player (immutable reference)
audioPlayer: ReturnType<typeof useAudioPlayer> | null;
// Hydration status
hasHydratedAgents: boolean;
hasHydratedWorkspaces: boolean;
@@ -264,7 +260,7 @@ interface SessionStoreState {
// Action types
interface SessionStoreActions {
// Session management
initializeSession: (serverId: string, client: DaemonClient, audioPlayer: ReturnType<typeof useAudioPlayer>) => void;
initializeSession: (serverId: string, client: DaemonClient) => void;
clearSession: (serverId: string) => void;
getSession: (serverId: string) => SessionState | undefined;
updateSessionClient: (serverId: string, client: DaemonClient) => void;
@@ -359,12 +355,14 @@ type SessionStore = SessionStoreState & SessionStoreActions;
const agentLastActivityCoalescer = createAgentLastActivityCoalescer();
// Helper to create initial session state
function createInitialSessionState(serverId: string, client: DaemonClient, audioPlayer: ReturnType<typeof useAudioPlayer>): SessionState {
function createInitialSessionState(
serverId: string,
client: DaemonClient
): SessionState {
return {
serverId,
client,
serverInfo: null,
audioPlayer,
hasHydratedAgents: false,
hasHydratedWorkspaces: false,
isPlayingAudio: false,
@@ -424,7 +422,7 @@ export const useSessionStore = create<SessionStore>()(
agentLastActivity: new Map(),
// Session management
initializeSession: (serverId, client, audioPlayer) => {
initializeSession: (serverId, client) => {
set((prev) => {
if (prev.sessions[serverId]) {
return prev;
@@ -433,7 +431,7 @@ export const useSessionStore = create<SessionStore>()(
...prev,
sessions: {
...prev.sessions,
[serverId]: createInitialSessionState(serverId, client, audioPlayer),
[serverId]: createInitialSessionState(serverId, client),
},
};
});

View File

@@ -136,4 +136,38 @@ describe("workspace-tabs-store retargetTab", () => {
expect(matchingTabs).toHaveLength(1);
expect(order).toEqual([draftTabId]);
});
it("openOrFocusTab re-focuses an existing file tab after the workspace focus changed", () => {
const key = buildWorkspaceTabPersistenceKey({ serverId: SERVER_ID, workspaceId: WORKSPACE_ID });
expect(key).toBeTruthy();
const workspaceKey = key as string;
const fileTabId = useWorkspaceTabsStore.getState().openOrFocusTab({
serverId: SERVER_ID,
workspaceId: WORKSPACE_ID,
target: { kind: "file", path: "/repo/worktree/src/index.ts" },
});
const terminalTabId = useWorkspaceTabsStore.getState().openOrFocusTab({
serverId: SERVER_ID,
workspaceId: WORKSPACE_ID,
target: { kind: "terminal", terminalId: "term-1" },
});
expect(fileTabId).toBe("file_/repo/worktree/src/index.ts");
expect(terminalTabId).toBe("terminal_term-1");
expect(useWorkspaceTabsStore.getState().focusedTabIdByWorkspace[workspaceKey]).toBe(
terminalTabId
);
const reopenedFileTabId = useWorkspaceTabsStore.getState().openOrFocusTab({
serverId: SERVER_ID,
workspaceId: WORKSPACE_ID,
target: { kind: "file", path: "/repo/worktree/src/index.ts" },
});
expect(reopenedFileTabId).toBe(fileTabId);
expect(useWorkspaceTabsStore.getState().focusedTabIdByWorkspace[workspaceKey]).toBe(
fileTabId
);
});
});

View File

@@ -0,0 +1,40 @@
import { describe, expect, it } from "vitest";
import { createMarkdownStyles } from "./markdown-styles";
import { darkTheme } from "./theme";
describe("createMarkdownStyles", () => {
it("applies shrink-and-wrap constraints to long markdown text and links", () => {
const styles = createMarkdownStyles(darkTheme);
expect(styles.body).toMatchObject({
flexShrink: 1,
minWidth: 0,
width: "100%",
});
expect(styles.paragraph).toMatchObject({
flexShrink: 1,
minWidth: 0,
width: "100%",
flexWrap: "wrap",
});
expect(styles.text).toMatchObject({
flexShrink: 1,
minWidth: 0,
overflowWrap: "anywhere",
});
expect(styles.link).toMatchObject({
flexShrink: 1,
minWidth: 0,
overflowWrap: "anywhere",
});
expect(styles.blocklink).toMatchObject({
flexShrink: 1,
minWidth: 0,
overflowWrap: "anywhere",
});
});
});

View File

@@ -18,10 +18,16 @@ export function createMarkdownStyles(theme: Theme) {
color: theme.colors.foreground,
fontSize: theme.fontSize.base,
lineHeight: 22,
flexShrink: 1,
minWidth: 0,
width: "100%" as const,
},
text: {
color: theme.colors.foreground,
flexShrink: 1,
minWidth: 0,
overflowWrap: "anywhere" as const,
},
paragraph: {
@@ -31,6 +37,9 @@ export function createMarkdownStyles(theme: Theme) {
flexDirection: "row" as const,
alignItems: "flex-start" as const,
justifyContent: "flex-start" as const,
flexShrink: 1,
minWidth: 0,
width: "100%" as const,
},
// =========================================================================
@@ -117,13 +126,19 @@ export function createMarkdownStyles(theme: Theme) {
},
link: {
color: theme.colors.primary,
textDecorationLine: "underline" as const,
color: theme.colors.accentBright,
textDecorationLine: "none" as const,
flexShrink: 1,
minWidth: 0,
overflowWrap: "anywhere" as const,
},
blocklink: {
color: theme.colors.primary,
textDecorationLine: "underline" as const,
color: theme.colors.accentBright,
textDecorationLine: "none" as const,
flexShrink: 1,
minWidth: 0,
overflowWrap: "anywhere" as const,
},
// =========================================================================
@@ -135,7 +150,8 @@ export function createMarkdownStyles(theme: Theme) {
color: theme.colors.foreground,
paddingHorizontal: theme.spacing[1],
paddingVertical: 2,
borderRadius: theme.borderRadius.sm,
borderRadius: theme.borderRadius.md,
borderWidth: 0,
fontFamily: Fonts.mono,
fontSize: theme.fontSize.sm,
},

View File

@@ -123,6 +123,7 @@ const lightSemanticColors = {
// Brand
accent: "#20744A",
accentBright: "#239956",
accentForeground: "#ffffff",
// Semantic
@@ -192,6 +193,7 @@ const darkSemanticColors = {
// Brand
accent: "#20744A",
accentBright: "#7ccba0",
accentForeground: "#ffffff",
// Semantic

View File

@@ -2,9 +2,6 @@ import { StyleSheet } from "react-native-unistyles";
// import { UnistylesRuntime } from "react-native-unistyles";
import { lightTheme, darkTheme } from "./theme";
console.log("[Unistyles] Configuring...");
// // Configure Unistyles with adaptive themes
StyleSheet.configure({
themes: {
light: lightTheme,
@@ -22,8 +19,6 @@ StyleSheet.configure({
},
});
console.log("[Unistyles] Configuration complete!");
// Type augmentation for TypeScript
type AppThemes = {
light: typeof lightTheme;
@@ -43,6 +38,4 @@ declare module "react-native-unistyles" {
export interface UnistylesBreakpoints extends AppBreakpoints {}
}
console.log(lightTheme.colors.background);
// UnistylesRuntime.setRootViewBackgroundColor(lightTheme.colors.background);

View File

@@ -0,0 +1,22 @@
import { AppState, Platform } from "react-native";
export function getIsAppActivelyVisible(
appState: string = AppState.currentState
): boolean {
if (appState !== "active") {
return false;
}
if (Platform.OS !== "web") {
return true;
}
const documentVisible =
typeof document === "undefined" || document.visibilityState === "visible";
const windowFocused =
typeof document === "undefined" ||
typeof document.hasFocus !== "function" ||
document.hasFocus();
return documentVisible && windowFocused;
}

View File

@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
import { parseInlinePathToken } from "./inline-path";
import { parseFileProtocolUrl, parseInlinePathToken } from "./inline-path";
describe("parseInlinePathToken", () => {
it("returns null for plain paths (no line)", () => {
@@ -31,3 +31,42 @@ describe("parseInlinePathToken", () => {
});
});
describe("parseFileProtocolUrl", () => {
it("parses file URLs with line fragments", () => {
expect(
parseFileProtocolUrl("file:///Users/test/project/src/app.tsx#L81")
).toEqual({
raw: "file:///Users/test/project/src/app.tsx#L81",
path: "/Users/test/project/src/app.tsx",
lineStart: 81,
lineEnd: undefined,
});
});
it("parses file URLs without line fragments", () => {
expect(
parseFileProtocolUrl("file:///Users/test/project/src/app.tsx")
).toEqual({
raw: "file:///Users/test/project/src/app.tsx",
path: "/Users/test/project/src/app.tsx",
lineStart: undefined,
lineEnd: undefined,
});
});
it("parses windows file URLs and line ranges", () => {
expect(
parseFileProtocolUrl("file:///C:/Users/test/project/src/app.tsx#L12-L20")
).toEqual({
raw: "file:///C:/Users/test/project/src/app.tsx#L12-L20",
path: "C:/Users/test/project/src/app.tsx",
lineStart: 12,
lineEnd: 20,
});
});
it("rejects non-file URLs and invalid ranges", () => {
expect(parseFileProtocolUrl("https://example.com/test.ts#L10")).toBeNull();
expect(parseFileProtocolUrl("file:///Users/test/project/src/app.tsx#L20-L12")).toBeNull();
});
});

View File

@@ -5,6 +5,8 @@ export interface InlinePathTarget {
lineEnd?: number;
}
const FILE_PROTOCOL = "file:";
function normalizePathToken(value: string): string | null {
const trimmed = value
.trim()
@@ -79,3 +81,64 @@ export function parseInlinePathToken(value: string): InlinePathTarget | null {
};
}
export function parseFileProtocolUrl(value: string): InlinePathTarget | null {
const trimmed = value?.trim();
if (!trimmed) {
return null;
}
let parsedUrl: URL;
try {
parsedUrl = new URL(trimmed);
} catch {
return null;
}
if (parsedUrl.protocol !== FILE_PROTOCOL) {
return null;
}
const normalizedPath = normalizeFileUrlPath(parsedUrl.pathname);
if (!normalizedPath) {
return null;
}
const rawFragment = parsedUrl.hash.startsWith("#")
? parsedUrl.hash.slice(1)
: parsedUrl.hash;
const lineMatch = rawFragment.match(/^L([0-9]+)(?:-L?([0-9]+))?$/i);
const lineStart = lineMatch?.[1] ? parseInt(lineMatch[1], 10) : undefined;
const lineEnd = lineMatch?.[2] ? parseInt(lineMatch[2], 10) : undefined;
if (
(lineStart !== undefined && (!Number.isFinite(lineStart) || lineStart <= 0)) ||
(lineEnd !== undefined && (!Number.isFinite(lineEnd) || lineEnd <= 0)) ||
(lineStart !== undefined && lineEnd !== undefined && lineEnd < lineStart)
) {
return null;
}
return {
raw: value,
path: normalizedPath,
lineStart,
lineEnd,
};
}
function normalizeFileUrlPath(pathname: string): string | null {
if (!pathname) {
return null;
}
const decoded = decodeURIComponent(pathname).replace(/\\/g, "/");
if (!decoded) {
return null;
}
if (/^\/[A-Za-z]:\//.test(decoded)) {
return decoded.slice(1);
}
return decoded;
}

View File

@@ -71,6 +71,7 @@ describe("sendOsNotification", () => {
});
afterEach(() => {
vi.doUnmock("@/desktop/notifications/desktop-notifications");
vi.doUnmock("react-native");
vi.restoreAllMocks();
vi.resetModules();
@@ -167,18 +168,13 @@ describe("sendOsNotification", () => {
expect(assign).toHaveBeenCalledWith("/h/srv%20with%20space/agent/agent%2F1");
});
it("uses Tauri notification module when available", async () => {
const isPermissionGranted = vi.fn(async () => false);
const requestPermission = vi.fn(async () => "granted");
const sendNotification = vi.fn(async () => undefined);
it("uses the desktop notification bridge when Tauri is available", async () => {
const sendDesktopNotification = vi.fn(async () => true);
vi.doMock("@/desktop/notifications/desktop-notifications", () => ({
sendDesktopNotification,
}));
(globalThis as { __TAURI__?: unknown }).__TAURI__ = {
notification: {
isPermissionGranted,
requestPermission,
sendNotification,
},
};
(globalThis as { __TAURI__?: unknown }).__TAURI__ = {};
const { sendOsNotification } = await loadModuleForPlatform("web");
const sent = await sendOsNotification({
@@ -188,11 +184,10 @@ describe("sendOsNotification", () => {
});
expect(sent).toBe(true);
expect(isPermissionGranted).toHaveBeenCalledTimes(1);
expect(requestPermission).toHaveBeenCalledTimes(1);
expect(sendNotification).toHaveBeenCalledWith({
expect(sendDesktopNotification).toHaveBeenCalledWith({
title: "Agent finished",
body: "Done",
data: { serverId: "srv-1", agentId: "agent-1" },
});
});
});

View File

@@ -1,4 +1,5 @@
import { Platform } from "react-native";
import { sendDesktopNotification } from "@/desktop/notifications/desktop-notifications";
import { buildNotificationRoute } from "./notification-routing";
import { getTauri, type TauriNotificationApi } from "@/utils/tauri";
@@ -14,13 +15,16 @@ export type WebNotificationClickDetail = {
type WebNotificationInstance = {
onclick?: ((event: Event) => void) | null;
onclose?: ((event: Event) => void) | null;
addEventListener?: (type: string, listener: (event: Event) => void) => void;
removeEventListener?: (type: string, listener: (event: Event) => void) => void;
close?: () => void;
};
export const WEB_NOTIFICATION_CLICK_EVENT = "paseo:web-notification-click";
let permissionRequest: Promise<boolean> | null = null;
const activeWebNotifications = new Set<WebNotificationInstance>();
function getTauriNotificationModule(): TauriNotificationApi | null {
if (Platform.OS !== "web") {
@@ -82,7 +86,6 @@ async function ensureTauriNotificationPermission(
try {
const granted = await notificationModule.isPermissionGranted();
if (granted) {
console.log("[OSNotifications][Tauri] Permission already granted");
return true;
}
} catch (error) {
@@ -102,7 +105,6 @@ async function ensureTauriNotificationPermission(
try {
const result = await notificationModule.requestPermission();
console.log("[OSNotifications][Tauri] requestPermission result:", result);
return result === "granted";
} catch (error) {
console.warn(
@@ -113,35 +115,6 @@ async function ensureTauriNotificationPermission(
}
}
async function sendTauriNotification(
payload: OsNotificationPayload,
notificationModule: TauriNotificationApi
): Promise<boolean> {
if (typeof notificationModule.sendNotification !== "function") {
console.warn(
"[OSNotifications][Tauri] notification.sendNotification is unavailable"
);
return false;
}
const granted = await ensureTauriNotificationPermission(notificationModule);
if (!granted) {
console.log("[OSNotifications][Tauri] Permission not granted");
return false;
}
try {
await notificationModule.sendNotification({
title: payload.title,
body: payload.body,
});
return true;
} catch (error) {
console.warn("[OSNotifications][Tauri] Failed to send notification", error);
return false;
}
}
function dispatchWebNotificationClick(detail: WebNotificationClickDetail): boolean {
const dispatch = (globalThis as { dispatchEvent?: (event: Event) => boolean }).dispatchEvent;
const CustomEventConstructor = (globalThis as { CustomEvent?: typeof CustomEvent })
@@ -183,6 +156,12 @@ function attachWebClickHandler(
notification: WebNotificationInstance,
data: Record<string, unknown> | undefined
): void {
activeWebNotifications.add(notification);
const cleanup = () => {
activeWebNotifications.delete(notification);
};
const onClick = () => {
const focus = (globalThis as { focus?: () => void }).focus;
if (typeof focus === "function") {
@@ -194,17 +173,24 @@ function attachWebClickHandler(
fallbackNavigateToNotificationTarget(data);
}
cleanup();
if (typeof notification.close === "function") {
notification.close();
}
};
const onClose = () => {
cleanup();
};
if (typeof notification.addEventListener === "function") {
notification.addEventListener("click", onClick);
notification.addEventListener("close", onClose);
return;
}
notification.onclick = onClick;
notification.onclose = onClose;
}
export async function sendOsNotification(
@@ -215,34 +201,22 @@ export async function sendOsNotification(
return false;
}
const tauriNotification = getTauriNotificationModule();
if (tauriNotification) {
console.log("[OSNotifications] Using Tauri notification module");
return await sendTauriNotification(payload, tauriNotification);
if (getTauri()) {
return await sendDesktopNotification(payload);
}
const NotificationConstructor = getWebNotificationConstructor();
if (!NotificationConstructor) {
console.log(
"[OSNotifications][Web] Notification constructor unavailable",
typeof window !== "undefined" && window.location
? window.location.origin
: "unknown-origin"
);
return false;
if (NotificationConstructor) {
const granted = await ensureNotificationPermission();
if (granted) {
const notification = new NotificationConstructor(payload.title, {
body: payload.body,
data: payload.data,
}) as WebNotificationInstance;
attachWebClickHandler(notification, payload.data);
return true;
}
}
const granted = await ensureOsNotificationPermission();
if (!granted) {
console.log(
"[OSNotifications][Web] Permission not granted:",
NotificationConstructor.permission
);
return false;
}
const notification = new NotificationConstructor(payload.title, {
body: payload.body,
data: payload.data,
}) as WebNotificationInstance;
attachWebClickHandler(notification, payload.data);
return true;
return false;
}

View File

@@ -0,0 +1,88 @@
export interface Pcm16Wav {
sampleRate: number;
samples: Int16Array;
}
export function parsePcm16Wav(buffer: ArrayBuffer): Pcm16Wav | null {
if (buffer.byteLength < 44) {
return null;
}
const view = new DataView(buffer);
function readAscii(offset: number, length: number): string {
let out = "";
for (let i = 0; i < length; i += 1) {
out += String.fromCharCode(view.getUint8(offset + i));
}
return out;
}
if (readAscii(0, 4) !== "RIFF" || readAscii(8, 4) !== "WAVE") {
return null;
}
let offset = 12;
let channels = 0;
let sampleRate = 0;
let bitsPerSample = 0;
let dataOffset = 0;
let dataSize = 0;
while (offset + 8 <= buffer.byteLength) {
const chunkId = readAscii(offset, 4);
const chunkSize = view.getUint32(offset + 4, true);
const chunkDataOffset = offset + 8;
if (chunkDataOffset + chunkSize > buffer.byteLength) {
return null;
}
if (chunkId === "fmt " && chunkSize >= 16) {
const audioFormat = view.getUint16(chunkDataOffset, true);
channels = view.getUint16(chunkDataOffset + 2, true);
sampleRate = view.getUint32(chunkDataOffset + 4, true);
bitsPerSample = view.getUint16(chunkDataOffset + 14, true);
if (audioFormat !== 1) {
return null;
}
}
if (chunkId === "data") {
dataOffset = chunkDataOffset;
dataSize = chunkSize;
break;
}
offset = chunkDataOffset + chunkSize + (chunkSize % 2);
}
if (!dataOffset || dataSize <= 0 || sampleRate <= 0 || bitsPerSample !== 16 || channels <= 0) {
return null;
}
const sampleCount = Math.floor(dataSize / 2);
const interleaved = new Int16Array(buffer, dataOffset, sampleCount);
if (channels === 1) {
return {
sampleRate,
samples: new Int16Array(interleaved),
};
}
const frameCount = Math.floor(interleaved.length / channels);
const mono = new Int16Array(frameCount);
for (let frame = 0; frame < frameCount; frame += 1) {
let sum = 0;
for (let channel = 0; channel < channels; channel += 1) {
sum += interleaved[frame * channels + channel] ?? 0;
}
mono[frame] = Math.round(sum / channels);
}
return {
sampleRate,
samples: mono,
};
}

View File

@@ -34,7 +34,7 @@ export function attachConsole(): void {
console.log = (...args: unknown[]) => {
originalLog.apply(console, args);
forwardToRust(0, args); // debug
forwardToRust(1, args); // info
};
console.info = (...args: unknown[]) => {
originalInfo.apply(console, args);

View File

@@ -0,0 +1,59 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
const tauriMock = vi.hoisted(() => {
let listenerCleanup: (() => void) | null = null;
const connect = vi.fn(async () => {
let listenerActive = false;
return {
addListener: vi.fn(() => {
listenerActive = true;
listenerCleanup = vi.fn(() => {
if (!listenerActive) {
throw new TypeError(
"undefined is not an object (evaluating 'listeners[eventId].handlerId')"
);
}
listenerActive = false;
});
return listenerCleanup;
}),
send: vi.fn(async () => undefined),
disconnect: vi.fn(async () => {
listenerCleanup?.();
}),
};
});
return {
connect,
getListenerCleanup: () => listenerCleanup,
};
});
vi.mock("./tauri", () => ({
getTauri: () => ({
websocket: {
connect: tauriMock.connect,
},
}),
}));
describe("tauri-daemon-transport", () => {
beforeEach(() => {
tauriMock.connect.mockClear();
});
it("does not unregister the websocket listener twice during close", async () => {
const mod = await import("./tauri-daemon-transport");
const transportFactory = mod.createTauriWebSocketTransportFactory();
expect(transportFactory).not.toBeNull();
const transport = transportFactory!({ url: "ws://localhost:6767/ws" });
await Promise.resolve();
expect(() => transport.close()).not.toThrow();
expect(tauriMock.getListenerCleanup()).toHaveBeenCalledTimes(1);
});
});

View File

@@ -229,14 +229,10 @@ export function createTauriWebSocketTransportFactory(): DaemonTransportFactory |
return {
...transport,
close: (code?: number, reason?: string) => {
wsListenerCleanup = null;
transport.close(code, reason);
disposed = true;
try {
wsListenerCleanup?.();
} catch {
// no-op
}
wsListenerCleanup = null;
ws = null;
},
};
};

View File

@@ -7,8 +7,22 @@ export interface TauriDialogAskOptions {
kind?: "info" | "warning" | "error";
}
export interface TauriDialogOpenOptions {
title?: string;
defaultPath?: string;
directory?: boolean;
multiple?: boolean;
filters?: Array<{
name: string;
extensions: string[];
}>;
}
export interface TauriDialogApi {
ask?: (message: string, options?: TauriDialogAskOptions) => Promise<boolean>;
open?: (
options?: TauriDialogOpenOptions
) => Promise<string | string[] | null>;
}
export interface TauriCoreApi {

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,76 @@
import { describe, expect, it } from "vitest";
import { parsePcm16Wav } from "@/utils/pcm16-wav";
interface BuildWavInput {
channels?: number;
sampleRate?: number;
samples: number[];
}
function buildPcm16Wav(input: BuildWavInput): ArrayBuffer {
const channels = input.channels ?? 1;
const sampleRate = input.sampleRate ?? 24000;
const bytesPerSample = 2;
const dataSize = input.samples.length * bytesPerSample;
const buffer = new ArrayBuffer(44 + dataSize);
const view = new DataView(buffer);
function writeAscii(offset: number, value: string): void {
for (let i = 0; i < value.length; i += 1) {
view.setUint8(offset + i, value.charCodeAt(i));
}
}
writeAscii(0, "RIFF");
view.setUint32(4, 36 + dataSize, true);
writeAscii(8, "WAVE");
writeAscii(12, "fmt ");
view.setUint32(16, 16, true);
view.setUint16(20, 1, true);
view.setUint16(22, channels, true);
view.setUint32(24, sampleRate, true);
view.setUint32(28, sampleRate * channels * bytesPerSample, true);
view.setUint16(32, channels * bytesPerSample, true);
view.setUint16(34, 16, true);
writeAscii(36, "data");
view.setUint32(40, dataSize, true);
input.samples.forEach((sample, index) => {
view.setInt16(44 + index * bytesPerSample, sample, true);
});
return buffer;
}
describe("parsePcm16Wav", () => {
it("parses mono PCM16 wav data", () => {
const buffer = buildPcm16Wav({
sampleRate: 16000,
samples: [100, -200, 300],
});
const parsed = parsePcm16Wav(buffer);
expect(parsed).not.toBeNull();
expect(parsed?.sampleRate).toBe(16000);
expect(Array.from(parsed?.samples ?? [])).toEqual([100, -200, 300]);
});
it("downmixes multichannel PCM16 wav data to mono", () => {
const buffer = buildPcm16Wav({
channels: 2,
samples: [1000, -1000, 3000, 1000],
});
const parsed = parsePcm16Wav(buffer);
expect(parsed).not.toBeNull();
expect(Array.from(parsed?.samples ?? [])).toEqual([0, 2000]);
});
it("rejects non-wav payloads", () => {
const buffer = new TextEncoder().encode("not-a-wav").buffer;
expect(parsePcm16Wav(buffer)).toBeNull();
});
});

View File

@@ -0,0 +1,33 @@
import { Asset } from "expo-asset";
import { File } from "expo-file-system";
import { Platform } from "react-native";
export { parsePcm16Wav, type Pcm16Wav } from "@/utils/pcm16-wav";
export const THINKING_TONE_REPEAT_GAP_MS = 350;
let thinkingToneArrayBufferPromise: Promise<ArrayBuffer> | null = null;
async function readThinkingToneArrayBuffer(): Promise<ArrayBuffer> {
const toneModule = require("../../assets/audio/thinking-tone.wav");
const asset = Asset.fromModule(toneModule);
if (Platform.OS === "web") {
const response = await fetch(asset.uri);
if (!response.ok) {
throw new Error(`Failed to fetch thinking tone asset: ${response.status}`);
}
return await response.arrayBuffer();
}
const resolvedAsset = asset.localUri ? asset : await asset.downloadAsync();
const fileUri = resolvedAsset.localUri ?? resolvedAsset.uri;
const file = new File(fileUri);
return await file.arrayBuffer();
}
export async function loadThinkingToneArrayBuffer(): Promise<ArrayBuffer> {
if (!thinkingToneArrayBufferPromise) {
thinkingToneArrayBufferPromise = readThinkingToneArrayBuffer();
}
return await thinkingToneArrayBufferPromise;
}

View File

@@ -140,4 +140,38 @@ describe("tool-call-display", () => {
expect(display.errorText).toBe('{\n "message": "boom"\n}');
});
it("shows terminal interaction with only the fixed label when no command is available", () => {
const display = buildToolCallDisplayModel({
name: "terminal",
status: "completed",
error: null,
detail: {
type: "plain_text",
icon: "square_terminal",
},
});
expect(display).toEqual({
displayName: "Interacted with terminal",
});
});
it("shows terminal interaction command as the summary when available", () => {
const display = buildToolCallDisplayModel({
name: "terminal",
status: "completed",
error: null,
detail: {
type: "plain_text",
label: "npm run test",
icon: "square_terminal",
},
});
expect(display).toEqual({
displayName: "Interacted with terminal",
summary: "npm run test",
});
});
});

View File

@@ -0,0 +1,29 @@
export interface AudioEngineCallbacks {
onCaptureData(pcm: Uint8Array): void;
onVolumeLevel(level: number): void;
onError?(error: Error): void;
}
export interface AudioPlaybackSource {
arrayBuffer(): Promise<ArrayBuffer>;
size: number;
type: string;
}
export interface AudioEngine {
initialize(): Promise<void>;
destroy(): Promise<void>;
startCapture(): Promise<void>;
stopCapture(): Promise<void>;
toggleMute(): boolean;
isMuted(): boolean;
play(audio: AudioPlaybackSource): Promise<number>;
stop(): void;
clearQueue(): void;
isPlaying(): boolean;
playLooping(pcm: Uint8Array, gapMs: number): void;
stopLooping(): void;
}

View File

@@ -0,0 +1 @@
export * from "./audio-engine.native";

View File

@@ -0,0 +1,402 @@
import {
addExpoTwoWayAudioEventListener,
getMicrophonePermissionsAsync,
initialize,
playPCMData,
requestMicrophonePermissionsAsync,
resumePlayback,
stopPlayback,
tearDown,
toggleRecording,
} from "@boudra/expo-two-way-audio";
import {
THINKING_TONE_REPEAT_GAP_MS,
} from "@/utils/thinking-tone";
import {
THINKING_TONE_NATIVE_PCM_BASE64,
THINKING_TONE_NATIVE_PCM_DURATION_MS,
} from "@/utils/thinking-tone.native-pcm";
import type {
AudioEngine,
AudioEngineCallbacks,
AudioPlaybackSource,
} from "@/voice/audio-engine-types";
import { Buffer } from "buffer";
interface QueuedAudio {
audio: AudioPlaybackSource;
resolve: (duration: number) => void;
reject: (error: Error) => void;
}
interface CuePcm {
pcm16k: Uint8Array;
durationMs: number;
}
interface AudioEngineTraceOptions {
traceLabel?: string;
}
function resamplePcm16(
pcm: Uint8Array,
fromRate: number,
toRate: number
): Uint8Array {
if (fromRate === toRate) {
return pcm;
}
const inputSamples = Math.floor(pcm.length / 2);
const outputSamples = Math.floor((inputSamples * toRate) / fromRate);
const output = new Uint8Array(outputSamples * 2);
const ratio = fromRate / toRate;
const readInt16 = (sampleIndex: number): number => {
const offset = sampleIndex * 2;
if (offset + 1 >= pcm.length) {
return 0;
}
const lo = pcm[offset]!;
const hi = pcm[offset + 1]!;
let value = (hi << 8) | lo;
if (value & 0x8000) {
value -= 0x10000;
}
return value;
};
const writeInt16 = (sampleIndex: number, value: number): void => {
const clamped = Math.max(-32768, Math.min(32767, Math.round(value)));
const offset = sampleIndex * 2;
output[offset] = clamped & 0xff;
output[offset + 1] = (clamped >> 8) & 0xff;
};
for (let i = 0; i < outputSamples; i += 1) {
const sourceIndex = i * ratio;
const i0 = Math.floor(sourceIndex);
const frac = sourceIndex - i0;
const s0 = readInt16(i0);
const s1 = readInt16(Math.min(inputSamples - 1, i0 + 1));
writeInt16(i, s0 + (s1 - s0) * frac);
}
return output;
}
function parsePcmSampleRate(mimeType: string): number | null {
const match = /rate=(\d+)/i.exec(mimeType);
if (!match) {
return null;
}
const rate = Number(match[1]);
return Number.isFinite(rate) && rate > 0 ? rate : null;
}
export function createAudioEngine(
callbacks: AudioEngineCallbacks,
_options?: AudioEngineTraceOptions
): AudioEngine {
const refs: {
initialized: boolean;
captureActive: boolean;
muted: boolean;
queue: QueuedAudio[];
processingQueue: boolean;
playbackTimeout: ReturnType<typeof setTimeout> | null;
activePlayback: {
resolve: (duration: number) => void;
reject: (error: Error) => void;
settled: boolean;
} | null;
looping: {
active: boolean;
token: number;
timeout: ReturnType<typeof setTimeout> | null;
};
thinkingTone: CuePcm | null;
destroyed: boolean;
} = {
initialized: false,
captureActive: false,
muted: false,
queue: [],
processingQueue: false,
playbackTimeout: null,
activePlayback: null,
looping: {
active: false,
token: 0,
timeout: null,
},
thinkingTone: null,
destroyed: false,
};
const microphoneSubscription = addExpoTwoWayAudioEventListener(
"onMicrophoneData",
(event) => {
if (!refs.captureActive || refs.muted) {
return;
}
callbacks.onCaptureData(event.data);
}
);
const volumeSubscription = addExpoTwoWayAudioEventListener(
"onInputVolumeLevelData",
(event) => {
if (!refs.captureActive) {
return;
}
const level = refs.muted ? 0 : event.data;
callbacks.onVolumeLevel(level);
}
);
async function ensureInitialized(): Promise<void> {
if (refs.initialized) {
return;
}
await initialize();
refs.initialized = true;
}
async function ensureMicrophonePermission(): Promise<void> {
let permission = await getMicrophonePermissionsAsync().catch(() => null);
if (!permission?.granted) {
permission = await requestMicrophonePermissionsAsync().catch(() => null);
}
if (!permission?.granted) {
throw new Error(
"Microphone permission is required to capture audio. Please enable microphone access in system settings."
);
}
}
async function ensureThinkingTone(): Promise<CuePcm> {
if (refs.thinkingTone) {
return refs.thinkingTone;
}
const pcm16k = Buffer.from(THINKING_TONE_NATIVE_PCM_BASE64, "base64");
const durationMs = THINKING_TONE_NATIVE_PCM_DURATION_MS;
refs.thinkingTone = { pcm16k, durationMs };
return refs.thinkingTone;
}
function clearPlaybackTimeout(): void {
if (refs.playbackTimeout) {
clearTimeout(refs.playbackTimeout);
refs.playbackTimeout = null;
}
}
async function playAudio(audio: AudioPlaybackSource): Promise<number> {
await ensureInitialized();
resumePlayback();
return await new Promise<number>(async (resolve, reject) => {
refs.activePlayback = { resolve, reject, settled: false };
try {
const arrayBuffer = await audio.arrayBuffer();
const pcm = new Uint8Array(arrayBuffer);
const inputRate = parsePcmSampleRate(audio.type || "") ?? 24000;
const pcm16k = resamplePcm16(pcm, inputRate, 16000);
const durationSec = pcm16k.length / 2 / 16000;
playPCMData(pcm16k);
clearPlaybackTimeout();
refs.playbackTimeout = setTimeout(() => {
clearPlaybackTimeout();
const active = refs.activePlayback;
if (!active || active.settled) {
return;
}
active.settled = true;
refs.activePlayback = null;
resolve(durationSec);
}, durationSec * 1000);
} catch (error) {
clearPlaybackTimeout();
const active = refs.activePlayback;
if (active && !active.settled) {
active.settled = true;
refs.activePlayback = null;
reject(error instanceof Error ? error : new Error(String(error)));
}
}
});
}
async function processQueue(): Promise<void> {
if (refs.processingQueue || refs.queue.length === 0) {
return;
}
refs.processingQueue = true;
while (refs.queue.length > 0) {
const item = refs.queue.shift()!;
try {
const duration = await playAudio(item.audio);
item.resolve(duration);
} catch (error) {
item.reject(error instanceof Error ? error : new Error(String(error)));
}
}
refs.processingQueue = false;
}
function stopLooping(): void {
refs.looping.active = false;
refs.looping.token += 1;
if (refs.looping.timeout) {
clearTimeout(refs.looping.timeout);
refs.looping.timeout = null;
}
stopPlayback();
}
return {
async initialize() {
await ensureInitialized();
await ensureThinkingTone();
},
async destroy() {
if (refs.destroyed) {
return;
}
refs.destroyed = true;
stopLooping();
this.stop();
this.clearQueue();
if (refs.captureActive) {
toggleRecording(false);
refs.captureActive = false;
}
clearPlaybackTimeout();
refs.muted = false;
callbacks.onVolumeLevel(0);
if (refs.initialized) {
tearDown();
refs.initialized = false;
}
microphoneSubscription.remove();
volumeSubscription.remove();
},
async startCapture() {
if (refs.captureActive) {
return;
}
try {
await ensureMicrophonePermission();
await ensureInitialized();
toggleRecording(true);
refs.captureActive = true;
} catch (error) {
const wrapped = error instanceof Error ? error : new Error(String(error));
callbacks.onError?.(wrapped);
throw wrapped;
}
},
async stopCapture() {
if (refs.captureActive) {
toggleRecording(false);
}
refs.captureActive = false;
refs.muted = false;
callbacks.onVolumeLevel(0);
},
toggleMute() {
refs.muted = !refs.muted;
if (refs.muted) {
callbacks.onVolumeLevel(0);
}
return refs.muted;
},
isMuted() {
return refs.muted;
},
async play(audio: AudioPlaybackSource) {
return await new Promise<number>((resolve, reject) => {
refs.queue.push({ audio, resolve, reject });
if (!refs.processingQueue) {
void processQueue();
}
});
},
stop() {
stopPlayback();
clearPlaybackTimeout();
const active = refs.activePlayback;
refs.activePlayback = null;
if (active && !active.settled) {
active.settled = true;
active.reject(new Error("Playback stopped"));
}
},
clearQueue() {
while (refs.queue.length > 0) {
refs.queue.shift()!.reject(new Error("Playback stopped"));
}
refs.processingQueue = false;
},
isPlaying() {
return refs.activePlayback !== null;
},
playLooping(audio, gapMs) {
if (refs.looping.active) {
return;
}
refs.looping.active = true;
const token = refs.looping.token + 1;
refs.looping.token = token;
void (async () => {
try {
await ensureInitialized();
const cue =
audio.byteLength > 0
? {
pcm16k: audio,
durationMs: (audio.byteLength / 2 / 16000) * 1000,
}
: await ensureThinkingTone();
const loop = () => {
if (!refs.looping.active || refs.looping.token !== token) {
return;
}
resumePlayback();
playPCMData(cue.pcm16k);
refs.looping.timeout = setTimeout(
loop,
cue.durationMs + (gapMs || THINKING_TONE_REPEAT_GAP_MS)
);
};
loop();
} catch (error) {
callbacks.onError?.(
error instanceof Error ? error : new Error(String(error))
);
}
})();
},
stopLooping,
};
}

View File

@@ -0,0 +1,548 @@
import { getTauri } from "@/utils/tauri";
import {
loadThinkingToneArrayBuffer,
THINKING_TONE_REPEAT_GAP_MS,
} from "@/utils/thinking-tone";
import type {
AudioEngine,
AudioEngineCallbacks,
AudioPlaybackSource,
} from "@/voice/audio-engine-types";
interface QueuedAudio {
audio: AudioPlaybackSource;
resolve: (duration: number) => void;
reject: (error: Error) => void;
}
function getAudioContextCtor(): (typeof AudioContext) | null {
if (typeof window === "undefined") {
return null;
}
const browserWindow = window as typeof window & {
webkitAudioContext?: typeof AudioContext;
};
return browserWindow.AudioContext ?? browserWindow.webkitAudioContext ?? null;
}
function floatToInt16(sample: number): number {
const clamped = Math.max(-1, Math.min(1, sample));
return clamped < 0 ? Math.round(clamped * 0x8000) : Math.round(clamped * 0x7fff);
}
function resampleToPcm16(
input: Float32Array,
inputRate: number,
outputRate: number
): Uint8Array {
if (input.length === 0) {
return new Uint8Array(0);
}
const ratio = inputRate / outputRate;
const outputLength = Math.max(1, Math.round(input.length / ratio));
const output = new Int16Array(outputLength);
for (let i = 0; i < outputLength; i += 1) {
const sourceIndex = i * ratio;
const i0 = Math.floor(sourceIndex);
const i1 = Math.min(input.length - 1, i0 + 1);
const frac = sourceIndex - i0;
const sample = input[i0]! * (1 - frac) + input[i1]! * frac;
output[i] = floatToInt16(sample);
}
return new Uint8Array(output.buffer, output.byteOffset, output.byteLength);
}
function parsePcmSampleRate(mimeType: string): number | null {
const match = /rate=(\d+)/i.exec(mimeType);
if (!match) {
return null;
}
const rate = Number(match[1]);
return Number.isFinite(rate) && rate > 0 ? rate : null;
}
function pcm16LeToAudioBuffer(
context: AudioContext,
bytes: Uint8Array,
sampleRate: number
): AudioBuffer {
const sampleCount = Math.floor(bytes.length / 2);
const audioBuffer = context.createBuffer(1, sampleCount, sampleRate);
const channel = audioBuffer.getChannelData(0);
for (let i = 0; i < sampleCount; i += 1) {
const lo = bytes[i * 2]!;
const hi = bytes[i * 2 + 1]!;
let value = (hi << 8) | lo;
if (value & 0x8000) {
value -= 0x10000;
}
channel[i] = value / 0x8000;
}
return audioBuffer;
}
async function decodeAudioData(
context: AudioContext,
buffer: ArrayBuffer
): Promise<AudioBuffer> {
const maybePromise = context.decodeAudioData(buffer.slice(0));
if (maybePromise && typeof (maybePromise as Promise<AudioBuffer>).then === "function") {
return maybePromise as Promise<AudioBuffer>;
}
return await new Promise<AudioBuffer>((resolve, reject) => {
context.decodeAudioData(buffer.slice(0), resolve, reject);
});
}
export function createAudioEngine(
callbacks: AudioEngineCallbacks,
_options?: { traceLabel?: string }
): AudioEngine {
const refs: {
playbackContext: AudioContext | null;
captureContext: AudioContext | null;
stream: MediaStream | null;
source: MediaStreamAudioSourceNode | null;
processor: ScriptProcessorNode | null;
gain: GainNode | null;
started: boolean;
muted: boolean;
queue: QueuedAudio[];
processingQueue: boolean;
activePlayback: {
source: AudioBufferSourceNode;
resolve: (duration: number) => void;
reject: (error: Error) => void;
settled: boolean;
} | null;
looping: {
token: number;
source: AudioBufferSourceNode | null;
gain: GainNode | null;
timeout: ReturnType<typeof setTimeout> | null;
};
thinkingTone: {
bytes: Uint8Array | null;
buffer: AudioBuffer | null;
};
} = {
playbackContext: null,
captureContext: null,
stream: null,
source: null,
processor: null,
gain: null,
started: false,
muted: false,
queue: [],
processingQueue: false,
activePlayback: null,
looping: {
token: 0,
source: null,
gain: null,
timeout: null,
},
thinkingTone: {
bytes: null,
buffer: null,
},
};
async function ensurePlaybackContext(): Promise<AudioContext> {
if (refs.playbackContext) {
if (refs.playbackContext.state === "suspended") {
await refs.playbackContext.resume().catch(() => undefined);
}
return refs.playbackContext;
}
const AudioContextCtor = getAudioContextCtor();
if (!AudioContextCtor) {
throw new Error("AudioContext unavailable");
}
const context = new AudioContextCtor();
if (context.state === "suspended") {
await context.resume().catch(() => undefined);
}
refs.playbackContext = context;
return context;
}
async function ensureCaptureContext(): Promise<AudioContext> {
if (refs.captureContext) {
if (refs.captureContext.state === "suspended") {
await refs.captureContext.resume().catch(() => undefined);
}
return refs.captureContext;
}
const AudioContextCtor = getAudioContextCtor();
if (!AudioContextCtor) {
throw new Error("AudioContext unavailable");
}
const context = new AudioContextCtor();
if (context.state === "suspended") {
await context.resume().catch(() => undefined);
}
refs.captureContext = context;
return context;
}
async function ensureThinkingToneBytes(): Promise<Uint8Array> {
if (refs.thinkingTone.bytes) {
return refs.thinkingTone.bytes;
}
const wav = await loadThinkingToneArrayBuffer();
refs.thinkingTone.bytes = new Uint8Array(wav);
return refs.thinkingTone.bytes;
}
async function ensureThinkingToneBuffer(): Promise<AudioBuffer> {
if (refs.thinkingTone.buffer) {
return refs.thinkingTone.buffer;
}
const context = await ensurePlaybackContext();
const wav = await loadThinkingToneArrayBuffer();
const audioBuffer = await decodeAudioData(context, wav);
refs.thinkingTone.buffer = audioBuffer;
return audioBuffer;
}
function stopLooping(): void {
refs.looping.token += 1;
if (refs.looping.timeout) {
clearTimeout(refs.looping.timeout);
refs.looping.timeout = null;
}
if (refs.looping.source) {
try {
refs.looping.source.onended = null;
refs.looping.source.stop();
} catch {
// Ignore best-effort stop errors.
}
refs.looping.source.disconnect();
refs.looping.source = null;
}
if (refs.looping.gain) {
refs.looping.gain.disconnect();
refs.looping.gain = null;
}
}
async function playAudio(audio: AudioPlaybackSource): Promise<number> {
const context = await ensurePlaybackContext();
const arrayBuffer = await audio.arrayBuffer();
const type = (audio.type || "").toLowerCase();
const audioBuffer = type.startsWith("audio/pcm")
? pcm16LeToAudioBuffer(
context,
new Uint8Array(arrayBuffer),
parsePcmSampleRate(type) ?? 24000
)
: await decodeAudioData(context, arrayBuffer);
const durationSec = audioBuffer.duration;
const source = context.createBufferSource();
source.buffer = audioBuffer;
source.connect(context.destination);
return await new Promise<number>((resolve, reject) => {
refs.activePlayback = { source, resolve, reject, settled: false };
const settle = (fn: () => void) => {
const active = refs.activePlayback;
if (!active || active.source !== source || active.settled) {
return;
}
active.settled = true;
refs.activePlayback = null;
fn();
};
source.onended = () => {
settle(() => resolve(durationSec));
};
try {
source.start();
} catch (error) {
settle(() =>
reject(error instanceof Error ? error : new Error(String(error)))
);
}
});
}
async function processQueue(): Promise<void> {
if (refs.processingQueue || refs.queue.length === 0) {
return;
}
refs.processingQueue = true;
while (refs.queue.length > 0) {
const item = refs.queue.shift()!;
try {
const duration = await playAudio(item.audio);
item.resolve(duration);
} catch (error) {
item.reject(error instanceof Error ? error : new Error(String(error)));
}
}
refs.processingQueue = false;
}
async function stopCapture(): Promise<void> {
refs.started = false;
try {
refs.processor?.disconnect();
refs.source?.disconnect();
refs.gain?.disconnect();
} catch {
// Ignore best-effort teardown errors.
}
if (refs.stream) {
for (const track of refs.stream.getTracks()) {
try {
track.stop();
} catch {
// Ignore best-effort teardown errors.
}
}
}
refs.stream = null;
refs.source = null;
refs.processor = null;
refs.gain = null;
refs.muted = false;
callbacks.onVolumeLevel(0);
const captureContext = refs.captureContext;
refs.captureContext = null;
if (captureContext && captureContext.state !== "closed") {
await captureContext.close().catch(() => undefined);
}
}
return {
async initialize() {
await ensurePlaybackContext();
await Promise.all([ensureThinkingToneBytes(), ensureThinkingToneBuffer()]);
},
async destroy() {
stopLooping();
this.stop();
this.clearQueue();
await stopCapture();
const playbackContext = refs.playbackContext;
refs.playbackContext = null;
refs.thinkingTone.buffer = null;
if (playbackContext && playbackContext.state !== "closed") {
await playbackContext.close().catch(() => undefined);
}
},
async startCapture() {
if (refs.started) {
return;
}
const missingNavigator =
typeof navigator === "undefined" ||
!navigator.mediaDevices ||
typeof navigator.mediaDevices.getUserMedia !== "function";
const secureContext =
typeof window !== "undefined" && typeof window.isSecureContext === "boolean"
? window.isSecureContext
: true;
const currentOrigin =
typeof window !== "undefined" && window.location
? window.location.origin
: "unknown";
const isTauri = getTauri() !== null;
if (missingNavigator) {
throw new Error("Microphone capture is not supported in this environment");
}
if (!secureContext && !isTauri) {
throw new Error(
`Microphone access requires HTTPS or localhost. Current origin: ${currentOrigin}`
);
}
try {
const context = await ensureCaptureContext();
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
channelCount: 1,
noiseSuppression: true,
echoCancellation: true,
autoGainControl: true,
},
});
const source = context.createMediaStreamSource(stream);
const processor = context.createScriptProcessor(4096, 1, 1);
const gain = context.createGain();
gain.gain.value = 0;
processor.onaudioprocess = (event) => {
if (!refs.started) {
return;
}
const input = event.inputBuffer.getChannelData(0);
let sumSquares = 0;
for (let i = 0; i < input.length; i += 1) {
const sample = input[i]!;
sumSquares += sample * sample;
}
const rms = Math.sqrt(sumSquares / Math.max(1, input.length));
const normalized = Math.min(1, Math.max(0, rms * 2));
callbacks.onVolumeLevel(normalized);
if (refs.muted) {
return;
}
callbacks.onCaptureData(resampleToPcm16(input, context.sampleRate, 16000));
};
source.connect(processor);
processor.connect(gain);
gain.connect(context.destination);
refs.started = true;
refs.stream = stream;
refs.source = source;
refs.processor = processor;
refs.gain = gain;
} catch (error) {
await stopCapture();
const wrapped = error instanceof Error ? error : new Error(String(error));
callbacks.onError?.(wrapped);
throw wrapped;
}
},
async stopCapture() {
await stopCapture();
},
toggleMute() {
refs.muted = !refs.muted;
if (refs.muted) {
callbacks.onVolumeLevel(0);
}
return refs.muted;
},
isMuted() {
return refs.muted;
},
async play(audio: AudioPlaybackSource) {
return await new Promise<number>((resolve, reject) => {
refs.queue.push({ audio, resolve, reject });
if (!refs.processingQueue) {
void processQueue();
}
});
},
stop() {
if (refs.activePlayback) {
const active = refs.activePlayback;
refs.activePlayback = null;
try {
active.source.onended = null;
active.source.stop();
} catch {
// Ignore best-effort stop errors.
}
if (!active.settled) {
active.settled = true;
active.reject(new Error("Playback stopped"));
}
}
},
clearQueue() {
while (refs.queue.length > 0) {
refs.queue.shift()!.reject(new Error("Playback stopped"));
}
refs.processingQueue = false;
},
isPlaying() {
return refs.activePlayback !== null;
},
playLooping(audio, gapMs) {
if (refs.looping.source || refs.looping.timeout) {
return;
}
const token = refs.looping.token + 1;
refs.looping.token = token;
void (async () => {
try {
const context = await ensurePlaybackContext();
const audioBuffer =
audio.byteLength > 0
? pcm16LeToAudioBuffer(context, audio, 16000)
: await ensureThinkingToneBuffer();
if (refs.looping.token !== token) {
return;
}
const gain = context.createGain();
gain.gain.value = 1;
gain.connect(context.destination);
refs.looping.gain = gain;
const playOnce = () => {
if (refs.looping.token !== token) {
return;
}
const source = context.createBufferSource();
source.buffer = audioBuffer;
source.connect(gain);
refs.looping.source = source;
source.onended = () => {
if (refs.looping.source === source) {
refs.looping.source = null;
}
if (refs.looping.token !== token) {
return;
}
refs.looping.timeout = setTimeout(
playOnce,
gapMs || THINKING_TONE_REPEAT_GAP_MS
);
};
source.start();
};
playOnce();
} catch (error) {
callbacks.onError?.(
error instanceof Error ? error : new Error(String(error))
);
}
})();
},
stopLooping,
};
}

View File

@@ -0,0 +1,243 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { DaemonServerInfo } from "@/stores/session-store";
import type { AudioEngine } from "@/voice/audio-engine-types";
import {
createVoiceRuntime,
type VoiceRuntime,
type VoiceSessionAdapter,
} from "@/voice/voice-runtime";
import { REALTIME_VOICE_VAD_CONFIG } from "@/voice/realtime-voice-config";
function createAudioEngineMock(): AudioEngine {
return {
initialize: vi.fn().mockResolvedValue(undefined),
destroy: vi.fn().mockResolvedValue(undefined),
startCapture: vi.fn().mockResolvedValue(undefined),
stopCapture: vi.fn().mockResolvedValue(undefined),
toggleMute: vi.fn().mockReturnValue(true),
isMuted: vi.fn().mockReturnValue(false),
play: vi.fn().mockResolvedValue(0.1),
stop: vi.fn(),
clearQueue: vi.fn(),
isPlaying: vi.fn().mockReturnValue(false),
playLooping: vi.fn(),
stopLooping: vi.fn(),
};
}
function createSessionAdapter(serverId = "server-1"): VoiceSessionAdapter {
return {
serverId,
setVoiceMode: vi.fn().mockResolvedValue(undefined),
sendVoiceAudioChunk: vi.fn().mockResolvedValue(undefined),
abortRequest: vi.fn().mockResolvedValue(undefined),
setAssistantAudioPlaying: vi.fn(),
};
}
function createServerInfo(): DaemonServerInfo {
return {
serverId: "server-1",
hostname: "host",
version: "1.0.0",
capabilities: {
voice: {
dictation: { enabled: true, reason: "" },
voice: { enabled: true, reason: "" },
},
},
};
}
function createRuntime(options?: {
engine?: AudioEngine;
getServerInfo?: (serverId: string) => DaemonServerInfo | null;
}) {
const engine = options?.engine ?? createAudioEngineMock();
const runtime = createVoiceRuntime({
engine,
getServerInfo: options?.getServerInfo ?? (() => createServerInfo()),
activateKeepAwake: vi.fn().mockResolvedValue(undefined),
deactivateKeepAwake: vi.fn().mockResolvedValue(undefined),
});
return { runtime, engine };
}
describe("voice runtime", () => {
beforeEach(() => {
vi.useFakeTimers();
});
afterEach(() => {
vi.useRealTimers();
});
it("starts voice when adapter is ready", async () => {
const adapter = createSessionAdapter();
const { runtime, engine } = createRuntime();
runtime.registerSession(adapter);
await runtime.startVoice("server-1", "agent-1");
expect(engine.initialize).toHaveBeenCalled();
expect(adapter.setVoiceMode).toHaveBeenCalledWith(true, "agent-1");
expect(engine.startCapture).toHaveBeenCalled();
expect(runtime.getSnapshot()).toMatchObject({
phase: "listening",
isVoiceMode: true,
activeServerId: "server-1",
activeAgentId: "agent-1",
});
});
it("transitions from capturing to submitting to waiting on the final chunk", async () => {
const adapter = createSessionAdapter();
const { runtime } = createRuntime();
runtime.registerSession(adapter);
await runtime.startVoice("server-1", "agent-1");
runtime.handleCaptureVolume(0.4);
await vi.advanceTimersByTimeAsync(
REALTIME_VOICE_VAD_CONFIG.speechConfirmationMs + 10
);
runtime.handleCaptureVolume(0.4);
runtime.handleCapturePcm(new Uint8Array(32000));
expect(runtime.getSnapshot().phase).toBe("capturing");
runtime.handleCapturePcm(new Uint8Array(4000));
runtime.handleCaptureVolume(0);
await vi.advanceTimersByTimeAsync(
REALTIME_VOICE_VAD_CONFIG.confirmedDropGracePeriodMs + 10
);
runtime.handleCaptureVolume(0);
await vi.advanceTimersByTimeAsync(
REALTIME_VOICE_VAD_CONFIG.silenceDurationMs + 10
);
runtime.handleCaptureVolume(0);
await Promise.resolve();
expect(adapter.sendVoiceAudioChunk).toHaveBeenLastCalledWith(
expect.any(String),
"audio/pcm;rate=16000;bits=16",
true
);
expect(runtime.getSnapshot().phase).toBe("waiting");
});
it("moves from waiting to playing on the first assistant audio", async () => {
const adapter = createSessionAdapter();
const { runtime, engine } = createRuntime();
runtime.registerSession(adapter);
await runtime.startVoice("server-1", "agent-1");
runtime.onAssistantAudioStarted("server-1");
expect(runtime.getSnapshot().phase).toBe("playing");
expect(engine.stopLooping).toHaveBeenCalled();
expect(adapter.setAssistantAudioPlaying).toHaveBeenCalledWith(true);
});
it("returns to waiting after assistant playback when the turn is still active", async () => {
const adapter = createSessionAdapter();
const { runtime, engine } = createRuntime();
runtime.registerSession(adapter);
await runtime.startVoice("server-1", "agent-1");
runtime.onTurnEvent("server-1", "agent-1", "turn_started");
runtime.onAssistantAudioStarted("server-1");
runtime.onAssistantAudioFinished("server-1");
expect(runtime.getSnapshot().phase).toBe("waiting");
expect(engine.playLooping).toHaveBeenCalled();
});
it("returns to listening after assistant playback once the turn is complete", async () => {
const adapter = createSessionAdapter();
const { runtime, engine } = createRuntime();
runtime.registerSession(adapter);
await runtime.startVoice("server-1", "agent-1");
runtime.onTurnEvent("server-1", "agent-1", "turn_started");
runtime.onAssistantAudioStarted("server-1");
runtime.onTurnEvent("server-1", "agent-1", "turn_completed");
runtime.onAssistantAudioFinished("server-1");
expect(runtime.getSnapshot().phase).toBe("listening");
expect(engine.playLooping).not.toHaveBeenCalled();
});
it("interrupts the active turn on barge-in after the grace period", async () => {
const adapter = createSessionAdapter();
const { runtime, engine } = createRuntime();
runtime.registerSession(adapter);
await runtime.startVoice("server-1", "agent-1");
runtime.onTurnEvent("server-1", "agent-1", "turn_started");
runtime.onAssistantAudioStarted("server-1");
runtime.handleCaptureVolume(0.5);
await vi.advanceTimersByTimeAsync(
REALTIME_VOICE_VAD_CONFIG.speechConfirmationMs + 10
);
await vi.advanceTimersByTimeAsync(
REALTIME_VOICE_VAD_CONFIG.interruptGracePeriodMs
);
expect(adapter.abortRequest).toHaveBeenCalled();
expect(engine.stop).toHaveBeenCalled();
expect(runtime.getSnapshot().phase).toBe("capturing");
});
it("authoritatively stops and suppresses later voice audio", async () => {
const adapter = createSessionAdapter();
const { runtime, engine } = createRuntime();
runtime.registerSession(adapter);
await runtime.startVoice("server-1", "agent-1");
await runtime.stopVoice();
expect(adapter.setVoiceMode).toHaveBeenLastCalledWith(false);
expect(engine.stopCapture).toHaveBeenCalled();
expect(runtime.getSnapshot().phase).toBe("disabled");
expect(runtime.shouldPlayVoiceAudio("server-1")).toBe(false);
});
it("returns an explicit not-ready error when the adapter is missing", async () => {
const { runtime } = createRuntime();
await expect(runtime.startVoice("server-1", "agent-1")).rejects.toThrow(
"Voice runtime is not ready for host server-1"
);
});
it("resyncs voice mode after connection recovers", async () => {
const adapter = createSessionAdapter();
const { runtime } = createRuntime();
runtime.registerSession(adapter);
await runtime.startVoice("server-1", "agent-1");
vi.mocked(adapter.setVoiceMode).mockClear();
runtime.updateSessionConnection("server-1", false);
runtime.updateSessionConnection("server-1", true);
await Promise.resolve();
expect(adapter.setVoiceMode).toHaveBeenCalledWith(true, "agent-1");
});
it("does not emit when the snapshot is unchanged", async () => {
const adapter = createSessionAdapter();
const { runtime } = createRuntime();
runtime.registerSession(adapter);
await runtime.startVoice("server-1", "agent-1");
const listener = vi.fn();
const unsubscribe = runtime.subscribe(listener);
runtime.handleCaptureVolume(0);
runtime.handleCaptureVolume(0);
expect(listener).not.toHaveBeenCalled();
unsubscribe();
});
});

View File

@@ -0,0 +1,751 @@
import type { AgentStreamEventPayload } from "@server/shared/messages";
import { resolveVoiceUnavailableMessage } from "@/utils/server-info-capabilities";
import type { DaemonServerInfo } from "@/stores/session-store";
import type { AudioEngine } from "@/voice/audio-engine-types";
import { REALTIME_VOICE_VAD_CONFIG } from "@/voice/realtime-voice-config";
import { SpeechSegmenter } from "@/voice/speech-segmenter";
const PCM_MIME_TYPE = "audio/pcm;rate=16000;bits=16";
const KEEP_AWAKE_TAG = "paseo:voice";
const THINKING_TONE_REPEAT_GAP_MS = 350;
const DISPLAY_VOLUME_PUBLISH_INTERVAL_MS = 120;
const DISPLAY_VOLUME_CHANGE_EPSILON = 0.02;
const DISPLAY_VOLUME_ATTACK = 0.35;
const DISPLAY_VOLUME_RELEASE = 0.18;
type TurnEventType = Extract<
AgentStreamEventPayload["type"],
"turn_started" | "turn_completed" | "turn_failed" | "turn_canceled"
>;
export type VoiceRuntimePhase =
| "disabled"
| "starting"
| "listening"
| "capturing"
| "submitting"
| "waiting"
| "playing"
| "stopping";
export interface VoiceRuntimeSnapshot {
phase: VoiceRuntimePhase;
isVoiceMode: boolean;
isVoiceSwitching: boolean;
isMuted: boolean;
activeServerId: string | null;
activeAgentId: string | null;
}
export interface VoiceRuntimeTelemetrySnapshot {
volume: number;
isDetecting: boolean;
isSpeaking: boolean;
segmentDuration: number;
}
export interface VoiceSessionAdapter {
serverId: string;
setVoiceMode(enabled: boolean, agentId?: string): Promise<void>;
sendVoiceAudioChunk(
audioData: string,
mimeType: string,
isLast: boolean
): Promise<void>;
abortRequest(): Promise<void>;
setAssistantAudioPlaying(isPlaying: boolean): void;
}
export interface VoiceRuntimeDeps {
engine: AudioEngine;
getServerInfo(serverId: string): DaemonServerInfo | null;
activateKeepAwake(tag: string): Promise<void>;
deactivateKeepAwake(tag: string): Promise<void>;
}
interface RuntimeSessionState {
adapter: VoiceSessionAdapter;
connected: boolean;
}
interface RuntimeState {
snapshot: VoiceRuntimeSnapshot;
telemetry: VoiceRuntimeTelemetrySnapshot;
turnInProgress: boolean;
transportReady: boolean;
generation: number;
speechInterruptTimer: ReturnType<typeof setTimeout> | null;
speechInterruptSent: boolean;
segmentDurationTimer: ReturnType<typeof setInterval> | null;
lastDisplayVolumePublishMs: number;
}
const INITIAL_SNAPSHOT: VoiceRuntimeSnapshot = {
phase: "disabled",
isVoiceMode: false,
isVoiceSwitching: false,
isMuted: false,
activeServerId: null,
activeAgentId: null,
};
const INITIAL_TELEMETRY: VoiceRuntimeTelemetrySnapshot = {
volume: 0,
isDetecting: false,
isSpeaking: false,
segmentDuration: 0,
};
function snapshotsEqual(
left: VoiceRuntimeSnapshot,
right: VoiceRuntimeSnapshot
): boolean {
return (
left.phase === right.phase &&
left.isVoiceMode === right.isVoiceMode &&
left.isVoiceSwitching === right.isVoiceSwitching &&
left.isMuted === right.isMuted &&
left.activeServerId === right.activeServerId &&
left.activeAgentId === right.activeAgentId
);
}
function telemetryEqual(
left: VoiceRuntimeTelemetrySnapshot,
right: VoiceRuntimeTelemetrySnapshot
): boolean {
return (
left.volume === right.volume &&
left.isDetecting === right.isDetecting &&
left.isSpeaking === right.isSpeaking &&
left.segmentDuration === right.segmentDuration
);
}
export interface VoiceRuntime {
subscribe(listener: () => void): () => void;
getSnapshot(): VoiceRuntimeSnapshot;
subscribeTelemetry(listener: () => void): () => void;
getTelemetrySnapshot(): VoiceRuntimeTelemetrySnapshot;
registerSession(adapter: VoiceSessionAdapter): () => void;
updateSessionConnection(serverId: string, connected: boolean): void;
handleCapturePcm(chunk: Uint8Array): void;
handleCaptureVolume(level: number): void;
startVoice(serverId: string, agentId: string): Promise<void>;
stopVoice(): Promise<void>;
destroy(): Promise<void>;
toggleMute(): void;
isVoiceModeForAgent(serverId: string, agentId: string): boolean;
shouldPlayVoiceAudio(serverId: string): boolean;
onAssistantAudioStarted(serverId: string): void;
onAssistantAudioFinished(serverId: string): void;
onTranscriptionResult(serverId: string, text: string): void;
onTurnEvent(serverId: string, agentId: string, eventType: TurnEventType): void;
}
export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
const listeners = new Set<() => void>();
const telemetryListeners = new Set<() => void>();
const sessions = new Map<string, RuntimeSessionState>();
const state: RuntimeState = {
snapshot: INITIAL_SNAPSHOT,
telemetry: INITIAL_TELEMETRY,
turnInProgress: false,
transportReady: false,
generation: 0,
speechInterruptTimer: null,
speechInterruptSent: false,
segmentDurationTimer: null,
lastDisplayVolumePublishMs: 0,
};
function emit(): void {
for (const listener of listeners) {
listener();
}
}
function emitTelemetry(): void {
for (const listener of telemetryListeners) {
listener();
}
}
function patchSnapshot(
patch:
| Partial<VoiceRuntimeSnapshot>
| ((previous: VoiceRuntimeSnapshot) => VoiceRuntimeSnapshot)
): void {
const next =
typeof patch === "function"
? patch(state.snapshot)
: { ...state.snapshot, ...patch };
if (snapshotsEqual(next, state.snapshot)) {
return;
}
const previous = state.snapshot;
state.snapshot = next;
emit();
}
function patchTelemetry(
patch:
| Partial<VoiceRuntimeTelemetrySnapshot>
| ((
previous: VoiceRuntimeTelemetrySnapshot
) => VoiceRuntimeTelemetrySnapshot)
): void {
const next =
typeof patch === "function"
? patch(state.telemetry)
: { ...state.telemetry, ...patch };
if (telemetryEqual(next, state.telemetry)) {
return;
}
state.telemetry = next;
emitTelemetry();
}
function getActiveSession(): RuntimeSessionState | null {
if (!state.snapshot.activeServerId) {
return null;
}
return sessions.get(state.snapshot.activeServerId) ?? null;
}
function clearSpeechInterruptTimer(): void {
if (state.speechInterruptTimer) {
clearTimeout(state.speechInterruptTimer);
state.speechInterruptTimer = null;
}
}
function clearSegmentDurationTimer(): void {
if (state.segmentDurationTimer) {
clearInterval(state.segmentDurationTimer);
state.segmentDurationTimer = null;
}
}
function reconcileSegmentDurationTimer(
segmenter: SpeechSegmenter
): void {
if (!state.telemetry.isDetecting && !state.telemetry.isSpeaking) {
clearSegmentDurationTimer();
patchTelemetry((prev) => ({ ...prev, segmentDuration: 0 }));
return;
}
if (state.segmentDurationTimer) {
return;
}
state.segmentDurationTimer = setInterval(() => {
const startedAt = segmenter.getSpeechDetectionStartMs();
patchTelemetry((prev) => ({
...prev,
segmentDuration: startedAt ? Date.now() - startedAt : 0,
}));
}, 100);
}
function canPlayCue(): boolean {
return (
state.snapshot.isVoiceMode &&
state.snapshot.phase === "waiting" &&
!state.telemetry.isDetecting &&
!state.telemetry.isSpeaking
);
}
function stopCue(): void {
deps.engine.stopLooping();
}
function resetSegmenter(segmenter: SpeechSegmenter): void {
segmenter.reset();
clearSpeechInterruptTimer();
clearSegmentDurationTimer();
patchTelemetry({ ...INITIAL_TELEMETRY });
}
function reconcileCue(): void {
if (!canPlayCue()) {
stopCue();
return;
}
deps.engine.playLooping(new Uint8Array(0), THINKING_TONE_REPEAT_GAP_MS);
}
async function interruptActiveTurn(): Promise<void> {
const activeSession = getActiveSession();
if (!activeSession) {
return;
}
stopCue();
deps.engine.stop();
deps.engine.clearQueue();
activeSession.adapter.setAssistantAudioPlaying(false);
patchSnapshot((prev) => ({ ...prev, phase: "capturing" }));
try {
await activeSession.adapter.abortRequest();
} catch (error) {
console.error("[VoiceRuntime] Failed to abort active turn:", error);
}
}
const segmenter = new SpeechSegmenter(
{
enableContinuousStreaming: false,
volumeThreshold: REALTIME_VOICE_VAD_CONFIG.volumeThreshold,
confirmedDropGracePeriodMs:
REALTIME_VOICE_VAD_CONFIG.confirmedDropGracePeriodMs,
silenceDurationMs: REALTIME_VOICE_VAD_CONFIG.silenceDurationMs,
speechConfirmationMs: REALTIME_VOICE_VAD_CONFIG.speechConfirmationMs,
detectionGracePeriodMs: REALTIME_VOICE_VAD_CONFIG.detectionGracePeriodMs,
},
{
onAudioSegment: ({ audioData, isLast }) => {
const activeSession = getActiveSession();
if (!activeSession || !state.transportReady || !state.snapshot.isVoiceMode) {
return;
}
const generation = state.generation;
patchSnapshot((prev) => ({
...prev,
phase: isLast ? "submitting" : "capturing",
}));
if (isLast) {
state.turnInProgress = true;
}
void activeSession.adapter
.sendVoiceAudioChunk(audioData, PCM_MIME_TYPE, isLast)
.then(() => {
if (
!isLast ||
generation !== state.generation ||
!state.snapshot.isVoiceMode ||
state.snapshot.activeServerId !== activeSession.adapter.serverId ||
state.snapshot.phase !== "submitting"
) {
return;
}
patchSnapshot((prev) => ({ ...prev, phase: "waiting" }));
reconcileCue();
})
.catch((error) => {
console.error("[VoiceRuntime] Failed to send audio segment:", error);
});
},
onSpeechStart: () => {
stopCue();
},
onSpeechEnd: () => {
clearSpeechInterruptTimer();
state.speechInterruptSent = false;
reconcileCue();
},
onDetectingChange: (isDetecting) => {
const previous = state.telemetry;
patchTelemetry((prev) => ({ ...prev, isDetecting }));
if (previous.isDetecting !== isDetecting) {
}
if (!previous.isDetecting && isDetecting) {
stopCue();
}
reconcileSegmentDurationTimer(segmenter);
reconcileCue();
},
onSpeakingChange: (isSpeaking) => {
const previous = state.telemetry;
patchTelemetry((prev) => ({ ...prev, isSpeaking }));
if (previous.isSpeaking !== isSpeaking) {
}
if (!previous.isSpeaking && isSpeaking) {
stopCue();
if (
state.snapshot.phase === "waiting" ||
state.snapshot.phase === "playing"
) {
clearSpeechInterruptTimer();
state.speechInterruptTimer = setTimeout(() => {
state.speechInterruptTimer = null;
if (
state.speechInterruptSent ||
!state.snapshot.isVoiceMode ||
!state.telemetry.isSpeaking
) {
return;
}
state.speechInterruptSent = true;
void interruptActiveTurn();
}, REALTIME_VOICE_VAD_CONFIG.interruptGracePeriodMs);
}
}
if (!isSpeaking) {
clearSpeechInterruptTimer();
state.speechInterruptSent = false;
}
reconcileSegmentDurationTimer(segmenter);
reconcileCue();
},
}
);
function resetToDisabledState(): void {
state.transportReady = false;
state.turnInProgress = false;
state.speechInterruptSent = false;
state.lastDisplayVolumePublishMs = 0;
resetSegmenter(segmenter);
patchSnapshot({ ...INITIAL_SNAPSHOT });
}
function publishDisplayVolume(level: number, nowMs: number): void {
const previousVolume = state.telemetry.volume;
const smoothing =
level >= previousVolume ? DISPLAY_VOLUME_ATTACK : DISPLAY_VOLUME_RELEASE;
const nextVolume = Math.max(
0,
Math.min(1, previousVolume + (level - previousVolume) * smoothing)
);
const enoughTimeElapsed =
nowMs - state.lastDisplayVolumePublishMs >=
DISPLAY_VOLUME_PUBLISH_INTERVAL_MS;
const enoughChange =
Math.abs(nextVolume - previousVolume) >= DISPLAY_VOLUME_CHANGE_EPSILON;
if (!enoughTimeElapsed && !enoughChange) {
return;
}
state.lastDisplayVolumePublishMs = nowMs;
patchTelemetry((prev) => ({
...prev,
volume: Number(nextVolume.toFixed(3)),
}));
}
async function performLocalStop(): Promise<void> {
stopCue();
deps.engine.stop();
deps.engine.clearQueue();
await deps.engine.stopCapture().catch(() => undefined);
await deps.deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
getActiveSession()?.adapter.setAssistantAudioPlaying(false);
resetToDisabledState();
}
async function resyncVoiceMode(serverId: string): Promise<void> {
if (
!state.snapshot.isVoiceMode ||
state.snapshot.activeServerId !== serverId ||
!state.snapshot.activeAgentId
) {
return;
}
const activeSession = getActiveSession();
if (!activeSession || !activeSession.connected) {
return;
}
patchSnapshot((prev) => ({ ...prev, isVoiceSwitching: true }));
try {
await activeSession.adapter.setVoiceMode(true, state.snapshot.activeAgentId);
state.transportReady = true;
} finally {
patchSnapshot((prev) => ({ ...prev, isVoiceSwitching: false }));
}
}
return {
subscribe(listener) {
listeners.add(listener);
return () => {
listeners.delete(listener);
};
},
getSnapshot() {
return state.snapshot;
},
subscribeTelemetry(listener) {
telemetryListeners.add(listener);
return () => {
telemetryListeners.delete(listener);
};
},
getTelemetrySnapshot() {
return state.telemetry;
},
registerSession(adapter) {
sessions.set(adapter.serverId, {
adapter,
connected: true,
});
return () => {
const activeServerId = state.snapshot.activeServerId;
sessions.delete(adapter.serverId);
if (activeServerId === adapter.serverId) {
void performLocalStop();
}
};
},
updateSessionConnection(serverId, connected) {
const session = sessions.get(serverId);
if (!session) {
return;
}
session.connected = connected;
if (state.snapshot.activeServerId !== serverId) {
return;
}
if (!connected) {
state.transportReady = false;
return;
}
void resyncVoiceMode(serverId);
},
handleCapturePcm(chunk) {
if (!state.snapshot.isVoiceMode || state.snapshot.isMuted) {
return;
}
segmenter.pushPcmChunk(chunk);
},
handleCaptureVolume(level) {
const nowMs = Date.now();
const displayLevel = state.snapshot.isMuted ? 0 : level;
publishDisplayVolume(displayLevel, nowMs);
if (!state.snapshot.isVoiceMode || state.snapshot.isMuted) {
return;
}
segmenter.pushVolumeLevel(level, nowMs);
},
async startVoice(serverId, agentId) {
const session = sessions.get(serverId);
if (!session) {
throw new Error(`Voice runtime is not ready for host ${serverId}`);
}
if (!session.connected) {
throw new Error(`Host ${serverId} is not connected`);
}
const serverInfo = deps.getServerInfo(serverId);
const unavailableMessage = resolveVoiceUnavailableMessage({
serverInfo,
mode: "voice",
});
if (unavailableMessage) {
throw new Error(unavailableMessage);
}
const previousServerId = state.snapshot.activeServerId;
const previousAgentId = state.snapshot.activeAgentId;
const generation = state.generation + 1;
state.generation = generation;
state.transportReady = false;
patchSnapshot((prev) => ({
...prev,
isVoiceSwitching: true,
phase: "starting",
activeServerId: serverId,
activeAgentId: agentId,
}));
try {
if (
state.snapshot.isVoiceMode &&
previousServerId &&
(previousServerId !== serverId || previousAgentId !== agentId)
) {
const previousSession = sessions.get(previousServerId);
if (previousSession) {
previousSession.adapter.setAssistantAudioPlaying(false);
await previousSession.adapter.setVoiceMode(false);
}
}
await deps.activateKeepAwake(KEEP_AWAKE_TAG).catch((error) => {
console.warn("[VoiceRuntime] Failed to activate keep-awake:", error);
});
await deps.engine.initialize();
await session.adapter.setVoiceMode(true, agentId);
await deps.engine.startCapture();
if (state.generation !== generation) {
return;
}
state.transportReady = true;
state.turnInProgress = false;
resetSegmenter(segmenter);
patchSnapshot((prev) => ({
...prev,
isVoiceMode: true,
isVoiceSwitching: false,
phase: "listening",
isMuted: deps.engine.isMuted(),
}));
} catch (error) {
await performLocalStop();
throw error;
}
},
async stopVoice() {
const activeSession = getActiveSession();
const generation = state.generation + 1;
state.generation = generation;
patchSnapshot((prev) => ({
...prev,
isVoiceSwitching: true,
phase: "stopping",
}));
try {
state.transportReady = false;
stopCue();
deps.engine.stop();
deps.engine.clearQueue();
activeSession?.adapter.setAssistantAudioPlaying(false);
if (activeSession) {
await activeSession.adapter.setVoiceMode(false);
}
await deps.engine.stopCapture();
await deps.deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
} finally {
if (state.generation === generation) {
resetToDisabledState();
}
}
},
async destroy() {
await this.stopVoice().catch(() => undefined);
await deps.engine.destroy();
listeners.clear();
telemetryListeners.clear();
sessions.clear();
},
toggleMute() {
const nextMuted = deps.engine.toggleMute();
if (nextMuted) {
resetSegmenter(segmenter);
patchSnapshot((prev) => ({
...prev,
isMuted: true,
}));
reconcileCue();
return;
}
patchSnapshot((prev) => ({ ...prev, isMuted: false }));
},
isVoiceModeForAgent(serverId, agentId) {
return (
state.snapshot.isVoiceMode &&
state.snapshot.activeServerId === serverId &&
state.snapshot.activeAgentId === agentId
);
},
shouldPlayVoiceAudio(serverId) {
return (
state.snapshot.isVoiceMode &&
state.snapshot.activeServerId === serverId &&
state.snapshot.phase !== "stopping" &&
state.snapshot.phase !== "disabled" &&
!state.telemetry.isDetecting &&
!state.telemetry.isSpeaking
);
},
onAssistantAudioStarted(serverId) {
if (
!state.snapshot.isVoiceMode ||
state.snapshot.activeServerId !== serverId
) {
return;
}
stopCue();
getActiveSession()?.adapter.setAssistantAudioPlaying(true);
patchSnapshot((prev) => ({ ...prev, phase: "playing" }));
},
onAssistantAudioFinished(serverId) {
if (state.snapshot.activeServerId !== serverId) {
return;
}
getActiveSession()?.adapter.setAssistantAudioPlaying(false);
if (!state.snapshot.isVoiceMode) {
return;
}
if (state.turnInProgress) {
patchSnapshot((prev) => ({ ...prev, phase: "waiting" }));
reconcileCue();
return;
}
patchSnapshot((prev) => ({ ...prev, phase: "listening" }));
reconcileCue();
},
onTranscriptionResult(serverId, text) {
if (
serverId !== state.snapshot.activeServerId ||
!state.snapshot.isVoiceMode
) {
return;
}
if (text.trim()) {
return;
}
state.turnInProgress = false;
patchSnapshot((prev) => ({ ...prev, phase: "listening" }));
stopCue();
},
onTurnEvent(serverId, agentId, eventType) {
if (
!state.snapshot.isVoiceMode ||
state.snapshot.activeServerId !== serverId ||
state.snapshot.activeAgentId !== agentId
) {
return;
}
if (eventType === "turn_started") {
state.turnInProgress = true;
return;
}
state.turnInProgress = false;
if (state.snapshot.phase !== "playing") {
patchSnapshot((prev) => ({ ...prev, phase: "listening" }));
}
stopCue();
},
};
}

View File

@@ -1,6 +1,6 @@
{
"name": "@getpaseo/cli",
"version": "0.1.26",
"version": "0.1.27",
"description": "Paseo CLI - control your AI coding agents from the command line",
"type": "module",
"files": [
@@ -24,8 +24,8 @@
},
"dependencies": {
"@clack/prompts": "^1.0.0",
"@getpaseo/relay": "0.1.26",
"@getpaseo/server": "0.1.26",
"@getpaseo/relay": "0.1.27",
"@getpaseo/server": "0.1.27",
"chalk": "^5.3.0",
"commander": "^12.0.0",
"mime-types": "^2.1.35",

View File

@@ -1,6 +1,6 @@
{
"name": "@getpaseo/desktop",
"version": "0.1.26",
"version": "0.1.27",
"private": true,
"description": "Paseo desktop app (Tauri wrapper)",
"scripts": {

View File

@@ -288,26 +288,23 @@ async function installPackedWorkspaces(runtimeRoot, bundledNodeRoot, tarballs) {
const npmCli = process.platform === "win32"
? path.join(bundledNodeRoot, "node_modules", "npm", "bin", "npm-cli.js")
: path.join(bundledNodeRoot, "lib", "node_modules", "npm", "bin", "npm-cli.js");
const install = spawnSync(
nodeExecutable,
[
npmCli,
"install",
"--omit=dev",
"--no-package-lock",
"--no-save",
...tarballs,
],
{
cwd: runtimeRoot,
stdio: "inherit",
env: {
...process.env,
npm_config_audit: "false",
npm_config_fund: "false",
},
}
);
const install = spawnSync(nodeExecutable, [
npmCli,
"install",
"--include=optional",
"--omit=dev",
"--no-package-lock",
"--no-save",
...tarballs,
], {
cwd: runtimeRoot,
stdio: "inherit",
env: {
...process.env,
npm_config_audit: "false",
npm_config_fund: "false",
},
});
if (install.status !== 0) {
throw new Error(
`Managed runtime dependency install failed with exit code ${install.status ?? 1}.`
@@ -315,6 +312,67 @@ async function installPackedWorkspaces(runtimeRoot, bundledNodeRoot, tarballs) {
}
}
function resolveSherpaNativePackageName() {
const sherpaPlatformMap = {
darwin: "darwin",
linux: "linux",
win32: "win",
};
const sherpaPlatform = sherpaPlatformMap[process.platform];
if (!sherpaPlatform) {
throw new Error(`Managed runtime sherpa package is not implemented for ${process.platform}.`);
}
return `sherpa-onnx-${sherpaPlatform}-${process.arch}`;
}
async function ensureSherpaNativePackage(runtimeRoot, bundledNodeRoot) {
const packageName = resolveSherpaNativePackageName();
const packageDir = path.join(runtimeRoot, "node_modules", packageName);
if (await pathExists(packageDir)) {
return;
}
const sherpaNodePackageJson = JSON.parse(
await fs.readFile(
path.join(runtimeRoot, "node_modules", "sherpa-onnx-node", "package.json"),
"utf8"
)
);
const packageVersion = sherpaNodePackageJson.version;
if (!packageVersion) {
throw new Error("Missing version in sherpa-onnx-node package.json.");
}
const nodeExecutable = process.platform === "win32"
? path.join(bundledNodeRoot, "node.exe")
: path.join(bundledNodeRoot, "bin", "node");
const npmCli = process.platform === "win32"
? path.join(bundledNodeRoot, "node_modules", "npm", "bin", "npm-cli.js")
: path.join(bundledNodeRoot, "lib", "node_modules", "npm", "bin", "npm-cli.js");
const install = spawnSync(nodeExecutable, [
npmCli,
"install",
"--include=optional",
"--omit=dev",
"--no-package-lock",
"--no-save",
`${packageName}@${packageVersion}`,
], {
cwd: runtimeRoot,
stdio: "inherit",
env: {
...process.env,
npm_config_audit: "false",
npm_config_fund: "false",
},
});
if (install.status !== 0) {
throw new Error(
`Managed runtime sherpa package install failed with exit code ${install.status ?? 1}.`
);
}
}
async function writeRuntimePackageJson(runtimeRoot) {
await fs.writeFile(
path.join(runtimeRoot, "package.json"),
@@ -422,71 +480,12 @@ async function pruneClaudeAgentSdk(runtimeRoot) {
}
}
async function pruneCodexCli(runtimeRoot) {
const codexTargetTriple = {
darwin: { arm64: "aarch64-apple-darwin", x64: "x86_64-apple-darwin" },
linux: { arm64: "aarch64-unknown-linux-musl", x64: "x86_64-unknown-linux-musl" },
win32: { arm64: "aarch64-pc-windows-msvc", x64: "x86_64-pc-windows-msvc" },
};
const codexPlatformPackages = [
"@openai/codex-darwin-arm64",
"@openai/codex-darwin-x64",
"@openai/codex-linux-arm64",
"@openai/codex-linux-x64",
"@openai/codex-win32-arm64",
"@openai/codex-win32-x64",
];
const currentTriple = codexTargetTriple[process.platform]?.[process.arch];
const currentPackage = currentTriple
? `@openai/codex-${process.platform === "win32" ? "win32" : process.platform}-${process.arch}`
: null;
const openaiScope = path.join(runtimeRoot, "node_modules", "@openai");
for (const pkg of codexPlatformPackages) {
const pkgName = pkg.replace("@openai/", "");
if (currentPackage && pkg === currentPackage) {
continue;
}
await removeIfExists(path.join(openaiScope, pkgName));
}
}
async function pruneOpenCodeCli(runtimeRoot) {
const opencodePlatformPackages = [
"opencode-darwin-arm64",
"opencode-darwin-x64",
"opencode-linux-arm64",
"opencode-linux-x64",
"opencode-linux-x64-baseline",
"opencode-linux-arm64-musl",
"opencode-linux-x64-musl",
"opencode-linux-x64-baseline-musl",
"opencode-windows-x64",
"opencode-windows-arm64",
];
const platformMap = { darwin: "darwin", linux: "linux", win32: "windows" };
const currentPlatform = platformMap[process.platform];
const currentPackage = currentPlatform
? `opencode-${currentPlatform}-${process.arch}`
: null;
const nodeModules = path.join(runtimeRoot, "node_modules");
for (const pkg of opencodePlatformPackages) {
if (currentPackage && pkg === currentPackage) {
continue;
}
await removeIfExists(path.join(nodeModules, pkg));
}
}
async function pruneManagedRuntime(runtimeRoot) {
await Promise.all([
pruneNodeDistribution(runtimeRoot),
pruneOnnxRuntime(runtimeRoot),
pruneNodePty(runtimeRoot),
pruneClaudeAgentSdk(runtimeRoot),
pruneCodexCli(runtimeRoot),
pruneOpenCodeCli(runtimeRoot),
]);
}
@@ -528,6 +527,7 @@ async function main() {
path.join(runtimeRoot, "node"),
tarballs.map((entry) => entry.path)
);
await ensureSherpaNativePackage(runtimeRoot, path.join(runtimeRoot, "node"));
await pruneManagedRuntime(runtimeRoot);
const nodeRelativePath = process.platform === "win32"

View File

@@ -2629,7 +2629,7 @@ dependencies = [
[[package]]
name = "paseo"
version = "0.1.26"
version = "0.1.27"
dependencies = [
"base64 0.22.1",
"dirs",
@@ -2637,6 +2637,7 @@ dependencies = [
"futures-util",
"http",
"log",
"mac-notification-sys",
"serde",
"serde_json",
"tauri",

View File

@@ -1,6 +1,6 @@
[package]
name = "paseo"
version = "0.1.26"
version = "0.1.27"
description = "Paseo Desktop"
authors = ["moboudra"]
license = "MIT"
@@ -41,6 +41,9 @@ tauri-plugin-websocket = "2"
tokio = { version = "1", features = ["net", "sync"] }
tokio-tungstenite = "0.24"
[target.'cfg(target_os = "macos")'.dependencies]
mac-notification-sys = "0.6"

View File

@@ -7,9 +7,9 @@
<key>CFBundleName</key>
<string>Paseo</string>
<key>CFBundleShortVersionString</key>
<string>0.1.26</string>
<string>0.1.27</string>
<key>CFBundleVersion</key>
<string>0.1.26</string>
<string>0.1.27</string>
<key>NSMicrophoneUsageDescription</key>
<string>Paseo needs access to your microphone for voice dictation and voice mode.</string>
</dict>

View File

@@ -0,0 +1,110 @@
use serde::{Deserialize, Serialize};
use serde_json::Value;
use tauri::{AppHandle, Runtime};
pub const DESKTOP_NOTIFICATION_CLICK_EVENT: &str = "desktop-notification-click";
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct DesktopNotificationInput {
pub title: String,
pub body: Option<String>,
pub data: Option<Value>,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct DesktopNotificationClickPayload {
data: Option<Value>,
}
#[cfg(target_os = "macos")]
mod platform {
use super::{
AppHandle, DesktopNotificationClickPayload, DesktopNotificationInput,
DESKTOP_NOTIFICATION_CLICK_EVENT,
};
use std::sync::OnceLock;
use tauri::{async_runtime, Emitter, Runtime};
const MACOS_NOTIFICATION_SENDER_BUNDLE_ID: &str = "dev.paseo.desktop";
static MACOS_NOTIFICATION_SENDER_INIT: OnceLock<Result<(), String>> = OnceLock::new();
fn ensure_notification_sender() -> Result<(), String> {
MACOS_NOTIFICATION_SENDER_INIT
.get_or_init(|| {
mac_notification_sys::set_application(MACOS_NOTIFICATION_SENDER_BUNDLE_ID)
.map_err(|error| {
format!(
"Failed to initialize macOS notification sender for {}: {error}",
MACOS_NOTIFICATION_SENDER_BUNDLE_ID
)
})
})
.clone()
}
pub fn send_notification<R: Runtime>(
app: AppHandle<R>,
input: DesktopNotificationInput,
) -> Result<(), String> {
let _task = async_runtime::spawn_blocking(move || {
if let Err(error) = ensure_notification_sender() {
log::warn!("{error}");
return;
}
let mut notification = mac_notification_sys::Notification::default();
notification
.title(&input.title)
.message(input.body.as_deref().unwrap_or(""))
.wait_for_click(true);
match notification.send() {
Ok(mac_notification_sys::NotificationResponse::Click) => {
let payload = DesktopNotificationClickPayload { data: input.data };
if let Err(error) = app.emit(DESKTOP_NOTIFICATION_CLICK_EVENT, payload) {
log::warn!("failed to emit desktop notification click event: {error}");
}
}
Ok(_) => {}
Err(error) => {
log::warn!("failed to send macOS notification: {error}");
}
}
});
Ok(())
}
}
#[cfg(not(target_os = "macos"))]
mod platform {
use super::{AppHandle, DesktopNotificationInput};
use tauri::Runtime;
use tauri_plugin_notification::NotificationExt;
pub fn send_notification<R: Runtime>(
app: AppHandle<R>,
input: DesktopNotificationInput,
) -> Result<(), String> {
let mut notification = app.notification().builder().title(&input.title);
if let Some(body) = &input.body {
notification = notification.body(body);
}
notification
.show()
.map_err(|error| format!("Failed to send desktop notification: {error}"))?;
Ok(())
}
}
#[tauri::command]
pub async fn send_desktop_notification<R: Runtime>(
app: AppHandle<R>,
input: DesktopNotificationInput,
) -> Result<(), String> {
platform::send_notification(app, input)
}

View File

@@ -11,7 +11,9 @@ use tauri::menu::{Menu, MenuItemBuilder, MenuItemKind, PredefinedMenuItem, Subme
use tauri::{AppHandle, Manager, WebviewWindow};
use tauri_plugin_updater::UpdaterExt;
mod desktop_notifications;
mod runtime_manager;
use desktop_notifications::send_desktop_notification;
use runtime_manager::{
cli_symlink_instructions, close_local_daemon_transport, managed_daemon_logs,
managed_daemon_pairing, managed_daemon_status, managed_runtime_status,
@@ -544,6 +546,7 @@ pub fn run() {
read_file_base64,
delete_attachment_file,
garbage_collect_attachment_files,
send_desktop_notification,
webview_log
])
.setup(|app| {

View File

@@ -821,6 +821,7 @@ pub fn run_managed_cli_from_current_process(args: Vec<String>) -> Result<i32, St
if let Some(contents_dir) = exe_dir.parent() {
resource_dirs.push(contents_dir.join("Resources"));
resource_dirs.push(contents_dir.join("resources"));
resource_dirs.push(contents_dir.join("lib").join("Paseo").join("resources"));
}
let bundled_root = resource_dirs
.into_iter()

View File

@@ -1,7 +1,7 @@
{
"$schema": "../node_modules/@tauri-apps/cli/config.schema.json",
"productName": "Paseo",
"version": "0.1.26",
"version": "0.1.27",
"identifier": "dev.paseo.desktop",
"build": {
"frontendDist": "../../app/dist",

View File

@@ -1,6 +1,6 @@
{
"name": "@getpaseo/relay",
"version": "0.1.26",
"version": "0.1.27",
"description": "Paseo relay for bridging daemon and client connections",
"type": "module",
"publishConfig": {

View File

@@ -1,6 +1,6 @@
{
"name": "@getpaseo/server",
"version": "0.1.26",
"version": "0.1.27",
"description": "Paseo backend server",
"type": "module",
"publishConfig": {
@@ -62,9 +62,8 @@
"dependencies": {
"@ai-sdk/openai": "2.0.52",
"@anthropic-ai/claude-agent-sdk": "^0.2.11",
"@openai/codex": "^0.114.0",
"@deepgram/sdk": "^3.4.0",
"@getpaseo/relay": "0.1.26",
"@getpaseo/relay": "0.1.27",
"@lezer/common": "^1.5.0",
"@lezer/css": "^1.3.0",
"@lezer/highlight": "^1.2.3",
@@ -88,13 +87,13 @@
"node-pty": "1.2.0-beta.11",
"onnxruntime-node": "^1.23.0",
"openai": "^4.20.0",
"opencode-ai": "^1.2.24",
"pino": "^10.2.0",
"pino-pretty": "^13.1.3",
"qrcode": "^1.5.4",
"rotating-file-stream": "^3.2.9",
"sherpa-onnx": "^1.12.23",
"sherpa-onnx-node": "^1.12.23",
"shell-env": "^4.0.3",
"sherpa-onnx": "1.12.28",
"sherpa-onnx-node": "1.12.28",
"strip-ansi": "^7.1.2",
"tiny-invariant": "^1.3.3",
"uuid": "^9.0.1",

View File

@@ -1,6 +1,7 @@
import { execFileSync, execSync } from "node:child_process";
import { existsSync } from "node:fs";
import { platform } from "node:os";
import { shellEnvSync } from "shell-env";
import { z } from "zod";
import type { AgentProvider } from "./agent-sdk-types.js";
@@ -104,11 +105,24 @@ export function resolveProviderCommandPrefix(
};
}
let cachedShellEnv: Record<string, string> | null = null;
export function resolveShellEnv(): Record<string, string> {
if (cachedShellEnv) return cachedShellEnv;
try {
cachedShellEnv = shellEnvSync();
} catch {
cachedShellEnv = { ...process.env } as Record<string, string>;
}
return cachedShellEnv;
}
export function applyProviderEnv(
baseEnv: Record<string, string | undefined>,
runtimeSettings?: ProviderRuntimeSettings
): Record<string, string | undefined> {
return {
...resolveShellEnv(),
...baseEnv,
...(runtimeSettings?.env ?? {}),
};

View File

@@ -96,6 +96,12 @@ const ClaudeToolDetailPass2Schema = z.union([
toolDetailBranchByName("search", ToolSearchInputSchema, z.unknown(), (input) =>
toSearchToolDetail(input)
),
toolDetailBranchByName("Grep", ToolSearchInputSchema, z.unknown(), (input) =>
toSearchToolDetail(input)
),
toolDetailBranchByName("grep", ToolSearchInputSchema, z.unknown(), (input) =>
toSearchToolDetail(input)
),
toolDetailBranchByName("Glob", ToolSearchInputSchema, z.unknown(), (input) =>
toSearchToolDetail(input)
),

View File

@@ -245,6 +245,30 @@ describe("claude tool-call mapper", () => {
});
});
it("maps Grep calls as search detail using pattern input", () => {
const item = expectMapped(
mapClaudeCompletedToolCall({
callId: "claude-grep-1",
name: "Grep",
input: {
pattern: '\\\\\\"cli\\\\\\""',
path: "/Users/moboudra/dev/paseo/packages/desktop/src-tauri/src",
output_mode: "content",
"-n": true,
},
output: { output: "No matches found" },
})
);
expect(item.status).toBe("completed");
expect(item.error).toBeNull();
expect(item.name).toBe("Grep");
expect(item.detail).toEqual({
type: "search",
query: '\\\\\\"cli\\\\\\""',
});
});
it("normalizes claude speak tool names through schema transforms", () => {
const item = expectMapped(
mapClaudeCompletedToolCall({

View File

@@ -75,6 +75,8 @@ const ClaudeSearchToolNameSchema = z.union([
z.literal("WebSearch"),
z.literal("web_search"),
z.literal("search"),
z.literal("Grep"),
z.literal("grep"),
z.literal("Glob"),
z.literal("glob"),
]);

View File

@@ -11,9 +11,12 @@ import {
} from "./codex-app-server-agent.js";
import { createTestLogger } from "../../../test-utils/test-logger.js";
import { agentConfigs } from "../../daemon-e2e/agent-configs.js";
import { AgentManager } from "../agent-manager.js";
import { AgentStorage } from "../agent-storage.js";
import type {
AgentPermissionRequest,
AgentPromptContentBlock,
AgentStreamEvent,
AgentTimelineItem,
} from "../agent-sdk-types.js";
@@ -179,6 +182,22 @@ function readRolloutTurnContextEfforts(rolloutPath: string): string[] {
return efforts;
}
type Deferred<T> = {
promise: Promise<T>;
resolve: (value: T) => void;
reject: (reason?: unknown) => void;
};
function deferred<T>(): Deferred<T> {
let resolve!: (value: T) => void;
let reject!: (reason?: unknown) => void;
const promise = new Promise<T>((res, rej) => {
resolve = res;
reject = rej;
});
return { promise, resolve, reject };
}
describe("Codex app-server provider (integration)", () => {
const logger = createTestLogger();
@@ -1457,6 +1476,134 @@ describe("Codex app-server provider (integration)", () => {
120000
);
test.runIf(isCodexInstalled())(
"replaceAgentRun keeps the replacement Codex stream alive when the previous interrupted turn completes late",
async () => {
const cleanup = useTempCodexSessionDir();
const cwd = tmpCwd("codex-replace-run-");
const storageDir = tmpCwd("codex-replace-run-storage-");
try {
const manager = new AgentManager({
clients: {
codex: new CodexAppServerAgentClient(logger),
},
registry: new AgentStorage(storageDir, logger),
logger,
});
const snapshot = await manager.createAgent({
provider: "codex",
cwd,
modeId: "auto",
model: CODEX_TEST_MODEL,
thinkingOptionId: CODEX_TEST_THINKING_OPTION_ID,
});
const managedAgent = ((manager as any).agents.get(snapshot.id) ?? null) as
| { session?: any }
| null;
const session = managedAgent?.session as
| {
client?: {
request: (method: string, params?: unknown, timeoutMs?: number) => Promise<unknown>;
} | null;
handleNotification?: (method: string, params: unknown) => void;
}
| undefined;
if (!session?.client || !session.handleNotification) {
throw new Error("Codex session internals unavailable for replaceAgentRun regression test");
}
let turnStartCount = 0;
const replacementTurnInjected = deferred<void>();
const originalRequest = session.client.request.bind(session.client);
session.client.request = async (method: string, params?: unknown, timeoutMs?: number) => {
if (method === "turn/start") {
turnStartCount += 1;
if (turnStartCount === 1) {
queueMicrotask(() => {
session.handleNotification?.("turn/started", {
turn: { id: "initial-turn" },
});
});
} else if (turnStartCount === 2) {
queueMicrotask(() => {
session.handleNotification?.("turn/completed", {
turn: { status: "interrupted" },
});
session.handleNotification?.("turn/started", {
turn: { id: "replacement-turn" },
});
session.handleNotification?.("turn/completed", {
turn: { status: "completed" },
});
replacementTurnInjected.resolve(undefined);
});
}
}
return originalRequest(method, params, timeoutMs);
};
const firstRun = manager.streamAgent(
snapshot.id,
"Keep working until you are interrupted."
);
const firstRunReady = deferred<void>();
const firstRunDrain = (async () => {
for await (const event of firstRun) {
if (event.type === "turn_started") {
firstRunReady.resolve(undefined);
}
}
})();
await Promise.race([
firstRunReady.promise,
new Promise<never>((_, reject) =>
setTimeout(
() => reject(new Error("Timed out waiting for initial Codex turn to start")),
15_000
)
),
]);
const replacementEvents: AgentStreamEvent[] = [];
await Promise.race([
(async () => {
for await (const event of manager.replaceAgentRun(
snapshot.id,
"Reply exactly REPLACED and stop."
)) {
replacementEvents.push(event);
}
})(),
new Promise<never>((_, reject) =>
setTimeout(
() =>
reject(
new Error("Timed out waiting for replacement Codex stream to finish")
),
60_000
)
),
]);
await replacementTurnInjected.promise;
await firstRunDrain;
expect(turnStartCount).toBeGreaterThanOrEqual(2);
expect(replacementEvents.some((event) => event.type === "turn_started")).toBe(true);
expect(replacementEvents.some((event) => event.type === "turn_completed")).toBe(true);
} finally {
cleanup();
rmSync(storageDir, { recursive: true, force: true });
rmSync(cwd, { recursive: true, force: true });
}
},
120000
);
test.runIf(isCodexInstalled())(
"persists session metadata and resumes with history",
async () => {

View File

@@ -27,10 +27,8 @@ import type { Logger } from "pino";
import { execSync, spawn } from "node:child_process";
import type { ChildProcessWithoutNullStreams } from "node:child_process";
import { randomUUID } from "node:crypto";
import { existsSync } from "node:fs";
import { existsSync, Dirent } from "node:fs";
import fs from "node:fs/promises";
import { Dirent } from "node:fs";
import { createRequire } from "node:module";
import os from "node:os";
import path from "node:path";
import readline from "node:readline";
@@ -42,9 +40,11 @@ import {
} from "./codex/tool-call-mapper.js";
import {
applyProviderEnv,
findExecutable,
resolveProviderCommandPrefix,
type ProviderRuntimeSettings,
} from "../provider-launch-config.js";
import { extractCodexTerminalSessionId, nonEmptyString } from "./tool-call-mapper-utils.js";
const DEFAULT_TIMEOUT_MS = 14 * 24 * 60 * 60 * 1000;
@@ -150,52 +150,13 @@ function mergeCodexConfiguredDefaults(
};
}
const CODEX_TARGET_TRIPLE: Record<string, Record<string, string>> = {
linux: { x64: "x86_64-unknown-linux-musl", arm64: "aarch64-unknown-linux-musl" },
darwin: { x64: "x86_64-apple-darwin", arm64: "aarch64-apple-darwin" },
win32: { x64: "x86_64-pc-windows-msvc", arm64: "aarch64-pc-windows-msvc" },
};
const CODEX_PLATFORM_PACKAGE: Record<string, string> = {
"x86_64-unknown-linux-musl": "@openai/codex-linux-x64",
"aarch64-unknown-linux-musl": "@openai/codex-linux-arm64",
"x86_64-apple-darwin": "@openai/codex-darwin-x64",
"aarch64-apple-darwin": "@openai/codex-darwin-arm64",
"x86_64-pc-windows-msvc": "@openai/codex-win32-x64",
"aarch64-pc-windows-msvc": "@openai/codex-win32-arm64",
};
function resolveBundledCodexBinary(): string | null {
const targetTriple = CODEX_TARGET_TRIPLE[process.platform]?.[process.arch];
if (!targetTriple) {
return null;
}
const platformPackage = CODEX_PLATFORM_PACKAGE[targetTriple];
if (!platformPackage) {
return null;
}
const binaryName = process.platform === "win32" ? "codex.exe" : "codex";
try {
const require = createRequire(import.meta.url);
const packageJsonPath = require.resolve(`${platformPackage}/package.json`);
const vendorRoot = path.join(path.dirname(packageJsonPath), "vendor");
const binaryPath = path.join(vendorRoot, targetTriple, "codex", binaryName);
if (existsSync(binaryPath)) {
return binaryPath;
}
} catch {
// Platform package not installed
}
return null;
}
function resolveCodexBinary(): string {
const bundled = resolveBundledCodexBinary();
if (bundled) {
return bundled;
const found = findExecutable("codex");
if (found) {
return found;
}
throw new Error(
`Bundled Codex binary not found for platform ${process.platform}-${process.arch}. Ensure the @openai/codex package and its platform-specific optional dependency are installed.`
"Codex binary not found. Install the Codex CLI (https://github.com/openai/codex) and ensure it is available in your shell PATH."
);
}
@@ -1094,6 +1055,32 @@ function mapCodexPatchNotificationToToolCall(params: {
return params.running ? toRunningToolCall(mapped) : mapped;
}
function mapCodexTerminalInteractionToToolCall(params: {
processId?: string | null;
fallbackCallId?: string | null;
command?: string | null;
}): ToolCallTimelineItem {
const processId = nonEmptyString(params.processId ?? undefined);
const callId =
processId
? `terminal-session-${processId}`
: nonEmptyString(params.fallbackCallId ?? undefined) ?? "terminal-interaction";
const label = nonEmptyString(params.command ?? undefined);
return {
type: "tool_call",
callId,
name: "terminal",
status: "completed",
error: null,
detail: {
type: "plain_text",
...(label ? { label } : {}),
icon: "square_terminal",
},
...(processId ? { metadata: { processId } } : {}),
};
}
function threadItemToTimeline(
item: any,
options?: { includeUserMessage?: boolean; cwd?: string | null }
@@ -1318,6 +1305,23 @@ const CodexEventExecCommandOutputDeltaNotificationSchema = z.object({
.passthrough(),
}).passthrough();
const CodexEventTerminalInteractionNotificationSchema = z.object({
msg: z
.object({
type: z.literal("terminal_interaction"),
call_id: z.string().optional(),
process_id: z.union([z.string(), z.number()]).optional(),
stdin: z.string().optional(),
})
.passthrough(),
}).passthrough();
const ItemCommandExecutionTerminalInteractionNotificationSchema = z.object({
itemId: z.string().optional(),
processId: z.union([z.string(), z.number()]).optional(),
stdin: z.string().optional(),
}).passthrough();
const CodexEventPatchApplyBeginNotificationSchema = z.object({
msg: z
.object({
@@ -1398,6 +1402,13 @@ type ParsedCodexNotification =
stream: string | null;
chunk: string | null;
}
| {
kind: "terminal_interaction";
source: "item" | "codex_event";
callId: string | null;
processId: string | null;
stdin: string | null;
}
| {
kind: "patch_apply_started";
callId: string | null;
@@ -1584,6 +1595,45 @@ const CodexNotificationSchema = z.union([
}).transform(
({ method, params }): ParsedCodexNotification => ({ kind: "invalid_payload", method, params })
),
z.object({
method: z.literal("codex/event/terminal_interaction"),
params: CodexEventTerminalInteractionNotificationSchema,
}).transform(
({ params }): ParsedCodexNotification => ({
kind: "terminal_interaction",
source: "codex_event",
callId: params.msg.call_id ?? null,
processId:
typeof params.msg.process_id === "number"
? String(params.msg.process_id)
: params.msg.process_id ?? null,
stdin: params.msg.stdin ?? null,
})
),
z.object({ method: z.literal("codex/event/terminal_interaction"), params: z.unknown() }).transform(
({ method, params }): ParsedCodexNotification => ({ kind: "invalid_payload", method, params })
),
z.object({
method: z.literal("item/commandExecution/terminalInteraction"),
params: ItemCommandExecutionTerminalInteractionNotificationSchema,
}).transform(
({ params }): ParsedCodexNotification => ({
kind: "terminal_interaction",
source: "item",
callId: params.itemId ?? null,
processId:
typeof params.processId === "number"
? String(params.processId)
: params.processId ?? null,
stdin: params.stdin ?? null,
})
),
z.object({
method: z.literal("item/commandExecution/terminalInteraction"),
params: z.unknown(),
}).transform(
({ method, params }): ParsedCodexNotification => ({ kind: "invalid_payload", method, params })
),
z.object({
method: z.literal("codex/event/patch_apply_begin"),
params: CodexEventPatchApplyBeginNotificationSchema,
@@ -1798,6 +1848,8 @@ class CodexAppServerAgentSession implements AgentSession {
private pendingReasoning = new Map<string, string[]>();
private pendingCommandOutputDeltas = new Map<string, string[]>();
private pendingFileChangeOutputDeltas = new Map<string, string[]>();
private terminalCommandByProcessId = new Map<string, string>();
private emittedTerminalInteractionKeys = new Set<string>();
private emittedExecCommandStartedCallIds = new Set<string>();
private emittedExecCommandCompletedCallIds = new Set<string>();
private emittedItemStartedIds = new Set<string>();
@@ -2626,7 +2678,6 @@ class CodexAppServerAgentSession implements AgentSession {
this.pendingCommandOutputDeltas.clear();
this.pendingFileChangeOutputDeltas.clear();
this.warnedIncompleteEditToolCallIds.clear();
this.eventQueue?.end();
return;
}
@@ -2708,11 +2759,13 @@ class CodexAppServerAgentSession implements AgentSession {
this.pendingCommandOutputDeltas,
parsed.callId
);
const resolvedOutput = parsed.output ?? bufferedOutput;
this.rememberTerminalProcessForCommand(parsed.command, resolvedOutput);
const timelineItem = mapCodexExecNotificationToToolCall({
callId: parsed.callId,
command: parsed.command,
cwd: parsed.cwd ?? this.config.cwd ?? null,
output: parsed.output ?? bufferedOutput,
output: resolvedOutput,
exitCode: parsed.exitCode,
success: parsed.success,
stderr: parsed.stderr,
@@ -2725,6 +2778,25 @@ class CodexAppServerAgentSession implements AgentSession {
return;
}
if (parsed.kind === "terminal_interaction") {
const interactionKey = [
parsed.processId ?? "",
parsed.stdin ?? "",
].join("\u0000");
if (!this.shouldEmitTerminalInteractionKey(interactionKey)) {
return;
}
const timelineItem = mapCodexTerminalInteractionToToolCall({
processId: parsed.processId,
fallbackCallId: parsed.callId,
command:
(parsed.processId ? this.terminalCommandByProcessId.get(parsed.processId) : undefined) ??
null,
});
this.emitEvent({ type: "timeline", provider: CODEX_PROVIDER, item: timelineItem });
return;
}
if (parsed.kind === "patch_apply_started") {
if (parsed.callId) {
this.pendingFileChangeOutputDeltas.delete(parsed.callId);
@@ -2917,6 +2989,33 @@ class CodexAppServerAgentSession implements AgentSession {
return buffered.join("");
}
private rememberTerminalProcessForCommand(command: unknown, output: string | null): void {
const normalizedCommand = normalizeCodexCommandValue(command);
if (!normalizedCommand) {
return;
}
const displayCommand =
typeof normalizedCommand === "string"
? normalizedCommand
: normalizedCommand.join(" ").trim();
if (!displayCommand) {
return;
}
const processId = extractCodexTerminalSessionId(output ?? undefined);
if (!processId) {
return;
}
this.terminalCommandByProcessId.set(processId, displayCommand);
}
private shouldEmitTerminalInteractionKey(key: string): boolean {
if (this.emittedTerminalInteractionKeys.has(key)) {
return false;
}
this.emittedTerminalInteractionKeys.add(key);
return true;
}
private warnOnIncompleteEditToolCall(
item: ToolCallTimelineItem,
source: string,

Some files were not shown because too many files have changed in this diff Show More