diff --git a/docs/floating-panels.md b/docs/floating-panels.md index 39ef49fae8..8b81ca0ad9 100644 --- a/docs/floating-panels.md +++ b/docs/floating-panels.md @@ -115,11 +115,10 @@ quietly relying on: Gate `visible` on a screen-focus signal. For panes inside `agent-panel`, the `isPaneFocused` prop already exists and flips on pane switches; pass `visible={isYourOwnVisible && isPaneFocused}`. -- **Transforms.** `KeyboardShiftProvider` owns the canonical keyboard shift - SharedValue, and `useKeyboardShiftStyle()` only adapts that value into - translate/padding styles. The composer and chat content must both read that - provider-owned value. A portal'd popover is outside the composer tree — it - does not get that transform unless you apply it yourself. +- **Transforms.** The composer moves with the keyboard through a + `KeyboardTranslateView` transform. A portal'd popover is outside that view, so + it does not move with the composer unless it renders its own + `KeyboardTranslateView` with the same translation. - **Layering.** The default root host renders after app content, so it sits above compact sidebars. Content overlays that must sit below sidebars should use the current `FloatingPanelPortalHost`. @@ -158,7 +157,7 @@ and the stream keeps its existing flick-to-dismiss gesture. The dock's private viewport and capacity model are extracted from chat. The app-wide `@/keyboard/shift` module owns shared motion for the dock, portals, streams, terminal, explorer, and dialogs. Import only its public entry point: the provider, -shift hooks, shift style, translate view, settled stream inset, and explorer padding +shift hooks, shift style, translate view, keyboard anchor, settled stream inset, and explorer padding policy. Its contexts and reconciliation stay private. Keep `KeyboardShiftProvider` at its root mount point outside the portal host when changing dock hosts. @@ -256,11 +255,27 @@ portal geometry, plus the `isMoving` worklet value. It reconciles native animation-end events because the controller can retain a nonzero value after the keyboard closes. -Measure portal anchors while the dock is at rest. If a popover opens during -motion, re-measure when `isMoving` settles. `measureInWindow` includes the -dock's current layout and transform. Snapshot the shift at measurement time and -apply only the subsequent shift delta to the animated `bottom`. Adding the full -shift moves the portal twice and can place it over the composer controls. +Anchor a portal to content inside a `KeyboardTranslateView` with +`useKeyboardAnchor()`. Read it on the anchor's side of the Portal: the +translation scope is React context and does not cross it (Gotcha 7). It returns: + +- `measureAtRest(element)`: the element's window rect with the keyboard + translation left out, or null once the anchor unmounts. Lay the popover out + from this rect. It assumes the translate view's parent does not move with the + keyboard, which holds for the composer dock. +- `translation`: spread it onto a `KeyboardTranslateView` inside the Portal, so + the popover moves with the anchor on the same driver, including centered + clearance and disabled translation. +- `reservedLift`: the anchor's lift at keyboard boundaries. Bound the popover's + height with it; never animate height per frame. + +Do not measure an anchor with `measureInWindow` and correct it by the keyboard +shift. On iOS the translate view uses the native driver, and `measureInWindow` +includes its transform only when the view happened to re-render since the +keyboard moved. Both results were observed, so any correction double-lifts the +popover or leaves it behind the keyboard. `measureAtRest` measures through +Fabric's transform-free `measureLayout` instead. Re-measure when `isMoving` +settles; the composer's resting position changes as the draft grows. The provider also reconciles iOS from the controller's native `onEnd` event. The controller's stock iOS shared values update at move start and during an @@ -273,7 +288,8 @@ not be required to repair the offset. The generic anchored-overlay rule is: -1. Measure the anchor with `measureInWindow`. +1. Measure the anchor with `measureInWindow`, or with `measureAtRest` when it + moves with the keyboard (Gotcha 3). 2. Measure the Portal host with `measureFloatingPanelPortalHost(hostName)`. 3. Position with anchor coordinates relative to the host: @@ -405,8 +421,8 @@ Before you write a new one, ask: 3. **Is the panel rendered in a Portal host?** Measure the host too. Never use raw window coordinates as local Portal coordinates. 4. **Does the panel sit above something that moves with the keyboard?** If - yes, slave a Reanimated transform to the same SharedValue (Gotcha 3). - If no, you can probably skip the transform entirely. + yes, anchor it with `useKeyboardAnchor()` and wrap it in a + `KeyboardTranslateView` with the anchor's translation (Gotcha 3). 5. **Will the panel's content height vary?** If yes, you need both `anchorRect` and `contentSize` for positioning → apply Gotcha 5 (return null until anchor, then opacity-0 until contentSize). If no — content has diff --git a/docs/mobile-testing.md b/docs/mobile-testing.md index 375da30708..917ebf9b53 100644 --- a/docs/mobile-testing.md +++ b/docs/mobile-testing.md @@ -53,7 +53,18 @@ hold-to-delete and background-tap dismissal scenario on chat, a workspace draft tab, and New workspace. It checks empty-content taps and header taps at the cap without changing the input height. Each host also opens command/file autocomplete, the model selector, the attachment -menu, and the forge picker from both keyboard states. It then submits a long +menu, and the forge picker from both keyboard states. Autocomplete opens with +the keyboard closed, before the keyboard opens, and after it settles. Each time +the expected best match must be drawn whole between the header and the composer, +above the IME frame, and tapping it there must produce the exact composer text. +Accessibility reports rows behind the keyboard too, so +`packages/app/e2e/mobile/composer-keyboard/autocomplete-placement.mjs` compares +bounds. Android leaves the chat and composer out of the accessibility tree while +suggestions are open, so the composer bounds come from a snapshot with an empty +input in the same keyboard state. The queries are partial names with known best +matches: in chat `/mo` finds the mock provider's `/mock` command, in drafts +`/release` finds this repository's `release-beta` skill for the first Claude model, +and `@app` finds `packages/app`. It then submits a long fixture from each host and checks native stream displacement twice: after the response settles, and after returning to the bottom and letting the scrollbar fade. The image comparison excludes the scrollbar; keyboard dismissal alone @@ -64,6 +75,25 @@ Metro, and device through the `PASEO_COMPOSER_KEYBOARD_*` variables in `packages/app/e2e/mobile/composer-keyboard/android.sh`. Use a software keyboard; the headless input helper cannot verify the [visible composer constraints](floating-panels.md#gotcha-3--keyboard-layout-and-portal-anchors). +To rerun only the autocomplete, model selector, attachment menu, and forge picker +checks on each host, skipping the chat baseline and the growth, dismissal, and +stream scenarios: + +```bash +PASEO_COMPOSER_KEYBOARD_SCENARIO=overlays npm run test:e2e:composer-keyboard:android +``` + +`node packages/app/e2e/mobile/composer-keyboard/ios.mjs` runs the same +autocomplete checks on iOS for chat, a workspace draft tab, and New workspace, +with commands, files, and files under a grown draft. Suggestions open while the +keyboard opens (`fill` taps and types at once), after it settles, with it closed, +and before it reopens. `test:e2e:mobile` cannot host this: its `.ad` flows assert +accessibility state and cannot compare bounds. Run it where `agent-device` +reaches the simulator, with the development app connected to the fixture host +and its Claude provider available. Set `PASEO_COMPOSER_KEYBOARD_SERVER_ID`, +`_AGENT_ID` (an idle mock chat), and `_WORKSPACE_ID` (its workspace); `_DEVICE`, +`_SESSION`, and `_DRAFT_ID` are optional. Artifacts go to +`.dev/agent-device-artifacts/composer-keyboard-ios`. The Android sidebar scroll-retention flow requires a connected host with enough workspaces to overflow the compact sidebar. Pass `TOP_WORKSPACE` and diff --git a/packages/app/e2e/mobile/composer-keyboard/android.sh b/packages/app/e2e/mobile/composer-keyboard/android.sh index 49c907dba9..809483ceff 100755 --- a/packages/app/e2e/mobile/composer-keyboard/android.sh +++ b/packages/app/e2e/mobile/composer-keyboard/android.sh @@ -19,6 +19,13 @@ MESSAGE=$'keyboard invariant line one\nline two\nline three\nline four' LONG_MESSAGE="$(node -e 'process.stdout.write(Array.from({ length: 180 }, (_, index) => `line${index + 1}`).join(" "))')" BLANK_LINE_DRAFT="$(node -e 'process.stdout.write("\n".repeat(22) + "ddjdj")')" AGENT_TITLE="Keyboard dismiss QA $(date +%s)" +# `overlays` runs only each host's autocomplete, model selector, attachment menu, and +# forge picker checks; `all` also runs the chat baseline and the full host scenario. +SCENARIO="${PASEO_COMPOSER_KEYBOARD_SCENARIO:-all}" +if [[ "${SCENARIO}" != all && "${SCENARIO}" != overlays ]]; then + echo "PASEO_COMPOSER_KEYBOARD_SCENARIO must be all or overlays" >&2 + exit 1 +fi if [[ -z "${SERVER_ID}" ]]; then if [[ ! -f "${DAEMON_HOME}/server-id" ]]; then @@ -173,8 +180,10 @@ sleep 3 # Create the fixture while the client is connected so its directory replica observes the insert. ad wait 'id="message-input-root"' 90000 +# The compact header names the workspace, not the agent, so the chat waits below find +# the fixture by its prompt bubble. run_json="$(env -u PASEO_CALLER_AGENT_ID -u PASEO_AGENT_ID -u PASEO_WORKSPACE_ID \ - npm run --silent cli -- run "Exercise the composer keyboard invariant flow" \ + npm run --silent cli -- run "${AGENT_TITLE}" \ --background \ --title "${AGENT_TITLE}" \ --provider mock \ @@ -206,238 +215,241 @@ clear_input ad keyboard dismiss || true ad wait 'label="Message, @files, /commands"' 10000 -snapshot_json "${ARTIFACTS_DIR}/baseline.json" -capture_screen "${ARTIFACTS_DIR}/baseline.png" -read -r input_x input_y baseline_input_height < <( - node "${ASSERT}" rect "${ARTIFACTS_DIR}/baseline.json" "Message, @files, /commands" -) -read -r model_x model_y _ < <( - node "${ASSERT}" rect "${ARTIFACTS_DIR}/baseline.json" "combined-model-selector" -) -read -r attach_x attach_y _ < <( - node "${ASSERT}" rect "${ARTIFACTS_DIR}/baseline.json" "message-input-attach-button" -) -read -r _ header_y _ < <( - node "${ASSERT}" rect "${ARTIFACTS_DIR}/baseline.json" "workspace-tab-switcher-trigger" -) +# Chat-only invariants, then the long-draft checks on New workspace. +run_chat_baseline() { + snapshot_json "${ARTIFACTS_DIR}/baseline.json" + capture_screen "${ARTIFACTS_DIR}/baseline.png" + read -r input_x input_y baseline_input_height < <( + node "${ASSERT}" rect "${ARTIFACTS_DIR}/baseline.json" "Message, @files, /commands" + ) + read -r model_x model_y _ < <( + node "${ASSERT}" rect "${ARTIFACTS_DIR}/baseline.json" "combined-model-selector" + ) + read -r attach_x attach_y _ < <( + node "${ASSERT}" rect "${ARTIFACTS_DIR}/baseline.json" "message-input-attach-button" + ) + read -r _ header_y _ < <( + node "${ASSERT}" rect "${ARTIFACTS_DIR}/baseline.json" "workspace-tab-switcher-trigger" + ) -adb shell input keyevent HOME -adb shell am start -n "${APP_ID}/.MainActivity" >/dev/null -sleep 1 -open_gboard "${input_x}" "${input_y}" -clear_focused_input_with_adb -keyboard_shift="$(read_keyboard_shift)" -adb shell input keyevent 76 -sleep 1 -capture_screen "${ARTIFACTS_DIR}/command-popover.png" -ad wait 'id="composer-autocomplete-popover"' 10000 -ad wait 'text="/exit"' 10000 -model_open_y="$((model_y - keyboard_shift))" -adb shell input tap "${model_x}" "${model_open_y}" -sleep 1 -capture_screen "${ARTIFACTS_DIR}/model-selector-open.png" -adb shell ime set "${HELPER_IME}" >/dev/null -ad wait 'id="compact-provider-list"' 10000 -ad press 'label="Close"' --settle -adb shell input tap "${input_x}" "${input_y}" -clear_focused_input_with_adb -ad wait 'editable=true' 10000 + adb shell input keyevent HOME + adb shell am start -n "${APP_ID}/.MainActivity" >/dev/null + sleep 1 + open_gboard "${input_x}" "${input_y}" + clear_focused_input_with_adb + keyboard_shift="$(read_keyboard_shift)" + adb shell input keyevent 76 + sleep 1 + capture_screen "${ARTIFACTS_DIR}/command-popover.png" + ad wait 'id="composer-autocomplete-popover"' 10000 + ad wait 'text="/exit"' 10000 + model_open_y="$((model_y - keyboard_shift))" + adb shell input tap "${model_x}" "${model_open_y}" + sleep 1 + capture_screen "${ARTIFACTS_DIR}/model-selector-open.png" + adb shell ime set "${HELPER_IME}" >/dev/null + ad wait 'id="compact-provider-list"' 10000 + ad press 'label="Close"' --settle + adb shell input tap "${input_x}" "${input_y}" + clear_focused_input_with_adb + ad wait 'editable=true' 10000 -adb shell input keyevent HOME -adb shell am start -n "${APP_ID}/.MainActivity" >/dev/null -sleep 1 -open_gboard "${input_x}" "${input_y}" -clear_focused_input_with_adb -adb shell input keyevent 76 -sleep 1 -adb shell ime set "${HELPER_IME}" >/dev/null -ad wait 'id="composer-autocomplete-popover"' 10000 -ad wait 'text="/exit"' 10000 -snapshot_json "${ARTIFACTS_DIR}/command-popover.json" -node "${ASSERT}" above-y \ - "${ARTIFACTS_DIR}/command-popover.json" \ - "/exit, Archive the current agent" \ - "$((input_y - baseline_input_height / 2))" -adb shell input keyevent 67 -ad wait 'label="Message, @files, /commands"' 10000 + adb shell input keyevent HOME + adb shell am start -n "${APP_ID}/.MainActivity" >/dev/null + sleep 1 + open_gboard "${input_x}" "${input_y}" + clear_focused_input_with_adb + adb shell input keyevent 76 + sleep 1 + adb shell ime set "${HELPER_IME}" >/dev/null + ad wait 'id="composer-autocomplete-popover"' 10000 + ad wait 'text="/exit"' 10000 + snapshot_json "${ARTIFACTS_DIR}/command-popover.json" + node "${ASSERT}" above-y \ + "${ARTIFACTS_DIR}/command-popover.json" \ + "/exit, Archive the current agent" \ + "$((input_y - baseline_input_height / 2))" + adb shell input keyevent 67 + ad wait 'label="Message, @files, /commands"' 10000 -ad fill 'label="Message, @files, /commands"' "${MESSAGE}" --settle -snapshot_json "${ARTIFACTS_DIR}/multiline-first.json" -ad fill 'editable=true focused=true' "${MESSAGE}" --settle -snapshot_json "${ARTIFACTS_DIR}/multiline-second.json" -node "${ASSERT}" same-input-height \ - "${ARTIFACTS_DIR}/multiline-first.json" \ - "${ARTIFACTS_DIR}/multiline-second.json" + ad fill 'label="Message, @files, /commands"' "${MESSAGE}" --settle + snapshot_json "${ARTIFACTS_DIR}/multiline-first.json" + ad fill 'editable=true focused=true' "${MESSAGE}" --settle + snapshot_json "${ARTIFACTS_DIR}/multiline-second.json" + node "${ASSERT}" same-input-height \ + "${ARTIFACTS_DIR}/multiline-first.json" \ + "${ARTIFACTS_DIR}/multiline-second.json" -adb shell input keyevent HOME -adb shell am start -n "${APP_ID}/.MainActivity" >/dev/null -sleep 1 -open_gboard "${input_x}" "${input_y}" + adb shell input keyevent HOME + adb shell am start -n "${APP_ID}/.MainActivity" >/dev/null + sleep 1 + open_gboard "${input_x}" "${input_y}" -keyboard_shift="$(read_keyboard_shift)" -capture_screen "${ARTIFACTS_DIR}/keyboard-open.png" + keyboard_shift="$(read_keyboard_shift)" + capture_screen "${ARTIFACTS_DIR}/keyboard-open.png" -node "${ASSERT}" same-header \ - "${ARTIFACTS_DIR}/baseline.png" \ - "${ARTIFACTS_DIR}/keyboard-open.png" \ - "$((header_y + 48))" + node "${ASSERT}" same-header \ + "${ARTIFACTS_DIR}/baseline.png" \ + "${ARTIFACTS_DIR}/keyboard-open.png" \ + "$((header_y + 48))" -adb shell ime set "${HELPER_IME}" >/dev/null -ad fill 'editable=true focused=true' "${LONG_MESSAGE}" --settle -open_gboard "${input_x}" "${input_y}" -capture_ui_xml "${ARTIFACTS_DIR}/long-draft-keyboard-open.xml" -capture_screen "${ARTIFACTS_DIR}/long-draft-keyboard-open.png" -node "${ASSERT}" xml-above-y \ - "${ARTIFACTS_DIR}/long-draft-keyboard-open.xml" \ - "Add attachment" \ - "$(read_ime_top)" -display_density="$(adb shell wm density | awk '/density:/ { density = $NF } END { print density }')" -node "${ASSERT}" xml-composer-contained \ - "${ARTIFACTS_DIR}/long-draft-keyboard-open.xml" \ - "$(read_ime_top)" \ - "${display_density}" -# Emptying a capped blank-line draft with Gboard's hold-to-delete must return the input to -# its baseline height. Fabric only re-measures the Android input when its text prop changes, -# so the editing input renders again after every edit -# (packages/app/src/components/ui/text-input/text-input.native.tsx). The sequence mirrors the -# report: grow to the cap, close and reopen the keyboard, hold delete from the middle of the -# draft, jump to the end, hold delete again. The first hold-delete must start mid-draft: from -# the end, the pre-fix input also returns to baseline. Backspace sits on Gboard's third key -# row on the paseo-api35 layout. -adb shell ime set "${HELPER_IME}" >/dev/null -ad fill 'editable=true' "${BLANK_LINE_DRAFT}" --settle -open_gboard "${input_x}" "${input_y}" -adb shell input keyevent BACK -wait_for_ime false -open_gboard "${input_x}" "${input_y}" -adb shell input keycombination 113 122 -adb shell input keyevent 20 20 20 20 20 20 20 20 -screen_width="$(adb shell wm size | sed -n 's/.*: \([0-9]*\)x.*/\1/p' | tail -1)" -backspace_x="$((screen_width - 85))" -backspace_y="$(($(read_ime_top) + 507))" -adb shell input swipe "${backspace_x}" "${backspace_y}" "${backspace_x}" "${backspace_y}" 8000 -adb shell input keycombination 113 123 -adb shell input swipe "${backspace_x}" "${backspace_y}" "${backspace_x}" "${backspace_y}" 12000 -adb shell ime set "${HELPER_IME}" >/dev/null -ad wait 'label="Message, @files, /commands"' 10000 -snapshot_json "${ARTIFACTS_DIR}/after-hold-delete.json" -node "${ASSERT}" same-input-height \ - "${ARTIFACTS_DIR}/baseline.json" \ - "${ARTIFACTS_DIR}/after-hold-delete.json" + adb shell ime set "${HELPER_IME}" >/dev/null + ad fill 'editable=true focused=true' "${LONG_MESSAGE}" --settle + open_gboard "${input_x}" "${input_y}" + capture_ui_xml "${ARTIFACTS_DIR}/long-draft-keyboard-open.xml" + capture_screen "${ARTIFACTS_DIR}/long-draft-keyboard-open.png" + node "${ASSERT}" xml-above-y \ + "${ARTIFACTS_DIR}/long-draft-keyboard-open.xml" \ + "Add attachment" \ + "$(read_ime_top)" + display_density="$(adb shell wm density | awk '/density:/ { density = $NF } END { print density }')" + node "${ASSERT}" xml-composer-contained \ + "${ARTIFACTS_DIR}/long-draft-keyboard-open.xml" \ + "$(read_ime_top)" \ + "${display_density}" + # Emptying a capped blank-line draft with Gboard's hold-to-delete must return the input to + # its baseline height. Fabric only re-measures the Android input when its text prop changes, + # so the editing input renders again after every edit + # (packages/app/src/components/ui/text-input/text-input.native.tsx). The sequence mirrors the + # report: grow to the cap, close and reopen the keyboard, hold delete from the middle of the + # draft, jump to the end, hold delete again. The first hold-delete must start mid-draft: from + # the end, the pre-fix input also returns to baseline. Backspace sits on Gboard's third key + # row on the paseo-api35 layout. + adb shell ime set "${HELPER_IME}" >/dev/null + ad fill 'editable=true' "${BLANK_LINE_DRAFT}" --settle + open_gboard "${input_x}" "${input_y}" + adb shell input keyevent BACK + wait_for_ime false + open_gboard "${input_x}" "${input_y}" + adb shell input keycombination 113 122 + adb shell input keyevent 20 20 20 20 20 20 20 20 + screen_width="$(adb shell wm size | sed -n 's/.*: \([0-9]*\)x.*/\1/p' | tail -1)" + backspace_x="$((screen_width - 85))" + backspace_y="$(($(read_ime_top) + 507))" + adb shell input swipe "${backspace_x}" "${backspace_y}" "${backspace_x}" "${backspace_y}" 8000 + adb shell input keycombination 113 123 + adb shell input swipe "${backspace_x}" "${backspace_y}" "${backspace_x}" "${backspace_y}" 12000 + adb shell ime set "${HELPER_IME}" >/dev/null + ad wait 'label="Message, @files, /commands"' 10000 + snapshot_json "${ARTIFACTS_DIR}/after-hold-delete.json" + node "${ASSERT}" same-input-height \ + "${ARTIFACTS_DIR}/baseline.json" \ + "${ARTIFACTS_DIR}/after-hold-delete.json" -adb shell ime set "${HELPER_IME}" >/dev/null -ad fill 'editable=true' "${MESSAGE}" --settle -open_gboard "${input_x}" "${input_y}" -keyboard_shift="$(read_keyboard_shift)" + adb shell ime set "${HELPER_IME}" >/dev/null + ad fill 'editable=true' "${MESSAGE}" --settle + open_gboard "${input_x}" "${input_y}" + keyboard_shift="$(read_keyboard_shift)" -read -r changes_x changes_y _ < <( - node "${ASSERT}" rect "${ARTIFACTS_DIR}/multiline-second.json" "Open Changes tab" -) -adb shell input tap "${changes_x}" "$((changes_y - keyboard_shift))" -ad wait 'id="changes-header"' 10000 -capture_screen "${ARTIFACTS_DIR}/changes-control-open.png" -ad press 'label="Close Explorer sidebar"' --settle -ad wait "text=\"${AGENT_TITLE}\"" 10000 -ad wait 'editable=true' 10000 + read -r changes_x changes_y _ < <( + node "${ASSERT}" rect "${ARTIFACTS_DIR}/multiline-second.json" "Open Changes tab" + ) + adb shell input tap "${changes_x}" "$((changes_y - keyboard_shift))" + ad wait 'id="changes-header"' 10000 + capture_screen "${ARTIFACTS_DIR}/changes-control-open.png" + ad press 'label="Close Explorer sidebar"' --settle + ad wait "text=\"${AGENT_TITLE}\"" 10000 + ad wait 'editable=true' 10000 -open_gboard "${input_x}" "${input_y}" -keyboard_shift="$(read_keyboard_shift)" -adb shell input tap "${attach_x}" "$((attach_y - keyboard_shift))" -sleep 1 -adb shell ime set "${HELPER_IME}" >/dev/null -ad wait 'id="message-input-attachment-menu-content"' 10000 -ad press 540 600 --settle + open_gboard "${input_x}" "${input_y}" + keyboard_shift="$(read_keyboard_shift)" + adb shell input tap "${attach_x}" "$((attach_y - keyboard_shift))" + sleep 1 + adb shell ime set "${HELPER_IME}" >/dev/null + ad wait 'id="message-input-attachment-menu-content"' 10000 + ad press 540 600 --settle -open_gboard "${input_x}" "${input_y}" -keyboard_shift="$(read_keyboard_shift)" -adb shell input tap 983 "$((attach_y - keyboard_shift))" -wait_for_ime false -adb shell ime set "${HELPER_IME}" >/dev/null -ad wait text "keyboard invariant line one" 10000 -snapshot_json "${ARTIFACTS_DIR}/after-submit.json" -node "${ASSERT}" has-exact-text "${ARTIFACTS_DIR}/after-submit.json" "${MESSAGE}" -node "${ASSERT}" same-input-height \ - "${ARTIFACTS_DIR}/baseline.json" \ - "${ARTIFACTS_DIR}/after-submit.json" + open_gboard "${input_x}" "${input_y}" + keyboard_shift="$(read_keyboard_shift)" + adb shell input tap 983 "$((attach_y - keyboard_shift))" + wait_for_ime false + adb shell ime set "${HELPER_IME}" >/dev/null + ad wait text "keyboard invariant line one" 10000 + snapshot_json "${ARTIFACTS_DIR}/after-submit.json" + node "${ASSERT}" has-exact-text "${ARTIFACTS_DIR}/after-submit.json" "${MESSAGE}" + node "${ASSERT}" same-input-height \ + "${ARTIFACTS_DIR}/baseline.json" \ + "${ARTIFACTS_DIR}/after-submit.json" -open_gboard "${input_x}" "${input_y}" -adb shell input keyevent BACK -wait_for_ime false -sleep 0.5 -capture_screen "${ARTIFACTS_DIR}/hidden-focused-baseline.png" + open_gboard "${input_x}" "${input_y}" + adb shell input keyevent BACK + wait_for_ime false + sleep 0.5 + capture_screen "${ARTIFACTS_DIR}/hidden-focused-baseline.png" -open_gboard "${input_x}" "${input_y}" -node "${STALL_HERMES}" "${METRO_PORT}" 3000 "${APP_ID}" & -stall_pid="$!" -sleep 0.25 -adb shell input keyevent BACK -wait_for_ime false -sleep 0.5 -capture_screen "${ARTIFACTS_DIR}/hidden-during-js-stall.png" -wait "${stall_pid}" -capture_screen "${ARTIFACTS_DIR}/hidden-after-js-stall.png" -node "${ASSERT}" same-region \ - "${ARTIFACTS_DIR}/hidden-focused-baseline.png" \ - "${ARTIFACTS_DIR}/hidden-during-js-stall.png" \ - 1400 \ - 430 -node "${ASSERT}" same-region \ - "${ARTIFACTS_DIR}/hidden-focused-baseline.png" \ - "${ARTIFACTS_DIR}/hidden-after-js-stall.png" \ - 1400 \ - 430 + open_gboard "${input_x}" "${input_y}" + node "${STALL_HERMES}" "${METRO_PORT}" 3000 "${APP_ID}" & + stall_pid="$!" + sleep 0.25 + adb shell input keyevent BACK + wait_for_ime false + sleep 0.5 + capture_screen "${ARTIFACTS_DIR}/hidden-during-js-stall.png" + wait "${stall_pid}" + capture_screen "${ARTIFACTS_DIR}/hidden-after-js-stall.png" + node "${ASSERT}" same-region \ + "${ARTIFACTS_DIR}/hidden-focused-baseline.png" \ + "${ARTIFACTS_DIR}/hidden-during-js-stall.png" \ + 1400 \ + 430 + node "${ASSERT}" same-region \ + "${ARTIFACTS_DIR}/hidden-focused-baseline.png" \ + "${ARTIFACTS_DIR}/hidden-after-js-stall.png" \ + 1400 \ + 430 -ad press 'id="menu-button"' --settle -ad press 'id="sidebar-global-new-workspace"' --settle -ad wait 'id="workspace-create-submit"' 10000 -adb shell ime set "${HELPER_IME}" >/dev/null -ad fill 'editable=true' "x" --settle -snapshot_json "${ARTIFACTS_DIR}/new-workspace-short-draft.json" -read -r new_input_x new_input_y _ < <( - node "${ASSERT}" rect "${ARTIFACTS_DIR}/new-workspace-short-draft.json" "editable" -) -open_gboard "${new_input_x}" "${new_input_y}" -capture_ui_xml "${ARTIFACTS_DIR}/new-workspace-short-draft-keyboard-open.xml" -# Typing more lines moves the setup fields up with the composer instead of -# hiding their bottom rows behind it. -adb shell input keycombination 113 123 -adb shell input keyevent 66 66 66 -adb shell input text grown -sleep 1 -capture_ui_xml "${ARTIFACTS_DIR}/new-workspace-grown-draft-keyboard-open.xml" -capture_screen "${ARTIFACTS_DIR}/new-workspace-grown-draft-keyboard-open.png" -node "${ASSERT}" xml-fields-follow-growth \ - "${ARTIFACTS_DIR}/new-workspace-short-draft-keyboard-open.xml" \ - "${ARTIFACTS_DIR}/new-workspace-grown-draft-keyboard-open.xml" + ad press 'id="menu-button"' --settle + ad press 'id="sidebar-global-new-workspace"' --settle + ad wait 'id="workspace-create-submit"' 10000 + adb shell ime set "${HELPER_IME}" >/dev/null + ad fill 'editable=true' "x" --settle + snapshot_json "${ARTIFACTS_DIR}/new-workspace-short-draft.json" + read -r new_input_x new_input_y _ < <( + node "${ASSERT}" rect "${ARTIFACTS_DIR}/new-workspace-short-draft.json" "editable" + ) + open_gboard "${new_input_x}" "${new_input_y}" + capture_ui_xml "${ARTIFACTS_DIR}/new-workspace-short-draft-keyboard-open.xml" + # Typing more lines moves the setup fields up with the composer instead of + # hiding their bottom rows behind it. + adb shell input keycombination 113 123 + adb shell input keyevent 66 66 66 + adb shell input text grown + sleep 1 + capture_ui_xml "${ARTIFACTS_DIR}/new-workspace-grown-draft-keyboard-open.xml" + capture_screen "${ARTIFACTS_DIR}/new-workspace-grown-draft-keyboard-open.png" + node "${ASSERT}" xml-fields-follow-growth \ + "${ARTIFACTS_DIR}/new-workspace-short-draft-keyboard-open.xml" \ + "${ARTIFACTS_DIR}/new-workspace-grown-draft-keyboard-open.xml" -adb shell ime set "${HELPER_IME}" >/dev/null -ad fill 'editable=true' "${LONG_MESSAGE}" --settle -snapshot_json "${ARTIFACTS_DIR}/new-workspace-long-draft.json" -read -r new_input_x new_input_y new_input_height < <( - node "${ASSERT}" rect "${ARTIFACTS_DIR}/new-workspace-long-draft.json" "editable" -) -# The editor shrinks when the keyboard opens. Its bottom stays anchored, while -# its old center can move into the setup fields after the height constraint. -new_input_y="$((new_input_y + new_input_height / 2 - 8))" -open_gboard "${new_input_x}" "${new_input_y}" -capture_ui_xml "${ARTIFACTS_DIR}/new-workspace-long-draft-keyboard-open.xml" -capture_screen "${ARTIFACTS_DIR}/new-workspace-long-draft-keyboard-open.png" -node "${ASSERT}" xml-composer-contained \ - "${ARTIFACTS_DIR}/new-workspace-long-draft-keyboard-open.xml" \ - "$(read_ime_top)" \ - "${display_density}" + adb shell ime set "${HELPER_IME}" >/dev/null + ad fill 'editable=true' "${LONG_MESSAGE}" --settle + snapshot_json "${ARTIFACTS_DIR}/new-workspace-long-draft.json" + read -r new_input_x new_input_y new_input_height < <( + node "${ASSERT}" rect "${ARTIFACTS_DIR}/new-workspace-long-draft.json" "editable" + ) + # The editor shrinks when the keyboard opens. Its bottom stays anchored, while + # its old center can move into the setup fields after the height constraint. + new_input_y="$((new_input_y + new_input_height / 2 - 8))" + open_gboard "${new_input_x}" "${new_input_y}" + capture_ui_xml "${ARTIFACTS_DIR}/new-workspace-long-draft-keyboard-open.xml" + capture_screen "${ARTIFACTS_DIR}/new-workspace-long-draft-keyboard-open.png" + node "${ASSERT}" xml-composer-contained \ + "${ARTIFACTS_DIR}/new-workspace-long-draft-keyboard-open.xml" \ + "$(read_ime_top)" \ + "${display_density}" -# Dismissing the keyboard moves the composer down at its editing height and -# brings the setup fields back fully visible above it. -adb shell input keyevent BACK -wait_for_ime false -capture_ui_xml "${ARTIFACTS_DIR}/new-workspace-long-draft-keyboard-closed.xml" -capture_screen "${ARTIFACTS_DIR}/new-workspace-long-draft-keyboard-closed.png" -node "${ASSERT}" same-input-height \ - "${ARTIFACTS_DIR}/new-workspace-long-draft-keyboard-open.xml" \ - "${ARTIFACTS_DIR}/new-workspace-long-draft-keyboard-closed.xml" -node "${ASSERT}" xml-fields-above-composer \ - "${ARTIFACTS_DIR}/new-workspace-long-draft-keyboard-closed.xml" + # Dismissing the keyboard moves the composer down at its editing height and + # brings the setup fields back fully visible above it. + adb shell input keyevent BACK + wait_for_ime false + capture_ui_xml "${ARTIFACTS_DIR}/new-workspace-long-draft-keyboard-closed.xml" + capture_screen "${ARTIFACTS_DIR}/new-workspace-long-draft-keyboard-closed.png" + node "${ASSERT}" same-input-height \ + "${ARTIFACTS_DIR}/new-workspace-long-draft-keyboard-open.xml" \ + "${ARTIFACTS_DIR}/new-workspace-long-draft-keyboard-closed.xml" + node "${ASSERT}" xml-fields-above-composer \ + "${ARTIFACTS_DIR}/new-workspace-long-draft-keyboard-closed.xml" +} # All hosts run the same scenario; only navigation differs. # For the shared scenario, tap the editor's blank right edge once. Tapping @@ -470,20 +482,80 @@ press_composer_control() { adb shell input tap "${x}" "${y}" } +# Opens suggestions with the keyboard closed, before it opens, or after it settled; +# checks their bounds and taps the best match where it is drawn. Accessibility keeps +# reporting rows behind the keyboard, so existence is not evidence. It also drops the +# composer while suggestions are open, so the composer bounds come from a reference +# snapshot with an empty input in the keyboard state the suggestions are checked in. +check_autocomplete() { + local host="$1" order="$2" kind="$3" query="$4" expected="$5" inserted="$6" + local prefix="${ARTIFACTS_DIR}/${host}-${kind}-${order}" + if [[ "${order}" == keyboard-closed ]]; then + prepare_composer closed + snapshot_json "${prefix}-reference.json" + adb shell input text "${query}" + else + prepare_composer open + # The composer follows the keyboard animation, which ends after the IME frame settles. + sleep 1 + snapshot_json "${prefix}-reference.json" + if [[ "${order}" == before-keyboard ]]; then + adb shell ime set "${HELPER_IME}" >/dev/null + wait_for_ime false + adb shell input text "${query}" + open_editor_keyboard "${overlay_x}" "${overlay_y}" + else + adb shell input text "${query}" + fi + fi + ad wait 'id="composer-autocomplete-popover"' 10000 + # The popover remeasures once keyboard motion settles. + sleep 1 + capture_screen "${prefix}.png" + snapshot_json "${prefix}.json" + local ime_top="" + if [[ "${order}" != keyboard-closed ]]; then ime_top="$(read_ime_top)"; fi + local best_x best_y + read -r best_x best_y < <( + node "${ASSERT}" autocomplete-best "${prefix}.json" "${prefix}-reference.json" "${expected}" "${ime_top}" + ) + adb shell input tap "${best_x}" "${best_y}" + sleep 1 + snapshot_json "${prefix}-selected.json" + node "${ASSERT}" composer-text "${prefix}-selected.json" "${inserted}" + clear_focused_input_with_adb + echo "PASS host=${host}: ${kind} ${order}, ${expected} drawn above composer and keyboard and inserted" +} + +# Drafts list commands for their selected provider, and a fresh install selects none. +select_claude_model() { + local x y h + press_composer_control combined-model-selector + ad wait 'id="agent-controls-model"' 10000 + ad press 'id="agent-controls-model"' --settle + ad press 'id="model-provider-claude"' --settle + snapshot_json "${ARTIFACTS_DIR}/model-list.json" + read -r x y h < <(node "${ASSERT}" rect "${ARTIFACTS_DIR}/model-list.json" 'model-row-claude-*') + adb shell input tap "${x}" "${y}" + ad wait 'id="agent-controls-thinking"' 10000 + ad press 'label="Close"' --settle +} + run_host_overlays() { local host="$1" + if [[ "${host}" != chat ]]; then + prepare_composer closed + select_claude_model + fi + # Chat lists the mock provider's command; drafts list the release skills in + # this repository's .claude/skills for the selected Claude model. + local command_query=/release command_match=/release-beta command_inserted="/release-beta " + if [[ "${host}" == chat ]]; then command_query=/mo command_match=/mock command_inserted="/mock "; fi + for order in keyboard-closed before-keyboard after-keyboard; do + check_autocomplete "${host}" "${order}" commands "${command_query}" "${command_match}" "${command_inserted}" + check_autocomplete "${host}" "${order}" files @app packages/app '"packages/app"' + done for mode in closed open; do - for query in / @README; do - prepare_composer closed - adb shell input text "${query}" - if [[ "${mode}" == open ]]; then open_editor_keyboard "${overlay_x}" "${overlay_y}"; fi - ad wait 'id="composer-autocomplete-popover"' 10000 - local kind=commands - if [[ "${query}" == @* ]]; then kind=files; fi - capture_screen "${ARTIFACTS_DIR}/${host}-${kind}-${mode}.png" - clear_focused_input_with_adb - echo "PASS host=${host}: ${kind} popover keyboard=${mode}" - done prepare_composer "${mode}" press_composer_control combined-model-selector ad wait 'id="compact-provider-list"' 10000 @@ -656,19 +728,29 @@ run_host_scenario() { run_host_scroll "${host}" } +run_host() { + if [[ "${SCENARIO}" == overlays ]]; then + run_host_overlays "$1" + else + run_host_scenario "$1" + fi +} + +if [[ "${SCENARIO}" == all ]]; then run_chat_baseline; fi adb shell am start -a android.intent.action.VIEW -d "paseo://h/${SERVER_ID}/agent/${agent_id}" "${APP_ID}" >/dev/null ad wait "text=\"${AGENT_TITLE}\"" 10000 -run_host_scenario chat +run_host chat ad keyboard dismiss || true ad press 'id="workspace-tab-switcher-trigger"' --settle ad wait 'id="workspace-tab-switcher-new-agent"' 10000 ad press 'id="workspace-tab-switcher-new-agent"' --settle -ad wait 'text="New Agent"' 10000 -run_host_scenario workspace-draft +# Only a draft offers to import a session. +ad wait 'id="composer-import-agent-pill"' 10000 +run_host workspace-draft ad keyboard dismiss || true ad press 'id="menu-button"' --settle ad press 'id="sidebar-global-new-workspace"' --settle ad wait 'id="workspace-create-submit"' 10000 -run_host_scenario new-workspace +run_host new-workspace echo "Composer keyboard invariants passed" diff --git a/packages/app/e2e/mobile/composer-keyboard/assert-composer-keyboard.mjs b/packages/app/e2e/mobile/composer-keyboard/assert-composer-keyboard.mjs index 6bd13a5da5..637a162682 100644 --- a/packages/app/e2e/mobile/composer-keyboard/assert-composer-keyboard.mjs +++ b/packages/app/e2e/mobile/composer-keyboard/assert-composer-keyboard.mjs @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; import process from "node:process"; import sharp from "sharp"; +import { resolveAutocompletePlacement } from "./autocomplete-placement.mjs"; const [, , command, ...args] = process.argv; @@ -115,11 +116,14 @@ if (command === "stream-moved") { `Composer contained: top=${composer.top}, header=${viewport.top}, bottom=${composer.bottom}, keyboard=${imeTopArgument}\n`, ); } else if (command === "rect") { + // A trailing `*` matches the first identifier with that prefix. const [snapshotPath, identifier, edge] = args; const snapshot = JSON.parse(await fs.readFile(snapshotPath, "utf8")); + const prefix = identifier.endsWith("*") ? identifier.slice(0, -1) : null; const node = snapshot.data.nodes.find( (candidate) => candidate.identifier === identifier || + (prefix !== null && candidate.identifier?.startsWith(prefix)) || candidate.label === identifier || (identifier === "editable" && candidate.type.endsWith("EditText")), ); @@ -232,6 +236,27 @@ if (command === "stream-moved") { if (!snapshot.data.nodes.some((candidate) => candidate.label === expected)) { throw new Error(`Missing exact text in ${snapshotPath}: ${expected}`); } +} else if (command === "autocomplete-best") { + // Prints the center of the expected best match after checking its placement + // against the composer in the reference snapshot. + const [snapshotPath, referencePath, expected, imeTopArgument] = args; + const snapshot = JSON.parse(await fs.readFile(snapshotPath, "utf8")); + const reference = JSON.parse(await fs.readFile(referencePath, "utf8")); + const best = resolveAutocompletePlacement(snapshot, { + expected, + keyboardTop: imeTopArgument ? Number(imeTopArgument) : undefined, + reference, + }); + process.stdout.write(`${best.x} ${best.y}\n`); +} else if (command === "composer-text") { + // Accessibility drops a trailing space, so compare without it. + const [snapshotPath, expected] = args; + const snapshot = JSON.parse(await fs.readFile(snapshotPath, "utf8")); + const input = snapshot.data.nodes.find((candidate) => candidate.type.endsWith("EditText")); + const text = input?.value ?? input?.label ?? ""; + if (text.trimEnd() !== expected.trimEnd()) { + throw new Error(`Composer text ${JSON.stringify(text)}, expected ${JSON.stringify(expected)}`); + } } else if (command === "same-region") { const [firstPath, secondPath, topArgument, heightArgument, failureMessage] = args; const crop = { diff --git a/packages/app/e2e/mobile/composer-keyboard/autocomplete-placement.mjs b/packages/app/e2e/mobile/composer-keyboard/autocomplete-placement.mjs new file mode 100644 index 0000000000..22c0b42e8f --- /dev/null +++ b/packages/app/e2e/mobile/composer-keyboard/autocomplete-placement.mjs @@ -0,0 +1,94 @@ +// Shared by the iOS and Android composer keyboard harnesses. Reads an agent-device +// `snapshot --raw --json` result. Accessibility keeps reporting suggestion rows that +// sit behind the keyboard, so existence is not evidence: compare their screen bounds. + +const bottomOf = (rect) => rect.y + rect.height; + +function findById(nodes, identifier) { + const node = nodes.find((candidate) => candidate.identifier === identifier && candidate.rect); + if (!node) throw new Error(`Missing node: ${identifier}`); + return node; +} + +function contains(outer, inner) { + return ( + inner.x >= outer.x - 1 && + inner.y >= outer.y - 1 && + inner.x + inner.width <= outer.x + outer.width + 1 && + bottomOf(inner) <= bottomOf(outer) + 1 + ); +} + +/** + * The best match is the list row nearest the composer. Check that the popover sits + * between the app's header controls and the composer, that the keyboard (when shown) + * is below both, and that the row nearest the composer is fully drawn and is + * `expected`. A clipped best match fails: the nearest whole row is then another one. + * Return where to tap it. Android leaves the composer out of the accessibility tree + * while suggestions are open, so its callers read the composer from `reference`: a + * snapshot in the same keyboard state with an empty input. + */ +export function resolveAutocompletePlacement(snapshot, { expected, keyboardTop, reference }) { + if (!snapshot.success || !snapshot.data?.nodes?.length) throw new Error("Empty device snapshot"); + const nodes = snapshot.data.nodes; + const popover = findById(nodes, "composer-autocomplete-popover").rect; + const composer = findById((reference ?? snapshot).data.nodes, "message-input-root").rect; + // The popover may cover the compact header, never the status bar above it. + const minimumTop = + nodes.find((node) => node.identifier === "menu-button" && node.rect)?.rect.y ?? 0; + // iOS reports the keyboard in the app's tree; Android callers pass the IME frame. + const keyboardY = + keyboardTop ?? nodes.find((node) => node.type === "Keyboard" && node.rect)?.rect.y; + const popoverBottom = bottomOf(popover); + if (popoverBottom > composer.y + 1) { + throw new Error( + `Autocomplete overlaps the composer: popover bottom=${popoverBottom}, composer top=${composer.y}`, + ); + } + if (popover.y < minimumTop - 1) { + throw new Error( + `Autocomplete extends above the header: top=${popover.y}, minimum=${minimumTop}`, + ); + } + if (keyboardY !== undefined && popoverBottom > keyboardY + 1) { + throw new Error( + `Autocomplete sits behind the keyboard: popover bottom=${popoverBottom}, keyboard top=${keyboardY}`, + ); + } + // Rows are the innermost labelled boxes spanning the popover. The list scrolls to + // its end, so the lowest row is the best match nearest the composer. + const candidates = nodes.filter( + (node) => + node.rect && + node.label && + node.identifier !== "composer-autocomplete-popover" && + !/scroll bar/i.test(node.label) && + node.rect.height > 0 && + // The popover border insets rows by a few pixels; their labels are narrower. + node.rect.width >= popover.width * 0.97 && + contains(popover, node.rect), + ); + const area = (rect) => rect.width * rect.height; + const rows = candidates.filter( + (outer) => + !candidates.some( + (inner) => area(inner.rect) < area(outer.rect) - 1 && contains(outer.rect, inner.rect), + ), + ); + if (rows.length === 0) throw new Error("Autocomplete has no visible rows"); + const best = rows.reduce((lowest, row) => + bottomOf(row.rect) > bottomOf(lowest.rect) ? row : lowest, + ); + const label = best.label.split(", ")[0]; + if (label !== expected) { + throw new Error(`Best visible match is ${label}, expected ${expected}`); + } + return { + label, + x: Math.round(best.rect.x + best.rect.width / 2), + y: Math.round(best.rect.y + best.rect.height / 2), + popover, + composer, + keyboardTop: keyboardY ?? null, + }; +} diff --git a/packages/app/e2e/mobile/composer-keyboard/ios.mjs b/packages/app/e2e/mobile/composer-keyboard/ios.mjs new file mode 100644 index 0000000000..e3fc4af072 --- /dev/null +++ b/packages/app/e2e/mobile/composer-keyboard/ios.mjs @@ -0,0 +1,229 @@ +// Composer autocomplete placement on iOS with the software keyboard: chat, workspace +// draft tab, and New workspace. Run where agent-device reaches the simulator, with the +// development app connected to the fixture host. See docs/mobile-testing.md. +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; +import { setTimeout as sleep } from "node:timers/promises"; +import { fileURLToPath } from "node:url"; +import { resolveAutocompletePlacement } from "./autocomplete-placement.mjs"; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../../../.."); +const env = (name, fallback) => process.env[`PASEO_COMPOSER_KEYBOARD_${name}`] ?? fallback; +const serverId = env("SERVER_ID"); +const agentId = env("AGENT_ID"); +const workspaceId = env("WORKSPACE_ID"); +if (!serverId || !agentId || !workspaceId) { + throw new Error( + "Set PASEO_COMPOSER_KEYBOARD_SERVER_ID, _AGENT_ID (an idle chat) and _WORKSPACE_ID (its workspace)", + ); +} +const draftId = env("DRAFT_ID", "composer-keyboard"); +const session = env("SESSION", "composer-keyboard-ios"); +const device = env("DEVICE", "iPhone 16 Pro"); +const artifactsDir = path.join(repoRoot, ".dev/agent-device-artifacts/composer-keyboard-ios"); +const grownPrefix = "grown line one\ngrown line two\ngrown line three\n"; +// Partial queries with known best matches. `inserted` is the composer text after +// tapping the best match. +const fileCase = { text: "@app", expected: "packages/app", inserted: '"packages/app"' }; +// The release skills ship in this repository's .claude/skills. +const skillCase = { text: "/release", expected: "/release-beta", inserted: "/release-beta " }; +// The chat fixture's mock provider lists one command. +const chatCommandCase = { text: "/mo", expected: "/mock", inserted: "/mock " }; + +fs.mkdirSync(artifactsDir, { recursive: true }); + +function ad(...args) { + return execFileSync("agent-device", [...args, "--session", session], { + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }); +} + +function snapshot(name) { + const output = ad("snapshot", "--raw", "--json"); + fs.writeFileSync(path.join(artifactsDir, `${name}.json`), output); + ad("screenshot", path.join(artifactsDir, `${name}.png`)); + return JSON.parse(output); +} + +const nodeById = (snap, id) => snap.data.nodes.find((node) => node.identifier === id && node.rect); +const keyboardShown = (snap) => snap.data.nodes.some((node) => node.type === "Keyboard"); +const composerText = (snap) => + snap.data.nodes.find((node) => node.type === "TextView" && node.rect)?.value ?? ""; + +async function waitFor(name, predicate, timeoutMs = 10_000) { + const deadline = Date.now() + timeoutMs; + for (;;) { + const snap = snapshot(name); + if (predicate(snap)) return snap; + if (Date.now() > deadline) throw new Error(`Timed out waiting for ${name}`); + await sleep(500); + } +} + +// Keyboard motion has settled once two snapshots agree on the composer position. +async function settledPlacement(name, keyboard) { + let previousTop; + return waitFor(name, (snap) => { + const composer = nodeById(snap, "message-input-root"); + if (!nodeById(snap, "composer-autocomplete-popover") || !composer) return false; + if (keyboardShown(snap) !== keyboard) return false; + const stable = composer.rect.y === previousTop; + previousTop = composer.rect.y; + return stable; + }); +} + +function assertPlacement(name, snap, expected) { + const placement = resolveAutocompletePlacement(snap, { expected }); + process.stdout.write( + `PASS ${name}: popover ${Math.round(placement.popover.y)}-${Math.round( + placement.popover.y + placement.popover.height, + )} above composer ${Math.round(placement.composer.y)}, ${expected} drawn\n`, + ); + return placement; +} + +// iOS has no generic keyboard dismissal. Tapping the chat stream dismisses it; a +// draft has no stream, so open and close the model sheet, which takes input focus. +async function dismissKeyboard(snap) { + const composer = nodeById(snap, "message-input-root").rect; + ad("press", "4", String(Math.round(composer.y - 8))); + await sleep(1_000); + if (!keyboardShown(snapshot("dismiss-keyboard"))) return; + const sheetOpen = (sheet) => + sheet.data.nodes.some((node) => node.type === "Button" && node.label === "Close"); + ad("press", 'id="combined-model-selector"'); + await waitFor("dismiss-keyboard-sheet", sheetOpen); + ad("press", 'label="Close"'); + // The input is not hittable until the sheet has gone. + await waitFor("dismiss-keyboard-done", (done) => !sheetOpen(done) && !keyboardShown(done)); +} + +// Tap past the end of the (short) last line so the caret lands after the mention. +// The input's right padding holds the fullscreen toggle; stay clear of it. +function focusAtEnd(snap) { + const input = snap.data.nodes.find((node) => node.type === "TextView" && node.rect)?.rect; + if (!input) throw new Error("Missing composer input"); + ad( + "press", + String(Math.round(input.x + input.width * 0.7)), + String(Math.round(input.y + input.height - 6)), + ); +} + +// `fill` cannot replace a multi-line draft or write an empty one, so delete from the end. +async function clearInput(name) { + focusAtEnd(snapshot(`${name}-before-clear`)); + const keyboard = await waitFor(`${name}-clear`, keyboardShown); + const deleteKey = keyboard.data.nodes.find( + (node) => node.type === "Key" && node.label === "delete" && node.rect, + )?.rect; + if (!deleteKey) throw new Error("Missing keyboard delete key"); + const length = composerText(keyboard).length; + ad( + "press", + String(Math.round(deleteKey.x + deleteKey.width / 2)), + String(Math.round(deleteKey.y + deleteKey.height / 2)), + "--count", + String(length + 2), + "--interval-ms", + "40", + ); +} + +// The accessibility value drops a trailing space. +const sameText = (actual, expected) => actual.trimEnd() === expected.trimEnd(); + +async function runCase(host, kind, { text, expected, inserted }) { + const name = `${host}-${kind}`; + await clearInput(name); + await dismissKeyboard(snapshot(`${name}-empty`)); + // fill taps the input and types at once, so suggestions open while the keyboard opens. + ad("fill", "editable=true", text); + assertPlacement( + `${name} opened while keyboard opens`, + await settledPlacement(`${name}-opening`, true), + expected, + ); + await clearInput(`${name}-settled`); + await waitFor(`${name}-settled-empty`, (snap) => composerText(snap) === ""); + ad("type", text); + assertPlacement( + `${name} opened after keyboard settled`, + await settledPlacement(`${name}-open`, true), + expected, + ); + await dismissKeyboard(snapshot(`${name}-before-close`)); + assertPlacement( + `${name} keyboard closed`, + await settledPlacement(`${name}-closed`, false), + expected, + ); + focusAtEnd(snapshot(`${name}-before-reopen`)); + const reopened = assertPlacement( + `${name} keyboard opened under open suggestions`, + await settledPlacement(`${name}-reopened`, true), + expected, + ); + ad("press", String(reopened.x), String(reopened.y)); + const selected = await waitFor(`${name}-selected`, (snap) => + sameText(composerText(snap), inserted), + ); + if (!keyboardShown(selected)) throw new Error(`Selecting ${expected} closed the keyboard`); + process.stdout.write( + `PASS ${name}: tapped ${expected}, composer now ${JSON.stringify(inserted)}\n`, + ); +} + +// Drafts list commands for their selected provider, and a fresh install selects none. +async function selectClaudeModel(host) { + ad("press", 'id="combined-model-selector"'); + await waitFor(`${host}-model-sheet`, (snap) => Boolean(nodeById(snap, "agent-controls-model"))); + ad("press", 'id="agent-controls-model"'); + await waitFor(`${host}-providers`, (snap) => Boolean(nodeById(snap, "model-provider-claude"))); + ad("press", 'id="model-provider-claude"'); + const models = await waitFor(`${host}-models`, (snap) => + snap.data.nodes.some((node) => node.identifier?.startsWith("model-row-claude-") && node.rect), + ); + const row = models.data.nodes.find( + (node) => node.identifier?.startsWith("model-row-claude-") && node.rect, + ).rect; + ad( + "press", + String(Math.round(row.x + row.width / 2)), + String(Math.round(row.y + row.height / 2)), + ); + await waitFor(`${host}-model-selected`, (snap) => + Boolean(nodeById(snap, "agent-controls-thinking")), + ); + ad("press", 'label="Close"'); + await waitFor(`${host}-model-closed`, (snap) => !nodeById(snap, "agent-controls-model")); +} + +async function runHost(host) { + if (host !== "chat") await selectClaudeModel(host); + await runCase(host, "commands", host === "chat" ? chatCommandCase : skillCase); + await runCase(host, "files", fileCase); + await runCase(host, "files-grown", { + text: `${grownPrefix}${fileCase.text}`, + expected: fileCase.expected, + inserted: `${grownPrefix}${fileCase.inserted}`, + }); + await clearInput(`${host}-done`); + await dismissKeyboard(snapshot(`${host}-done`)); +} + +const hosts = [ + ["chat", `paseo://h/${serverId}/agent/${agentId}`], + ["workspace-draft", `paseo://h/${serverId}/workspace/${workspaceId}?open=draft%3A${draftId}`], + ["new-workspace", `paseo://new?serverId=${serverId}`], +]; +for (const [host, url] of hosts) { + ad("open", url, "--platform", "ios", "--device", device); + await waitFor(host, (snap) => Boolean(nodeById(snap, "message-input-root")), 60_000); + await runHost(host); +} +process.stdout.write(`Composer autocomplete placement passed. Artifacts: ${artifactsDir}\n`); diff --git a/packages/app/src/assistant-file-links/use-file-link.test.tsx b/packages/app/src/assistant-file-links/use-file-link.test.tsx index 386cb429f1..18d2727e82 100644 --- a/packages/app/src/assistant-file-links/use-file-link.test.tsx +++ b/packages/app/src/assistant-file-links/use-file-link.test.tsx @@ -2,9 +2,9 @@ * @vitest-environment jsdom */ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import { act, renderHook, waitFor } from "@testing-library/react"; +import { act, cleanup, renderHook, waitFor } from "@testing-library/react"; import React, { useCallback, useMemo, useState, type ReactNode } from "react"; -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import type { ToastApi } from "@/components/toast-host"; import type { InlinePathTarget } from "./parse"; import { AssistantFileLinkResolverProvider } from "./provider"; @@ -99,6 +99,12 @@ function createWrapper(input: { client: TestClient; openedFiles: OpenedFile[]; t } describe("useFileLink", () => { + // Lookups resolve after a test's last assertion; an unmounted hook cannot re-render + // once jsdom is torn down. + afterEach(() => { + cleanup(); + }); + it("returns the same object across no-op parent rerenders", () => { const getDirectorySuggestions = vi.fn(async () => resolvedSuggestions([])); const queryClient = createQueryClient(); diff --git a/packages/app/src/components/ui/autocomplete-popover.tsx b/packages/app/src/components/ui/autocomplete-popover.tsx index 9b1dcaf23d..fa5e724944 100644 --- a/packages/app/src/components/ui/autocomplete-popover.tsx +++ b/packages/app/src/components/ui/autocomplete-popover.tsx @@ -9,32 +9,20 @@ import { } from "react"; import { View, useWindowDimensions } from "react-native"; import { Portal } from "@gorhom/portal"; -import Animated, { - useAnimatedReaction, - useAnimatedStyle, - useSharedValue, -} from "react-native-reanimated"; +import Animated, { useAnimatedReaction, useAnimatedStyle } from "react-native-reanimated"; import { scheduleOnRN } from "react-native-worklets"; import { useSafeAreaInsets } from "react-native-safe-area-context"; -import { StyleSheet } from "react-native-unistyles"; import { Autocomplete, type AutocompleteOption } from "@/components/ui/autocomplete"; import { measureFloatingPanelPortalHost, useFloatingPanelPortalHostName, } from "@/components/ui/floating-panel-portal"; -import { useKeyboardShift } from "@/keyboard/shift"; +import { KeyboardTranslateView, useKeyboardAnchor, useKeyboardShift } from "@/keyboard/shift"; import { SPACING } from "@/styles/theme"; import { inlineUnistylesStyle } from "@/styles/unistyles-inline-style"; const OFFSET_FROM_ANCHOR = SPACING[3]; -interface Rect { - x: number; - y: number; - width: number; - height: number; -} - interface RelativeAnchorRect { x: number; y: number; @@ -42,14 +30,6 @@ interface RelativeAnchorRect { hostHeight: number; } -function measureElement(element: View): Promise { - return new Promise((resolve) => { - element.measureInWindow((x, y, width, height) => { - resolve({ x, y, width, height }); - }); - }); -} - interface AutocompletePopoverProps { visible: boolean; anchorRef: RefObject; @@ -79,8 +59,9 @@ export function AutocompletePopover({ const windowDimensions = useWindowDimensions(); const safeAreaInsets = useSafeAreaInsets(); const portalHostName = useFloatingPanelPortalHostName(); - const { shift, isMoving } = useKeyboardShift(); - const measuredShift = useSharedValue(0); + const { isMoving } = useKeyboardShift(); + const anchor = useKeyboardAnchor(); + const { measureAtRest } = anchor; const measurementGeneration = useRef(0); const canMeasure = visible && (options.length === 0 || selectedIndex >= 0); @@ -90,20 +71,21 @@ export function AutocompletePopover({ if (!anchorElement) return; const generation = measurementGeneration.current; void Promise.all([ - measureElement(anchorElement), + measureAtRest(anchorElement), measureFloatingPanelPortalHost(portalHostName), ]).then(([anchorRect, hostRect]) => { - if (generation !== measurementGeneration.current || !hostRect) return undefined; + if (generation !== measurementGeneration.current || !anchorRect || !hostRect) { + return undefined; + } setRelativeAnchorRect({ x: anchorRect.x - hostRect.x, y: anchorRect.y - hostRect.y, width: anchorRect.width, hostHeight: hostRect.height, }); - measuredShift.value = shift.value; return undefined; }); - }, [anchorRef, canMeasure, measuredShift, portalHostName, shift]); + }, [anchorRef, canMeasure, measureAtRest, portalHostName]); useEffect(() => { measurementGeneration.current += 1; @@ -131,37 +113,38 @@ export function AutocompletePopover({ [isMoving, remeasure], ); + // Lay out at the anchor's resting position; the KeyboardTranslateView below lifts + // it with the anchor. Height changes only at keyboard boundaries. const baseStyle = useMemo(() => { if (!relativeAnchorRect) return null; return inlineUnistylesStyle({ position: "absolute" as const, left: relativeAnchorRect.x, width: relativeAnchorRect.width, + bottom: relativeAnchorRect.hostHeight - relativeAnchorRect.y + OFFSET_FROM_ANCHOR, }); }, [relativeAnchorRect]); const anchorY = relativeAnchorRect?.y ?? 0; - const baseBottom = relativeAnchorRect - ? relativeAnchorRect.hostHeight - relativeAnchorRect.y + OFFSET_FROM_ANCHOR - : 0; - const keyboardLayoutStyle = useAnimatedStyle(() => { - const shiftDelta = shift.value - measuredShift.value; - return { - bottom: baseBottom + shiftDelta, - maxHeight: Math.max(0, anchorY - shiftDelta - safeAreaInsets.top - OFFSET_FROM_ANCHOR * 2), - }; - }, [anchorY, baseBottom, safeAreaInsets.top]); + const reservedLift = anchor.reservedLift; + const heightStyle = useAnimatedStyle( + () => ({ + maxHeight: Math.max( + 0, + anchorY - reservedLift.value - safeAreaInsets.top - OFFSET_FROM_ANCHOR * 2, + ), + }), + [anchorY, reservedLift, safeAreaInsets.top], + ); if (!visible || !relativeAnchorRect || !baseStyle) return null; if (options.length > 0 && selectedIndex < 0) return null; + // The translate view is the popover's own frame, so it lifts only the popover. return ( - - + + - + ); } - -const styles = StyleSheet.create(() => ({ - overlay: { - position: "absolute", - top: 0, - right: 0, - bottom: 0, - left: 0, - }, -})); diff --git a/packages/app/src/keyboard/shift/index.ts b/packages/app/src/keyboard/shift/index.ts index 5f057d4ecd..e54fc7ace9 100644 --- a/packages/app/src/keyboard/shift/index.ts +++ b/packages/app/src/keyboard/shift/index.ts @@ -1,4 +1,5 @@ export { KeyboardShiftProvider, useKeyboardShiftStyle } from "./internal/provider"; export { useKeyboardShift, useSettledKeyboardShift } from "./internal/context"; export { KeyboardTranslateView } from "./internal/translate"; +export { useKeyboardAnchor } from "./internal/anchor"; export { useKeyboardStreamInset } from "./internal/stream"; diff --git a/packages/app/src/keyboard/shift/internal/anchor.ts b/packages/app/src/keyboard/shift/internal/anchor.ts new file mode 100644 index 0000000000..ff34216682 --- /dev/null +++ b/packages/app/src/keyboard/shift/internal/anchor.ts @@ -0,0 +1,31 @@ +import { useContext, useMemo } from "react"; +import { useDerivedValue, type DerivedValue } from "react-native-reanimated"; +import { + KeyboardTranslationContext, + useKeyboardShift, + type KeyboardTranslationScope, +} from "./context"; +import { resolveKeyboardLift } from "./policy"; + +export interface KeyboardAnchor extends KeyboardTranslationScope { + /** The anchor's lift at keyboard boundaries. Bound overlay height with this, never per frame. */ + reservedLift: DerivedValue; +} + +/** + * Position portal content against an element inside a `KeyboardTranslateView`. + * Lay the content out from `measureAtRest` and pass `translation` to a portal'd + * `KeyboardTranslateView`: the content then rides the anchor's own animation + * driver, frame for frame. + */ +export function useKeyboardAnchor(): KeyboardAnchor { + const { translation, measureAtRest } = useContext(KeyboardTranslationContext); + const { layoutShift } = useKeyboardShift(); + const reservedLift = useDerivedValue(() => + resolveKeyboardLift({ shift: layoutShift.value, ...translation }), + ); + return useMemo( + () => ({ measureAtRest, translation, reservedLift }), + [measureAtRest, reservedLift, translation], + ); +} diff --git a/packages/app/src/keyboard/shift/internal/context.ts b/packages/app/src/keyboard/shift/internal/context.ts index dcb0b4cf64..f010c01255 100644 --- a/packages/app/src/keyboard/shift/internal/context.ts +++ b/packages/app/src/keyboard/shift/internal/context.ts @@ -1,5 +1,7 @@ import { createContext, useContext } from "react"; +import type { View } from "react-native"; import type { SharedValue } from "react-native-reanimated"; +import { measureInWindow } from "./measure"; export interface KeyboardShiftContextValue { shift: SharedValue; @@ -10,6 +12,35 @@ export interface KeyboardShiftContextValue { } export const KeyboardShiftContext = createContext(null); + +/** How the nearest KeyboardTranslateView moves its descendants with the keyboard. */ +export interface KeyboardTranslation { + enabled: boolean; + bottomClearance: number; +} + +export interface WindowRect { + x: number; + y: number; + width: number; + height: number; +} + +/** What the nearest KeyboardTranslateView tells the content it moves. */ +export interface KeyboardTranslationScope { + translation: KeyboardTranslation; + /** + * The element's window rect with the keyboard closed, whatever the keyboard is doing + * now; null once the element or the translate view has unmounted. + */ + measureAtRest: (element: View) => Promise; +} + +/** Views outside a KeyboardTranslateView stay put. */ +export const KeyboardTranslationContext = createContext({ + translation: { enabled: false, bottomClearance: 0 }, + measureAtRest: measureInWindow, +}); export const SettledKeyboardShiftContext = createContext(null); /** Read the app-wide keyboard inset without loading its native provider implementation. */ diff --git a/packages/app/src/keyboard/shift/internal/measure.ts b/packages/app/src/keyboard/shift/internal/measure.ts new file mode 100644 index 0000000000..95b07e4ae8 --- /dev/null +++ b/packages/app/src/keyboard/shift/internal/measure.ts @@ -0,0 +1,60 @@ +import { useMemo, type RefObject } from "react"; +import type { View } from "react-native"; +import type { KeyboardTranslation, KeyboardTranslationScope, WindowRect } from "./context"; + +export function measureInWindow(element: View): Promise { + return new Promise((resolve) => { + element.measureInWindow((x, y, width, height) => resolve({ x, y, width, height })); + }); +} + +/** + * The element's window rect with `frame`'s own transform left out. Whether + * `measureInWindow` includes a transform depends on the driver: Reanimated commits + * it to the shadow tree, RN's native driver commits it only when the view happens + * to re-render. Fabric's `measureLayout` and a view's `measure` origin ignore + * transforms. The frame's parent must not move with the keyboard; the composer dock + * is the only anchor host, and its parent is the stationary dock viewport. + */ +function measureInUntranslatedWindow(frame: View, element: View): Promise { + const parent = frame.parentElement; + if (!parent) return Promise.resolve(null); + const parentRect = parent.getBoundingClientRect(); + const frameOrigin = new Promise<{ x: number; y: number }>((resolve) => { + frame.measure((x, y) => resolve({ x, y })); + }); + const offset = new Promise((resolve) => { + element.measureLayout( + frame, + (x, y, width, height) => resolve({ x, y, width, height }), + () => resolve(null), + ); + }); + return Promise.all([frameOrigin, offset]).then(([origin, rect]) => + rect + ? { + x: parentRect.x + origin.x + rect.x, + y: parentRect.y + origin.y + rect.y, + width: rect.width, + height: rect.height, + } + : null, + ); +} + +/** What a KeyboardTranslateView publishes to the content it moves. */ +export function useKeyboardTranslationScope( + frameRef: RefObject, + translation: KeyboardTranslation, +): KeyboardTranslationScope { + return useMemo( + () => ({ + translation, + measureAtRest: (element: View) => + frameRef.current + ? measureInUntranslatedWindow(frameRef.current, element) + : Promise.resolve(null), + }), + [frameRef, translation], + ); +} diff --git a/packages/app/src/keyboard/shift/internal/policy.test.ts b/packages/app/src/keyboard/shift/internal/policy.test.ts index 7c4902afe1..6867f3ffa1 100644 --- a/packages/app/src/keyboard/shift/internal/policy.test.ts +++ b/packages/app/src/keyboard/shift/internal/policy.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { resolveStreamKeyboardInset, shouldReconcileHiddenKeyboardEnd, + resolveKeyboardLift, resolveKeyboardShift, reserveKeyboardLayoutShift, } from "./policy"; @@ -89,6 +90,21 @@ describe("resolveKeyboardShift", () => { }); }); +describe("resolveKeyboardLift", () => { + it("lifts a docked view by the whole shift", () => { + expect(resolveKeyboardLift({ shift: 302, enabled: true, bottomClearance: 0 })).toBe(302); + }); + + it("lets the keyboard fill a centered form's clearance before lifting it", () => { + expect(resolveKeyboardLift({ shift: 403, enabled: true, bottomClearance: 267 })).toBe(136); + expect(resolveKeyboardLift({ shift: 403, enabled: true, bottomClearance: 455 })).toBe(0); + }); + + it("keeps a disabled view in place", () => { + expect(resolveKeyboardLift({ shift: 302, enabled: false, bottomClearance: 0 })).toBe(0); + }); +}); + describe("shouldReconcileHiddenKeyboardEnd", () => { it("closes stale iOS keyboard state without letting a late visible end resurrect it", () => { expect( diff --git a/packages/app/src/keyboard/shift/internal/policy.ts b/packages/app/src/keyboard/shift/internal/policy.ts index fe496fd110..4e618c8a49 100644 --- a/packages/app/src/keyboard/shift/internal/policy.ts +++ b/packages/app/src/keyboard/shift/internal/policy.ts @@ -52,6 +52,16 @@ export function resolveKeyboardShift(input: { return Math.max(0, input.rawKeyboardHeight - input.bottomInset); } +/** Distance a KeyboardTranslateView lifts its content for a given keyboard shift. */ +export function resolveKeyboardLift(input: { + shift: number; + enabled: boolean; + bottomClearance: number; +}): number { + "worklet"; + return input.enabled ? Math.max(0, input.shift - input.bottomClearance) : 0; +} + export function shouldReconcileHiddenKeyboardEnd(input: { height: number; progress: number; diff --git a/packages/app/src/keyboard/shift/internal/translate.android.tsx b/packages/app/src/keyboard/shift/internal/translate.android.tsx index 2a9e9d581d..cec2f0fcfd 100644 --- a/packages/app/src/keyboard/shift/internal/translate.android.tsx +++ b/packages/app/src/keyboard/shift/internal/translate.android.tsx @@ -1,7 +1,9 @@ -import type { ReactNode } from "react"; -import type { ViewProps } from "react-native"; +import { useMemo, useRef, type ReactNode } from "react"; +import type { View, ViewProps } from "react-native"; import Animated, { useAnimatedStyle } from "react-native-reanimated"; -import { useKeyboardShift } from "./context"; +import { KeyboardTranslationContext, useKeyboardShift } from "./context"; +import { useKeyboardTranslationScope } from "./measure"; +import { resolveKeyboardLift } from "./policy"; interface KeyboardTranslateViewProps extends ViewProps { children: ReactNode; @@ -18,13 +20,18 @@ export function KeyboardTranslateView({ ...props }: KeyboardTranslateViewProps) { const { shift } = useKeyboardShift(); + const frameRef = useRef(null); + const translation = useMemo(() => ({ enabled, bottomClearance }), [bottomClearance, enabled]); + const scope = useKeyboardTranslationScope(frameRef, translation); const keyboardStyle = useAnimatedStyle(() => ({ - transform: [{ translateY: enabled ? -Math.max(0, shift.value - bottomClearance) : 0 }], + transform: [{ translateY: -resolveKeyboardLift({ shift: shift.value, ...translation }) }], })); return ( - - {children} + + + {children} + ); } diff --git a/packages/app/src/keyboard/shift/internal/translate.ios.tsx b/packages/app/src/keyboard/shift/internal/translate.ios.tsx index 1fea4fab0c..8628ba8b8c 100644 --- a/packages/app/src/keyboard/shift/internal/translate.ios.tsx +++ b/packages/app/src/keyboard/shift/internal/translate.ios.tsx @@ -1,7 +1,9 @@ -import { useMemo, useEffect, type ReactNode } from "react"; -import { Animated, useAnimatedValue, type ViewProps } from "react-native"; +import { useMemo, useEffect, useRef, type ReactNode } from "react"; +import { Animated, useAnimatedValue, type View, type ViewProps } from "react-native"; import { useKeyboardAnimation } from "react-native-keyboard-controller"; import { useSafeAreaInsets } from "react-native-safe-area-context"; +import { KeyboardTranslationContext } from "./context"; +import { useKeyboardTranslationScope } from "./measure"; interface KeyboardTranslateViewProps extends ViewProps { children: ReactNode; @@ -35,7 +37,10 @@ export function KeyboardTranslateView({ // Fullscreen resizes this view while disabling translation. iOS Fabric can // reapply its previous transform during that layout commit. Keep the graph // attached and apply the mode on the native animation frame; a static zero or - // setValue during the React commit gets overwritten by the resize. + // setValue during the React commit gets overwritten by the resize. A rebuilt + // graph (a new bottomClearance after rotation) starts from JS-side keyboard + // values, so run it on the native frame too or it stays stale until the + // keyboard next moves. const enabledValue = useAnimatedValue(enabled ? 1 : 0); useEffect(() => { const animation = Animated.timing(enabledValue, { @@ -45,15 +50,21 @@ export function KeyboardTranslateView({ }); animation.start(); return () => animation.stop(); - }, [enabled, enabledValue]); + }, [enabled, enabledValue, translateY]); const keyboardStyle = useMemo( () => ({ transform: [{ translateY: Animated.multiply(translateY, enabledValue) }] }), [enabledValue, translateY], ); + const frameRef = useRef(null); + const translation = useMemo(() => ({ enabled, bottomClearance }), [bottomClearance, enabled]); + const scope = useKeyboardTranslationScope(frameRef, translation); + return ( - - {children} + + + {children} + ); } diff --git a/packages/server/src/server/agent/mcp-server.test.ts b/packages/server/src/server/agent/mcp-server.test.ts index 0599281f15..2d066efdc9 100644 --- a/packages/server/src/server/agent/mcp-server.test.ts +++ b/packages/server/src/server/agent/mcp-server.test.ts @@ -1375,17 +1375,17 @@ describe("terminal MCP tools", () => { output += message.data; } }); + // Every key travels through the PTY and back; on loaded Windows runners one round + // trip can take seconds, so each wait gets the same budget as the first. + const roundTrip = { timeout: 5000 }; try { - await vi.waitFor( - async () => { - const capture = await terminalManager.captureTerminal(terminal.id); - expect(capture.lines).toContain("READY"); - }, - { timeout: 5000 }, - ); + await vi.waitFor(async () => { + const capture = await terminalManager.captureTerminal(terminal.id); + expect(capture.lines).toContain("READY"); + }, roundTrip); const tool = registeredTool(server, "send_terminal_keys"); await tool.handler({ terminalId: terminal.id, keys: "application", literal: true }); - await vi.waitFor(() => expect(output).toContain("APPLICATION_READY")); + await vi.waitFor(() => expect(output).toContain("APPLICATION_READY"), roundTrip); for (const [keys, hex] of [ ["Down", "1b4f42"], ["Up", "1b4f41"], @@ -1400,15 +1400,15 @@ describe("terminal MCP tools", () => { await vi.waitFor(async () => { const capture = await terminalManager.captureTerminal(terminal.id); expect(capture.lines.join("\n")).toContain(`HEX:${hex}`); - }); + }, roundTrip); } await tool.handler({ terminalId: terminal.id, keys: "normal", literal: true }); - await vi.waitFor(() => expect(output).toContain("NORMAL_READY")); + await vi.waitFor(() => expect(output).toContain("NORMAL_READY"), roundTrip); await tool.handler({ terminalId: terminal.id, keys: "Down" }); await vi.waitFor(async () => { const capture = await terminalManager.captureTerminal(terminal.id); expect(capture.lines).toContain("HEX:1b5b42"); - }); + }, roundTrip); } finally { unsubscribe(); await server.close(); diff --git a/packages/server/src/server/agent/providers/mock-load-test-agent.test.ts b/packages/server/src/server/agent/providers/mock-load-test-agent.test.ts index c9c0f07161..da10deb869 100644 --- a/packages/server/src/server/agent/providers/mock-load-test-agent.test.ts +++ b/packages/server/src/server/agent/providers/mock-load-test-agent.test.ts @@ -8,6 +8,7 @@ import { AgentManager } from "../agent-manager.js"; import type { AgentStreamEvent, AgentTimelineItem } from "../agent-sdk-types.js"; import { MOCK_LOAD_TEST_DEFAULT_MODEL_ID, + MOCK_LOAD_TEST_HANDLED_COMMAND, MockLoadTestAgentClient, } from "./mock-load-test-agent.js"; @@ -49,6 +50,18 @@ describe("MockLoadTestAgentClient", () => { }); }); + test("lists the command it handles without starting a turn", async () => { + const client = new MockLoadTestAgentClient(); + const session = await client.createSession({ provider: "mock", cwd: process.cwd() }); + + const commands = (await session.listCommands?.()) ?? []; + + expect(commands.map((command) => `/${command.name} ${command.argumentHint}`)).toEqual([ + MOCK_LOAD_TEST_HANDLED_COMMAND, + ]); + expect(session.tryHandleOutOfBand?.(MOCK_LOAD_TEST_HANDLED_COMMAND)).not.toBeNull(); + }); + test("rejects the configured number of prompts before starting a retry", async () => { const client = new MockLoadTestAgentClient(); const session = await client.createSession({ diff --git a/packages/server/src/server/agent/providers/mock-load-test-agent.ts b/packages/server/src/server/agent/providers/mock-load-test-agent.ts index ce9714c5f2..b7b301540a 100644 --- a/packages/server/src/server/agent/providers/mock-load-test-agent.ts +++ b/packages/server/src/server/agent/providers/mock-load-test-agent.ts @@ -18,6 +18,7 @@ import type { AgentRuntimeInfo, AgentSession, AgentSessionConfig, + AgentSlashCommand, AgentStreamEvent, AgentTimelineItem, FetchCatalogOptions, @@ -35,7 +36,13 @@ import { getAgentProviderDefinition } from "@getpaseo/protocol/provider-manifest export const MOCK_LOAD_TEST_PROVIDER_ID = "mock"; export const MOCK_LOAD_TEST_DEFAULT_MODEL_ID = "five-minute-stream"; -export const MOCK_LOAD_TEST_HANDLED_COMMAND = "/mock handled-command"; +const MOCK_LOAD_TEST_COMMAND: AgentSlashCommand = { + name: "mock", + description: "Reply without starting a turn", + argumentHint: "handled-command", + kind: "command", +}; +export const MOCK_LOAD_TEST_HANDLED_COMMAND = `/${MOCK_LOAD_TEST_COMMAND.name} ${MOCK_LOAD_TEST_COMMAND.argumentHint}`; const MOCK_LOAD_TEST_MODE_ID = "load-test"; const MOCK_LOAD_TEST_DURATION_MS = 5 * 60 * 1000; const MOCK_LOAD_TEST_INTERVAL_MS = 40; @@ -954,6 +961,10 @@ export class MockLoadTestAgentSession implements AgentSession { return { turnId }; } + async listCommands(): Promise { + return [MOCK_LOAD_TEST_COMMAND]; + } + tryHandleOutOfBand( prompt: AgentPromptInput, ): { run(ctx: { emit: (event: AgentStreamEvent) => void }): Promise } | null {