From d93d6a268a0ea40fb977ff9f54737369758f31b5 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 10 Jul 2026 14:29:43 +0200 Subject: [PATCH] fix(desktop): keep browser input out of the composer Unhandled webview keys could be redispatched into the active host window, allowing agent Enter to submit a draft prompt. Give pages first refusal for ordinary shortcuts and contain automation at the guest boundary. --- docs/architecture.md | 16 +- docs/browser-capture-harness.md | 14 +- packages/app/src/desktop/host.ts | 18 +- .../app/src/hooks/use-keyboard-shortcuts.ts | 225 ++++++++----- packages/app/src/keyboard/actions.ts | 1 + .../src/keyboard/browser-shortcuts.test.ts | 177 ++++++++++ .../app/src/keyboard/browser-shortcuts.ts | 144 +++++++++ .../src/keyboard/keyboard-shortcuts.test.ts | 25 +- .../app/src/keyboard/keyboard-shortcuts.ts | 39 ++- packages/desktop/capture-harness/index.html | 45 +++ packages/desktop/capture-harness/main.js | 291 ++++++++++++++++- .../features/browser-automation/ipc.test.ts | 28 +- .../src/features/browser-automation/ipc.ts | 42 +-- .../browser-automation/service.test.ts | 81 +++-- .../features/browser-automation/service.ts | 18 +- .../browser-automation/trusted-input.test.ts | 43 ++- .../browser-automation/trusted-input.ts | 75 ++--- .../browser-keyboard/guest-preload.ts | 78 +++++ .../features/browser-keyboard/index.test.ts | 281 ++++++++++++++++ .../src/features/browser-keyboard/index.ts | 199 ++++++++++++ .../features/browser-keyboard/policy.test.ts | 74 +++++ .../src/features/browser-keyboard/policy.ts | 132 ++++++++ .../features/browser-webviews/index.test.ts | 42 ++- .../src/features/browser-webviews/index.ts | 67 +++- .../browser-webviews/registry.test.ts | 306 ++++++++++++++++-- .../src/features/browser-webviews/registry.ts | 242 +++++++++++--- packages/desktop/src/features/menu.test.ts | 70 ++++ packages/desktop/src/features/menu.ts | 44 ++- packages/desktop/src/main.ts | 134 +++----- packages/desktop/src/preload.ts | 19 +- 30 files changed, 2517 insertions(+), 453 deletions(-) create mode 100644 packages/app/src/keyboard/browser-shortcuts.test.ts create mode 100644 packages/app/src/keyboard/browser-shortcuts.ts create mode 100644 packages/desktop/src/features/browser-keyboard/guest-preload.ts create mode 100644 packages/desktop/src/features/browser-keyboard/index.test.ts create mode 100644 packages/desktop/src/features/browser-keyboard/index.ts create mode 100644 packages/desktop/src/features/browser-keyboard/policy.test.ts create mode 100644 packages/desktop/src/features/browser-keyboard/policy.ts create mode 100644 packages/desktop/src/features/menu.test.ts diff --git a/docs/architecture.md b/docs/architecture.md index ff920a58d..167e74737 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -141,8 +141,22 @@ Electron wrapper for macOS, Linux, and Windows. > **In-app browser profile.** Every browser guest uses one stable persistent Electron session, so cookies, authentication, cache, and site storage are shared across tabs, workspaces, and desktop windows and survive tab or app closure. Browser identity is independent of that storage partition: after `did-attach`, the renderer explicitly registers its browser id, workspace id, and guest `WebContents` id, and main accepts the registration only when that guest belongs to the calling renderer and the shared profile. Settings > General > Clear browser data is the sole profile-deletion path; it clears the shared session and reloads live guests without deleting saved tabs or URLs. > > **In-app browser window opens.** Ordinary link opens, including Shift-clicked links, become Paseo workspace tabs. Script-created opens with popup features or a named window target and POST-backed opens remain secured Electron child windows in the shared browser profile, preserving `window.opener`, `postMessage`, named-window reuse, request bodies, and `window.close()` for OAuth, payment, and similar popup protocols. Unsupported URL schemes are denied before either path. +> +> **In-app browser ownership.** Each registered guest records its owning host window. The active browser is keyed by `(host window, workspace)`, and application-menu Reload / Force Reload resolve only within the window Electron supplies to the menu callback. A non-null active update must name a browser owned by that host; a null update clears only that host/workspace. Browser automation continues to target explicit browser ids returned by `browser_new_tab` or `browser_list_tabs`. +> +> **Browser keyboard boundary.** Guest pages receive renderer-published shortcuts first. `Cmd/Ctrl+L` and `Cmd/Ctrl+R` are explicit guest-shell reservations; ordinary Paseo shortcuts run only after the page declines them. The sandboxed guest preload runs in every frame so focused iframes use the same boundary, while Node integration remains disabled. Human guest input disables Electron's menu fallback for plain keys. Agent-generated keys use guest `sendInputEvent` with `skipIfUnhandled`, so an unhandled Enter stops at the guest instead of reaching the host composer. Main selects the preload; it exposes no APIs to guest pages. -> **In-app browser targets are not yet per-window.** Browser webviews are still tracked by one process-global registry that keeps a single current `WebContents` per browser id. Human focus records the workspace-active browser for UI state and `list_tabs` reporting, while agent automation targets explicit browser ids returned by `browser_new_tab` or `browser_list_tabs`. Explicit attached-guest registration prevents concurrent windows from swapping different browser ids, but rendering the same saved browser tab in multiple windows can still make menu actions target the most recently registered guest. Making the registry window-scoped remains a follow-up. +```text +Human key -> guest WebContents + |-- Cmd/Ctrl+T/L/R ----------> reserved browser-shell action + `-- page keydown + |-- page prevents ------> page owns it + `-- published shortcut -> guest preload -> IPC(browserId) -> Paseo resolver + +Agent browser_keypress -> guest sendInputEvent(skipIfUnhandled) + |-- guest handles ------------> page owns it + `-- guest does not handle ----> stop; never redispatch to the host window +``` ### `packages/website` — Marketing site diff --git a/docs/browser-capture-harness.md b/docs/browser-capture-harness.md index 5907b1a76..14db85756 100644 --- a/docs/browser-capture-harness.md +++ b/docs/browser-capture-harness.md @@ -13,7 +13,12 @@ It validates the compositor behavior that unit tests cannot see: - both viewport `capturePage` and full-page CDP screenshots return real pixels from the permanent production parking state; - guest background throttling can be disabled once at attach without per-capture - renderer coordination. + renderer coordination; +- the real-Electron host-composer sentinel proves guest Enter cannot submit a focused + host composer; +- the automation group loads the compiled production keyboard boundary and guest + preload, then proves a page-handled shortcut stays in the page while an unhandled + shortcut crosses once. Run it with the repo Electron: @@ -21,9 +26,11 @@ Run it with the repo Electron: npm run capture-harness --workspace=@getpaseo/desktop ``` -Run the browser automation fixture with: +Build the desktop main process before the automation group so its production guest +preload is available: ```bash +npm run build:main --workspace=@getpaseo/desktop PASEO_CAPTURE_HARNESS_GROUP=automation npm run capture-harness --workspace=@getpaseo/desktop ``` @@ -43,6 +50,9 @@ ARIA-like snapshot text includes headings, static text, and controls; refs survi `pushState` when the element still matches; same-URL rerenders stale old refs; and a file-input ref can be resolved to a CDP backend node id for upload. It also verifies page-context evaluation, including passing a resolved ref element as the function argument. +Keyboard containment runs last because the host-composer sentinel intentionally leaves +native focus in the host. It reuses an existing fixture button: adding a test-only control +changes the inline fixture geometry exercised by the earlier actionability checks. On macOS the harness process must set `app.setActivationPolicy("accessory")` and hide the Dock icon before creating any window. `showInactive()` only prevents window diff --git a/packages/app/src/desktop/host.ts b/packages/app/src/desktop/host.ts index bc26b91c5..8c8b71a85 100644 --- a/packages/app/src/desktop/host.ts +++ b/packages/app/src/desktop/host.ts @@ -1,5 +1,6 @@ import { Platform } from "react-native"; import { getElectronHost } from "@/desktop/electron/host"; +import type { BrowserKeyboardPolicy } from "@/keyboard/browser-shortcuts"; import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages"; type BrowserAutomationExecuteRequest = Extract< @@ -24,7 +25,6 @@ export interface DesktopDialogOpenOptions { title?: string; defaultPath?: string; directory?: boolean; - createDirectory?: boolean; multiple?: boolean; filters?: Array<{ name: string; @@ -66,15 +66,13 @@ export interface DesktopEditorTargetDescriptor { id: string; label: string; kind: "editor" | "file-manager"; - icon: { kind: "image"; dataUrl: string } | { kind: "symbol"; name: "folder" | "terminal" }; } export interface DesktopEditorOpenTargetInput { editorId: string; - workspacePath: string; - filePath?: string; - line?: number; - column?: number; + path: string; + cwd?: string; + mode?: "open" | "reveal"; } export interface DesktopEditorBridge { @@ -122,10 +120,9 @@ export interface DesktopEventsBridge { on?: (event: string, handler: (payload: unknown) => void) => Promise<() => void> | (() => void); } -export interface DesktopBrowserShortcutEvent { - browserId?: string; - action: "focus-url"; -} +export type DesktopBrowserShortcutEvent = + | { browserId?: string; action: "focus-url" } + | { browserId: string; action: "new-tab" }; export interface DesktopBrowserNewTabRequestEvent { sourceBrowserId: string; @@ -139,6 +136,7 @@ export interface DesktopAttachedBrowserRegistration { } export interface DesktopBrowserBridge { + setShortcutPolicy?: (input: BrowserKeyboardPolicy) => Promise; readonly profilePartition?: string; registerAttachedBrowser?: (input: DesktopAttachedBrowserRegistration) => Promise; unregisterWorkspaceBrowser?: (browserId: string) => Promise; diff --git a/packages/app/src/hooks/use-keyboard-shortcuts.ts b/packages/app/src/hooks/use-keyboard-shortcuts.ts index de6e370c4..75192f704 100644 --- a/packages/app/src/hooks/use-keyboard-shortcuts.ts +++ b/packages/app/src/hooks/use-keyboard-shortcuts.ts @@ -7,11 +7,17 @@ import { navigateToWorkspace } from "@/stores/navigation-active-workspace-store" import { keyboardActionDispatcher } from "@/keyboard/keyboard-action-dispatcher"; import { type ChordState, + type KeyboardShortcutInput, resolveKeyboardShortcut, buildEffectiveBindings, getWorkspaceIndexJumpModifierKey, } from "@/keyboard/keyboard-shortcuts"; import { resolveKeyboardFocusScope } from "@/keyboard/focus-scope"; +import { + buildBrowserShortcutPolicy, + parseBrowserShortcutInput, +} from "@/keyboard/browser-shortcuts"; +import type { KeyboardFocusScope, KeyboardShortcutPayload } from "@/keyboard/actions"; import { routeKeyboardShortcut, type ShortcutAction, @@ -47,6 +53,8 @@ export function useKeyboardShortcuts({ const resetModifiers = useKeyboardShortcutsStore((s) => s.resetModifiers); const { overrides } = useKeyboardShortcutOverrides(); const bindings = useMemo(() => buildEffectiveBindings(overrides), [overrides]); + const isDesktopApp = getIsElectronRuntime(); + const isMac = getShortcutOs() === "mac"; const chordStateRef = useRef({ candidateIndices: [], step: 0, @@ -62,14 +70,27 @@ export function useKeyboardShortcuts({ } }, [activeWorkspaceSelection]); + useEffect(() => { + if (!isDesktopApp) { + return; + } + + const prefixes = + enabled && !isMobile + ? buildBrowserShortcutPolicy({ + bindings, + isMac, + isDesktop: isDesktopApp, + }) + : []; + void getDesktopHost()?.browser?.setShortcutPolicy?.({ prefixes }); + }, [bindings, enabled, isDesktopApp, isMac, isMobile]); + useEffect(() => { if (!enabled) return; if (isNative) return; if (isMobile) return; - const isDesktopApp = getIsElectronRuntime(); - const isMac = getShortcutOs() === "mac"; - // Only the modifier that actually performs the workspace-index jump on this // runtime should reveal the sidebar number badges (Alt on web, Cmd on // desktop Mac, Ctrl on desktop non-Mac). The store ORs altDown/cmdOrCtrlDown @@ -106,7 +127,10 @@ export function useKeyboardShortcuts({ "cycle-theme": cycleTheme, }; - const performShortcutAction = (action: ShortcutAction, event: KeyboardEvent): boolean => { + const performShortcutAction = ( + action: ShortcutAction, + event: KeyboardEvent | null, + ): boolean => { switch (action.kind) { case "none": return false; @@ -137,7 +161,7 @@ export function useKeyboardShortcuts({ callbacksByName[action.name]?.(); return true; case "command-center-toggle": { - if (action.nextOpen) { + if (action.nextOpen && event) { captureCommandCenterFocusRestore(event); } useKeyboardShortcutsStore.getState().setCommandCenterOpen(action.nextOpen); @@ -149,6 +173,80 @@ export function useKeyboardShortcuts({ } }; + const routeAndPerformShortcut = (input: { + action: string; + payload: KeyboardShortcutPayload; + domEvent: KeyboardEvent | null; + }): boolean => { + const store = useKeyboardShortcutsStore.getState(); + const shortcutAction = routeKeyboardShortcut( + { action: input.action, payload: input.payload }, + { + pathname, + isMobile, + sidebarShortcutTargets: store.sidebarShortcutWorkspaceTargets, + navigationActiveWorkspace: + keyboardWorkspaceSelectionRef.current ?? activeWorkspaceSelection, + commandCenterOpen: store.commandCenterOpen, + shortcutsDialogOpen: store.shortcutsDialogOpen, + }, + ); + return performShortcutAction(shortcutAction, input.domEvent); + }; + + const resolveAndPerformShortcut = (input: { + event: KeyboardShortcutInput; + focusScope: KeyboardFocusScope; + domEvent: KeyboardEvent | null; + }) => { + const store = useKeyboardShortcutsStore.getState(); + const result = resolveKeyboardShortcut({ + event: input.event, + context: { + isMac, + isDesktop: isDesktopApp, + focusScope: input.focusScope, + commandCenterOpen: store.commandCenterOpen, + }, + chordState: chordStateRef.current, + onChordReset: () => { + chordStateRef.current = { + candidateIndices: [], + step: 0, + timeoutId: null, + }; + }, + bindings, + }); + + chordStateRef.current = result.nextChordState; + + if (result.preventDefault && input.domEvent) { + input.domEvent.preventDefault(); + input.domEvent.stopPropagation(); + } + + if (!result.match) { + return; + } + + const handled = routeAndPerformShortcut({ + action: result.match.action, + payload: result.match.payload, + domEvent: input.domEvent, + }); + if (!handled || !input.domEvent) { + return; + } + + if (result.match.preventDefault) { + input.domEvent.preventDefault(); + } + if (result.match.stopPropagation) { + input.domEvent.stopPropagation(); + } + }; + const handleKeyDown = (event: KeyboardEvent) => { if (!shouldHandle()) { return; @@ -180,60 +278,11 @@ export function useKeyboardShortcuts({ target: event.target, commandCenterOpen: store.commandCenterOpen, }); - const result = resolveKeyboardShortcut({ + resolveAndPerformShortcut({ event, - context: { - isMac, - isDesktop: isDesktopApp, - focusScope, - commandCenterOpen: store.commandCenterOpen, - }, - chordState: chordStateRef.current, - onChordReset: () => { - chordStateRef.current = { - candidateIndices: [], - step: 0, - timeoutId: null, - }; - }, - bindings, + focusScope, + domEvent: event, }); - - chordStateRef.current = result.nextChordState; - - if (result.preventDefault) { - event.preventDefault(); - event.stopPropagation(); - } - - if (!result.match) { - return; - } - - const shortcutAction = routeKeyboardShortcut( - { action: result.match.action, payload: result.match.payload }, - { - pathname, - isMobile, - sidebarShortcutTargets: store.sidebarShortcutWorkspaceTargets, - navigationActiveWorkspace: - keyboardWorkspaceSelectionRef.current ?? activeWorkspaceSelection, - commandCenterOpen: store.commandCenterOpen, - shortcutsDialogOpen: store.shortcutsDialogOpen, - }, - ); - - const handled = performShortcutAction(shortcutAction, event); - if (!handled) { - return; - } - - if (result.match.preventDefault) { - event.preventDefault(); - } - if (result.match.stopPropagation) { - event.stopPropagation(); - } }; const handleKeyUp = (event: KeyboardEvent) => { @@ -252,22 +301,39 @@ export function useKeyboardShortcuts({ window.addEventListener("blur", handleBlurOrHide); document.addEventListener("visibilitychange", handleBlurOrHide); - const forwardedKeySubscription = isElectronRuntime() - ? getDesktopHost()?.events?.on?.("browser-forwarded-key", (payload) => { - if (!payload || typeof payload !== "object") return; - const p = payload as Record; - if (typeof p.key !== "string") return; - window.dispatchEvent( - new KeyboardEvent("keydown", { - key: p.key, - code: typeof p.code === "string" ? p.code : "", - metaKey: p.meta === true, - ctrlKey: p.control === true, - shiftKey: p.shift === true, - altKey: p.alt === true, - bubbles: true, - }), - ); + const browserShortcutSubscription = isElectronRuntime() + ? getDesktopHost()?.events?.on?.("browser-shortcut-input", (payload) => { + const input = parseBrowserShortcutInput(payload); + if (!input) { + return; + } + resolveAndPerformShortcut({ + event: input, + focusScope: "browser", + domEvent: null, + }); + }) + : null; + const browserReservedShortcutSubscription = isElectronRuntime() + ? getDesktopHost()?.events?.on?.("browser-shortcut", (payload) => { + if (typeof payload !== "object" || payload === null || Array.isArray(payload)) { + return; + } + if (!("action" in payload) || payload.action !== "new-tab") { + return; + } + if ( + !("browserId" in payload) || + typeof payload.browserId !== "string" || + payload.browserId.length === 0 + ) { + return; + } + routeAndPerformShortcut({ + action: "workspace.tab.new", + payload: null, + domEvent: null, + }); }) : null; @@ -284,10 +350,15 @@ export function useKeyboardShortcuts({ window.removeEventListener("keyup", handleKeyUp, true); window.removeEventListener("blur", handleBlurOrHide); document.removeEventListener("visibilitychange", handleBlurOrHide); - if (typeof forwardedKeySubscription === "function") { - forwardedKeySubscription(); + if (typeof browserShortcutSubscription === "function") { + browserShortcutSubscription(); } else { - void forwardedKeySubscription?.then((dispose) => dispose()); + void browserShortcutSubscription?.then((dispose) => dispose()); + } + if (typeof browserReservedShortcutSubscription === "function") { + browserReservedShortcutSubscription(); + } else { + void browserReservedShortcutSubscription?.then((dispose) => dispose()); } }; }, [ @@ -295,6 +366,8 @@ export function useKeyboardShortcuts({ cycleTheme, enabled, activeWorkspaceSelection, + isDesktopApp, + isMac, isMobile, openProjectPickerAction, pathname, diff --git a/packages/app/src/keyboard/actions.ts b/packages/app/src/keyboard/actions.ts index d0246594c..7f776cd0a 100644 --- a/packages/app/src/keyboard/actions.ts +++ b/packages/app/src/keyboard/actions.ts @@ -3,6 +3,7 @@ export type KeyboardFocusScope = | "message-input" | "command-center" | "editable" + | "browser" | "other"; export type MessageInputKeyboardActionKind = diff --git a/packages/app/src/keyboard/browser-shortcuts.test.ts b/packages/app/src/keyboard/browser-shortcuts.test.ts new file mode 100644 index 000000000..f5dbe8b0f --- /dev/null +++ b/packages/app/src/keyboard/browser-shortcuts.test.ts @@ -0,0 +1,177 @@ +import { describe, expect, it } from "vitest"; +import { buildBrowserShortcutPolicy, parseBrowserShortcutInput } from "./browser-shortcuts"; +import { buildEffectiveBindings } from "./keyboard-shortcuts"; + +describe("buildBrowserShortcutPolicy", () => { + it("publishes the effective browser page-first shortcut prefixes", () => { + const bindings = buildEffectiveBindings({ + "workspace-tab-new-ctrl-t-non-mac": "Ctrl+Y", + "workspace-terminal-new-ctrl-shift-t-non-mac": "Ctrl+F12 Ctrl+F11", + }); + + const policy = buildBrowserShortcutPolicy({ bindings, isMac: false, isDesktop: true }); + + expect(policy).toContainEqual({ + alt: false, + code: "KeyY", + control: true, + key: "y", + meta: false, + shift: false, + }); + expect(policy).toContainEqual({ + alt: false, + code: "F12", + control: true, + meta: false, + shift: false, + }); + expect(policy).toContainEqual({ + alt: false, + code: "F11", + control: true, + meta: false, + shift: false, + }); + }); + + it("rejects an entire chord when a continuation cannot cross the browser boundary", () => { + const bindings = buildEffectiveBindings({ + "settings-toggle-ctrl-comma-non-mac": "Ctrl+F10 F9", + }); + + const policy = buildBrowserShortcutPolicy({ bindings, isMac: false, isDesktop: true }); + + expect(policy).not.toContainEqual({ + alt: false, + code: "F10", + control: true, + meta: false, + shift: false, + }); + expect(policy).not.toContainEqual({ + alt: false, + code: "F9", + control: false, + meta: false, + shift: false, + }); + }); + + it("publishes Mod bindings for the current shortcut platform", () => { + const bindings = buildEffectiveBindings({ + "workspace-tab-new-cmd-t-mac": "Mod+Y", + }); + + expect(buildBrowserShortcutPolicy({ bindings, isMac: true, isDesktop: true })).toContainEqual({ + alt: false, + code: "KeyY", + control: false, + key: "y", + meta: true, + shift: false, + }); + }); + + it("does not publish plain browser keys", () => { + const bindings = buildEffectiveBindings({}); + const policy = buildBrowserShortcutPolicy({ bindings, isMac: false, isDesktop: true }); + + expect(policy).not.toContainEqual({ + alt: false, + code: "Enter", + control: false, + meta: false, + shift: false, + }); + expect(policy).not.toContainEqual({ + alt: false, + code: "Slash", + control: false, + meta: false, + shift: true, + }); + }); + + it("publishes Cmd+B with its logical key for non-QWERTY layouts", () => { + const bindings = buildEffectiveBindings({}); + const policy = buildBrowserShortcutPolicy({ bindings, isMac: true, isDesktop: true }); + + expect(policy).toContainEqual({ + alt: false, + code: "KeyB", + control: false, + key: "b", + meta: true, + shift: false, + }); + }); + + it("publishes the physical code needed for macOS Option shortcuts", () => { + const bindings = buildEffectiveBindings({}); + const policy = buildBrowserShortcutPolicy({ bindings, isMac: true, isDesktop: true }); + + expect(policy).toContainEqual({ + alt: true, + code: "KeyT", + control: false, + key: "t", + meta: true, + shift: false, + }); + }); +}); + +describe("parseBrowserShortcutInput", () => { + it("normalizes browser shortcut input without losing its identity", () => { + expect( + parseBrowserShortcutInput({ + browserId: "browser-1", + key: "t", + code: "KeyT", + meta: false, + control: true, + shift: false, + alt: false, + }), + ).toEqual({ + browserId: "browser-1", + key: "t", + code: "KeyT", + metaKey: false, + ctrlKey: true, + shiftKey: false, + altKey: false, + repeat: false, + }); + }); + + it.each([ + { + name: "a missing browser identity", + payload: { + key: "t", + code: "KeyT", + meta: false, + control: true, + shift: false, + alt: false, + }, + }, + { + name: "a malformed repeat flag", + payload: { + browserId: "browser-1", + key: "t", + code: "KeyT", + meta: false, + control: true, + shift: false, + alt: false, + repeat: "yes", + }, + }, + ])("rejects $name", ({ payload }) => { + expect(parseBrowserShortcutInput(payload)).toBeNull(); + }); +}); diff --git a/packages/app/src/keyboard/browser-shortcuts.ts b/packages/app/src/keyboard/browser-shortcuts.ts new file mode 100644 index 000000000..ae27becc2 --- /dev/null +++ b/packages/app/src/keyboard/browser-shortcuts.ts @@ -0,0 +1,144 @@ +import { + matchesKeyboardShortcutContext, + type KeyboardShortcutInput, + type ParsedShortcutBinding, +} from "./keyboard-shortcuts"; +import type { KeyCombo } from "./shortcut-string"; + +export interface BrowserShortcutPrefix { + alt: boolean; + code: string; + codeFallback?: true; + control: boolean; + key?: string; + meta: boolean; + repeat?: false; + shift: boolean; + shiftedKey?: string; +} + +export interface BrowserShortcutInput extends KeyboardShortcutInput { + browserId: string; +} + +interface BrowserShortcutPolicyInput { + bindings: readonly ParsedShortcutBinding[]; + isMac: boolean; + isDesktop: boolean; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function parseBrowserShortcutInput(value: unknown): BrowserShortcutInput | null { + if (!isRecord(value)) { + return null; + } + const { browserId, code, key } = value; + if (typeof browserId !== "string" || browserId.length === 0) { + return null; + } + if (typeof code !== "string" || typeof key !== "string") { + return null; + } + if ( + typeof value.alt !== "boolean" || + typeof value.control !== "boolean" || + typeof value.meta !== "boolean" || + typeof value.shift !== "boolean" || + (value.repeat !== undefined && typeof value.repeat !== "boolean") + ) { + return null; + } + + return { + browserId, + key, + code, + altKey: value.alt, + ctrlKey: value.control, + metaKey: value.meta, + shiftKey: value.shift, + repeat: value.repeat ?? false, + }; +} + +function prefixFromCombo(combo: KeyCombo, isMac: boolean): BrowserShortcutPrefix | null { + const prefix: BrowserShortcutPrefix = { + alt: combo.alt === true, + code: combo.code, + control: combo.ctrl === true || (!isMac && combo.mod === true), + meta: combo.meta === true || (isMac && combo.mod === true), + shift: combo.shift === true, + }; + if (combo.codeFallback === true) { + prefix.codeFallback = true; + } + if (combo.key) { + prefix.key = combo.key; + } + if (combo.repeat === false) { + prefix.repeat = false; + } + if (combo.shiftedKey) { + prefix.shiftedKey = combo.shiftedKey; + } + return prefix.meta || prefix.control || prefix.alt ? prefix : null; +} + +function prefixesFromBinding( + binding: ParsedShortcutBinding, + isMac: boolean, +): BrowserShortcutPrefix[] | null { + const prefixes: BrowserShortcutPrefix[] = []; + for (const combo of binding.parsedChord) { + const prefix = prefixFromCombo(combo, isMac); + if (!prefix) { + return null; + } + prefixes.push(prefix); + } + return prefixes; +} + +function prefixKey(prefix: BrowserShortcutPrefix): string { + return [ + prefix.code, + prefix.key ?? "", + prefix.shiftedKey ?? "", + prefix.codeFallback ?? "", + prefix.control, + prefix.meta, + prefix.alt, + prefix.shift, + prefix.repeat ?? "", + ].join(":"); +} + +export function buildBrowserShortcutPolicy( + input: BrowserShortcutPolicyInput, +): BrowserShortcutPrefix[] { + const prefixes = new Map(); + const context = { + isMac: input.isMac, + isDesktop: input.isDesktop, + focusScope: "browser" as const, + commandCenterOpen: false, + }; + + for (const binding of input.bindings) { + if (!matchesKeyboardShortcutContext(binding.when, context)) { + continue; + } + const bindingPrefixes = prefixesFromBinding(binding, input.isMac); + if (!bindingPrefixes) { + continue; + } + for (const prefix of bindingPrefixes) { + prefixes.set(prefixKey(prefix), prefix); + } + } + + return [...prefixes.values()]; +} diff --git a/packages/app/src/keyboard/keyboard-shortcuts.test.ts b/packages/app/src/keyboard/keyboard-shortcuts.test.ts index 5e1c905ec..496288bb8 100644 --- a/packages/app/src/keyboard/keyboard-shortcuts.test.ts +++ b/packages/app/src/keyboard/keyboard-shortcuts.test.ts @@ -7,10 +7,11 @@ import { resolveKeyboardShortcut, type ChordState, type KeyboardShortcutContext, + type KeyboardShortcutInput, type ParsedShortcutBinding, } from "./keyboard-shortcuts"; -function keyboardEvent(overrides: Partial): KeyboardEvent { +function keyboardInput(overrides: Partial): KeyboardShortcutInput { return { key: "", code: "", @@ -20,7 +21,7 @@ function keyboardEvent(overrides: Partial): KeyboardEvent { shiftKey: false, repeat: false, ...overrides, - } as KeyboardEvent; + }; } function shortcutContext( @@ -44,14 +45,14 @@ function initialChordState(): ChordState { } function resolveShortcut(input: { - event: Partial; + event: Partial; context?: Partial; chordState?: ChordState; onChordReset?: () => void; bindings?: readonly ParsedShortcutBinding[]; }) { return resolveKeyboardShortcut({ - event: keyboardEvent(input.event), + event: keyboardInput(input.event), context: shortcutContext(input.context), chordState: input.chordState ?? initialChordState(), onChordReset: input.onChordReset ?? (() => undefined), @@ -60,7 +61,7 @@ function resolveShortcut(input: { } function expectShortcutResolution(input: { - event: Partial; + event: Partial; context?: Partial; action: string; payload?: unknown; @@ -83,7 +84,7 @@ function expectShortcutResolution(input: { } function expectNoShortcutResolution(input: { - event: Partial; + event: Partial; context?: Partial; }) { const result = resolveShortcut({ @@ -98,7 +99,7 @@ function expectNoShortcutResolution(input: { interface MatchingShortcutCase { name: string; - event: Partial; + event: Partial; context?: Partial; action: string; payload?: unknown; @@ -108,7 +109,7 @@ interface MatchingShortcutCase { interface NonMatchingShortcutCase { name: string; - event: Partial; + event: Partial; context?: Partial; } @@ -548,6 +549,14 @@ describe("keyboard-shortcuts", () => { expect(secondResult.nextChordState).toEqual(initialChordState()); }); + it("resolves a browser-origin shortcut with browser focus instead of host focus", () => { + expectShortcutResolution({ + event: { key: "t", code: "KeyT", ctrlKey: true }, + context: { isDesktop: true, focusScope: "browser" }, + action: "workspace.tab.new", + }); + }); + it("schedules a chord reset timeout for advancing candidates", () => { vi.useFakeTimers(); diff --git a/packages/app/src/keyboard/keyboard-shortcuts.ts b/packages/app/src/keyboard/keyboard-shortcuts.ts index e67158475..60d0d2f77 100644 --- a/packages/app/src/keyboard/keyboard-shortcuts.ts +++ b/packages/app/src/keyboard/keyboard-shortcuts.ts @@ -18,6 +18,16 @@ export interface KeyboardShortcutContext { commandCenterOpen: boolean; } +export interface KeyboardShortcutInput { + key: string; + code: string; + altKey: boolean; + ctrlKey: boolean; + metaKey: boolean; + shiftKey: boolean; + repeat: boolean; +} + export interface KeyboardShortcutMatch { action: KeyboardActionId; payload: KeyboardShortcutPayload; @@ -1060,8 +1070,8 @@ export function buildEffectiveBindings(overrides: Record): Parse // --- Matching engine --- -function parseDigit(event: KeyboardEvent): number | null { - const code = event.code ?? ""; +function parseDigit(event: KeyboardShortcutInput): number | null { + const code = event.code; if (code.startsWith("Digit")) { const value = Number(code.slice("Digit".length)); return Number.isFinite(value) && value >= 1 && value <= 9 ? value : null; @@ -1070,14 +1080,14 @@ function parseDigit(event: KeyboardEvent): number | null { const value = Number(code.slice("Numpad".length)); return Number.isFinite(value) && value >= 1 && value <= 9 ? value : null; } - const key = event.key ?? ""; + const key = event.key; if (key >= "1" && key <= "9") { return Number(key); } return null; } -function matchesKeyOrCode(combo: KeyCombo, event: KeyboardEvent): boolean { +function matchesKeyOrCode(combo: KeyCombo, event: KeyboardShortcutInput): boolean { if (combo.key === undefined) { return event.code === combo.code; } @@ -1095,7 +1105,7 @@ function matchesKeyOrCode(combo: KeyCombo, event: KeyboardEvent): boolean { return combo.codeFallback === true && event.code === combo.code; } -function matchesCombo(combo: KeyCombo, event: KeyboardEvent, isMac: boolean): boolean { +function matchesCombo(combo: KeyCombo, event: KeyboardShortcutInput, isMac: boolean): boolean { if (combo.mod) { if (isMac) { if (!event.metaKey) return false; @@ -1118,7 +1128,10 @@ function matchesCombo(combo: KeyCombo, event: KeyboardEvent, isMac: boolean): bo return matchesKeyOrCode(combo, event); } -function matchesWhen(when: ShortcutWhen | undefined, context: KeyboardShortcutContext): boolean { +export function matchesKeyboardShortcutContext( + when: ShortcutWhen | undefined, + context: KeyboardShortcutContext, +): boolean { if (!when) return true; if (when.mac !== undefined && when.mac !== context.isMac) return false; if (when.desktop !== undefined && when.desktop !== context.isDesktop) return false; @@ -1136,7 +1149,7 @@ function matchesWhen(when: ShortcutWhen | undefined, context: KeyboardShortcutCo function resolvePayload( def: ShortcutPayloadDef | undefined, - event: KeyboardEvent, + event: KeyboardShortcutInput, ): KeyboardShortcutPayload { if (!def) return null; switch (def.type) { @@ -1187,7 +1200,7 @@ function helpMatchesPlatform( function buildMatchFromBinding( binding: ParsedShortcutBinding, - event: KeyboardEvent, + event: KeyboardShortcutInput, ): KeyboardShortcutMatch { return { action: binding.action, @@ -1198,7 +1211,7 @@ function buildMatchFromBinding( } function resolveInitialChordStep(input: { - event: KeyboardEvent; + event: KeyboardShortcutInput; context: KeyboardShortcutContext; chordState: ChordState; onChordReset: () => void; @@ -1220,7 +1233,7 @@ function resolveInitialChordStep(input: { if (!matchesCombo(firstCombo, event, context.isMac)) { continue; } - if (!matchesWhen(binding.when, context)) { + if (!matchesKeyboardShortcutContext(binding.when, context)) { continue; } if (binding.parsedChord.length > 1) { @@ -1252,7 +1265,7 @@ function resolveInitialChordStep(input: { } function resolveAdvancingChordStep(input: { - event: KeyboardEvent; + event: KeyboardShortcutInput; context: KeyboardShortcutContext; chordState: ChordState; onChordReset: () => void; @@ -1278,7 +1291,7 @@ function resolveAdvancingChordStep(input: { if (!matchesCombo(combo, event, context.isMac)) { continue; } - if (!matchesWhen(binding.when, context)) { + if (!matchesKeyboardShortcutContext(binding.when, context)) { continue; } if (chordState.step + 1 === binding.parsedChord.length) { @@ -1317,7 +1330,7 @@ function resolveAdvancingChordStep(input: { } export function resolveKeyboardShortcut(input: { - event: KeyboardEvent; + event: KeyboardShortcutInput; context: KeyboardShortcutContext; chordState: ChordState; onChordReset: () => void; diff --git a/packages/desktop/capture-harness/index.html b/packages/desktop/capture-harness/index.html index f603bda64..3befdcd94 100644 --- a/packages/desktop/capture-harness/index.html +++ b/packages/desktop/capture-harness/index.html @@ -38,12 +38,40 @@ +
+ + +