diff --git a/packages/app/e2e/browser/assistant-selection-copy.spec.ts b/packages/app/e2e/browser/assistant-selection-copy.spec.ts index 323b2b0c15..bf8e455c88 100644 --- a/packages/app/e2e/browser/assistant-selection-copy.spec.ts +++ b/packages/app/e2e/browser/assistant-selection-copy.spec.ts @@ -168,6 +168,36 @@ async function selectAssistantImage(page: Page): Promise { }); } +/** Press in the gap below the image and drag to the end of `text` in the block after it. */ +async function dragFromBelowImageTo(page: Page, text: string): Promise { + const blocks = assistantMessageBlocks(page); + const target = blocks.getByText(text); + await target.scrollIntoViewIfNeeded(); + const frame = await blocks.locator('[data-paseo-markdown-tag="img"]').boundingBox(); + const end = await target.evaluate((element, selectedText) => { + const walker = document.createTreeWalker(element, NodeFilter.SHOW_TEXT); + while (walker.nextNode()) { + const node = walker.currentNode as Text; + const offset = node.textContent?.indexOf(selectedText) ?? -1; + if (offset >= 0) { + const range = document.createRange(); + range.setStart(node, offset); + range.setEnd(node, offset + selectedText.length); + const box = range.getBoundingClientRect(); + return { x: box.right - 1, y: box.top + box.height / 2, left: box.left }; + } + } + throw new Error(`Could not find ${selectedText}`); + }, text); + if (!frame) { + throw new Error("Expected a rendered image"); + } + await page.mouse.move(end.left + 5, frame.y + frame.height + 4); + await page.mouse.down(); + await page.mouse.move(end.x, end.y, { steps: 10 }); + await page.mouse.up(); +} + async function selectAssistantText(page: Page, text: string): Promise { await selectAssistantTextRange(page, text, text); } @@ -705,6 +735,11 @@ test("copying a selection across an assistant image keeps its Markdown source an await copySelection(page); expect((await readRichClipboard(page)).plainText).toBe(`![chart](${image})`); + + await dragFromBelowImageTo(page, "After the"); + await copySelection(page); + + expect((await readRichClipboard(page)).plainText).toBe("After the"); } finally { await agent.cleanup(); } diff --git a/packages/app/src/assistant-selection-copy/content.browser.test.ts b/packages/app/src/assistant-selection-copy/content.browser.test.ts index 5fe8e0fddc..aecb4ba5b4 100644 --- a/packages/app/src/assistant-selection-copy/content.browser.test.ts +++ b/packages/app/src/assistant-selection-copy/content.browser.test.ts @@ -143,6 +143,52 @@ describe("assistant selection copy ranges", () => { expect(copiedMarkdown(selectRange(blocks[0]!, 6, blocks[1]!, 6))).toBe("paragraph.\n\nSecond"); }); + // Chrome anchors a drag that starts in the gap below an image at the start of the + // image's rendered internals: the background-image layer, ahead of the hidden `img`. + // Only the text after it is highlighted. + it("leaves out an image when the selection starts in the gap below it", () => { + const transcript = mountTranscript([ + { + messageId: "message-1", + blocks: [ + "Before the image.", + '
', + "After the image.", + ], + }, + ]); + const frame = fixtureElement(transcript, '[data-paseo-markdown-tag="img"]'); + const range = document.createRange(); + range.setStart(frame.firstElementChild!.firstElementChild!, 0); + range.setEnd(textNode(fixtureElement(transcript, '[data-paseo-markdown-tag="p"]', 2)), 9); + const selection = window.getSelection()!; + selection.removeAllRanges(); + selection.addRange(range); + + expect(copiedMarkdown(selection)).toBe("After the"); + }); + + it("keeps a failed image when the selection starts in its error text", () => { + const transcript = mountTranscript([ + { + messageId: "message-1", + blocks: [ + '
Image failed to load
', + "After the image.", + ], + }, + ]); + const frame = fixtureElement(transcript, '[data-paseo-markdown-tag="img"]'); + const selection = selectRange( + frame.firstElementChild!, + 6, + fixtureElement(transcript, '[data-paseo-markdown-tag="p"]', 1), + 9, + ); + + expect(copiedMarkdown(selection)).toBe("![chart](https://example.test/chart.png)\n\nAfter the"); + }); + it("does not replace the browser clipboard for a selection reaching a second message", () => { const transcript = mountTranscript([ { messageId: "message-1", blocks: ["First paragraph.", "Second paragraph."] }, diff --git a/packages/app/src/assistant-selection-copy/content.web.ts b/packages/app/src/assistant-selection-copy/content.web.ts index 60ff6d2165..e59a71aeca 100644 --- a/packages/app/src/assistant-selection-copy/content.web.ts +++ b/packages/app/src/assistant-selection-copy/content.web.ts @@ -24,6 +24,7 @@ const CHAT_SCROLL_SELECTOR = '[data-testid="agent-chat-scroll"]'; const messageRowSelector = (messageId: string) => `[data-message-id="${CSS.escape(messageId)}"]`; const CODE_BLOCK_SELECTOR = `[${MARKDOWN_COPY_TAG_ATTRIBUTE}="pre"]`; const CODE_REGION_SELECTOR = `${CODE_BLOCK_SELECTOR}, [${MARKDOWN_COPY_TAG_ATTRIBUTE}="code"]`; +const IMAGE_FRAME_SELECTOR = `[${MARKDOWN_COPY_TAG_ATTRIBUTE}="img"]`; const turndown = new TurndownService({ bulletListMarker: "-", @@ -66,7 +67,7 @@ export function createAssistantSelectionClipboardContent( return null; } - const range = selection.getRangeAt(0); + const range = startAfterImageFrame(selection.getRangeAt(0)); const parts = selectedMessageParts(range)?.filter((part) => selectsContent(part.range)); if (!parts?.length) { return null; @@ -93,6 +94,33 @@ export function createAssistantSelectionClipboardContent( return { ...content, html: flattenClipboardListMarkup(content.html) }; } +/** + * A drag that starts in the gap below an image anchors at the start of the image's + * rendered internals, ahead of the hidden `img`, while only the text after it is + * highlighted. Pressing on the image itself opens it instead of starting a selection, + * so a selection that starts inside an image, before any of its text, and ends past it + * starts after the image. A failed image shows its error as text, and a selection + * starting in that text keeps the image. + */ +function startAfterImageFrame(range: Range): Range { + const start = + range.startContainer instanceof Element + ? range.startContainer + : range.startContainer.parentElement; + const frame = start?.closest(IMAGE_FRAME_SELECTOR); + if (!frame || frame.contains(range.endContainer)) { + return range; + } + const insideFrame = range.cloneRange(); + insideFrame.setEnd(frame, frame.childNodes.length); + if (insideFrame.toString()) { + return range; + } + const afterFrame = range.cloneRange(); + afterFrame.setStartAfter(frame); + return afterFrame; +} + /** * Partly-selected code copies as the code itself, never through Turndown. *