diff --git a/CLAUDE.md b/CLAUDE.md index fef3c042a0..945f56c436 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -49,7 +49,7 @@ At the start of non-trivial work, list `docs/` and skim anything relevant to the | [docs/permissions.md](docs/permissions.md) | Semantic daemon permissions, principals, credentials, pairing invitations, and Hub authority | | [docs/terminal-performance.md](docs/terminal-performance.md) | Terminal latency pipeline, coalescing/backpressure invariants, benchmark + perf spec usage | | [docs/agent-stream-performance.md](docs/agent-stream-performance.md) | Assistant text pipeline — coalescing window, paced reveal, why arrival lumps are smoothed at render | -| [docs/native-text-selection.md](docs/native-text-selection.md) | Shared Markdown prose runs, selection boundaries, renderer flavor and library constraints | +| [docs/native-text-selection.md](docs/native-text-selection.md) | Native prose runs, selection boundaries, renderer flavor and library constraints | | [docs/file-observation.md](docs/file-observation.md) | Recursive watcher ownership, Linux constraints, teardown invariants, and Parcel comparison | | [docs/testing.md](docs/testing.md) | TDD workflow, determinism, real dependencies over mocks, test organization | | [docs/qa.md](docs/qa.md) | QA evidence bar for pull requests — platform matrix, version drift, performance, UI proof | diff --git a/docs/agent-stream-performance.md b/docs/agent-stream-performance.md index c8e3e3ef99..02d9a42848 100644 --- a/docs/agent-stream-performance.md +++ b/docs/agent-stream-performance.md @@ -31,8 +31,7 @@ So arrival sets a _target_ and the reveal rate is derived from the backlog inste - **The store holds the full text; only the rendered slice is paced.** Copy, selection, the chat outline, and scroll geometry all read the same string the user can see. Pacing the store instead would leave the bottom anchor chasing a content height that is ahead of the reveal. - **Markdown rows belong to presentation, and every assistant message is a block group.** `agent-stream/presentation.ts` keeps the source message whole for plugins, then splits it for - display. Native uses the shared Markdown renderer’s pure `parts` grouper for completed prose runs; web retains individual blocks. - Presentation owns row ids and reuse; the Markdown module owns classification and rendering. + display. Native groups completed adjacent prose into selectable runs; web retains individual blocks. The growing last source block stays on the existing block renderer and streaming parser. It joins the preceding run when the next block starts or the turn ends. Code, tables, images, HTML-like content and blockquotes end a native run. See diff --git a/docs/native-text-selection.md b/docs/native-text-selection.md index 07ba311b70..e59ea89cde 100644 --- a/docs/native-text-selection.md +++ b/docs/native-text-selection.md @@ -1,12 +1,8 @@ # Native text selection -The shared `components/markdown/renderer.tsx` renders native prose with `react-native-enriched-markdown` 1.1.1 on iOS and Android. Native selection belongs to one text view, so separate paragraph rows prevent dragging a selection between them. +Assistant prose uses `react-native-enriched-markdown` 1.1.1 on iOS and Android. Native selection belongs to one text view, so separate paragraph rows prevent dragging a selection between them. -Every `MarkdownRenderer` consumer gets prose runs without surface-specific wiring: chat, Markdown file previews, plan cards, pull request activity and changelogs. The renderer owns classification, native preparation and the platform component under `components/markdown/`. Its pure `parts` boundary groups adjacent paragraphs, headings, lists and horizontal rules; code, tables, images and HTML-like content retain Paseo's block renderers. Parse HTML-like containers and image/text layout before grouping prose; blank-line splitting of the raw document severs `
` from its body. Expanded bodies use the same renderer. Keep whole containers when splitting around an image would lose formatting or list context. - -Chat uses the same grouper to create virtualized display rows. Presentation owns message-to-row ids, promoted/history identity and the growing last source block. It passes an already grouped part to `MarkdownRenderer`; the growing block uses the existing streaming parser and joins the preceding run when another block starts or the turn ends. Selection across that paragraph becomes available at completion. Chat supplies its file-link actions; native prose does not require a chat provider. Other surfaces keep their parser's link policy and compact typography. - -Reference definitions retain their source scope; grouping text for selection must not change which references resolve. Runs are capped at `ASSISTANT_MESSAGE_RENDER_CHARACTER_LIMIT`. Oversized source blocks bypass prose classification and remain on the block renderer. +Keep completed adjacent paragraphs, headings, lists and horizontal rules in prose runs. The growing last source block uses Paseo’s existing block renderer and streaming parser. It joins the preceding run when another block starts or the turn ends, retaining the run’s id. Selection across that paragraph becomes available at completion. Code, tables, images and HTML-like content need Paseo's existing renderers. Preserve whole containers when splitting around an image would lose formatting or list context. Reference definitions retain their original block scope; grouping text for selection must not change which references resolve. Selection stops at preserved blocks, blockquotes and run boundaries required by the 32,000-character render cap. Consecutive paragraphs inside a quote share its text segment; nested quotes and embedded blocks introduce further library segments. Web keeps its existing renderer and selection behavior. diff --git a/packages/app/src/agent-stream/presentation.ts b/packages/app/src/agent-stream/presentation.ts index 0210234dfd..390e52c7f7 100644 --- a/packages/app/src/agent-stream/presentation.ts +++ b/packages/app/src/agent-stream/presentation.ts @@ -9,7 +9,7 @@ import { type ToolCallDetailLevel, } from "@/tool-calls/detail-level/projection"; -import { createMarkdownProseGrouper } from "@/components/markdown/parts"; +import { groupMarkdownProse } from "@/components/markdown-prose"; interface PresentationInput { tail: StreamItem[]; @@ -58,8 +58,6 @@ function isReusableBlock(block: AssistantMessageItem, source: AssistantMessageIt export function createStreamPresentation({ markdown = "blocks", }: { markdown?: "blocks" | "prose-runs" } = {}) { - const groupMarkdownProse = createMarkdownProseGrouper(); - const rowsByPart = new WeakMap(); const runsBySource = new WeakMap(); const growingRunsBySource = new WeakMap(); const userMessageCache = new WeakMap(); @@ -148,17 +146,6 @@ export function createStreamPresentation({ return blocks; } - function presentMarkdownPart(part: AssistantMessageItem): AssistantMessageItem { - const retained = rowsByPart.get(part); - if (retained) return retained; - const row = { - ...part, - id: part.markdownKind === "prose" ? part.id.replace(":block:", ":run:") : part.id, - }; - rowsByPart.set(part, row); - return row; - } - function presentRows(item: StreamItem, growing = false): StreamItem[] { const blocks = nativeBlocks(item); if (markdown === "blocks" || item.kind !== "assistant_message") return blocks; @@ -168,9 +155,7 @@ export function createStreamPresentation({ const previousSource = liveSources.get(item.id); const previousBlocks = previousSource && blocksBySource.get(previousSource); // nativeBlocks emits only assistant blocks for an assistant source. - const runs = groupMarkdownProse(blocks as AssistantMessageItem[], previousBlocks, growing).map( - presentMarkdownPart, - ); + const runs = groupMarkdownProse(blocks as AssistantMessageItem[], previousBlocks, growing); cache.set(item, runs); return runs; } diff --git a/packages/app/src/assistant-file-links/index.ts b/packages/app/src/assistant-file-links/index.ts index 7cecbfa9b6..723945409f 100644 --- a/packages/app/src/assistant-file-links/index.ts +++ b/packages/app/src/assistant-file-links/index.ts @@ -14,4 +14,4 @@ export { type AssistantFileLinkResolverProviderProps, } from "./provider"; export type { AssistantFileLinkSource } from "./resolver"; -export { useAssistantFileLinkActions, type AssistantFileLinkActions } from "./use-file-link"; +export { useAssistantFileLinkActions } from "./use-file-link"; diff --git a/packages/app/src/components/markdown-prose/index.tsx b/packages/app/src/components/markdown-prose/index.tsx new file mode 100644 index 0000000000..94ceb0d13a --- /dev/null +++ b/packages/app/src/components/markdown-prose/index.tsx @@ -0,0 +1,2 @@ +export { MarkdownProse } from "./renderer"; +export { groupMarkdownProse } from "./runs"; diff --git a/packages/app/src/components/markdown/prose/prepare.test.ts b/packages/app/src/components/markdown-prose/prepare.test.ts similarity index 92% rename from packages/app/src/components/markdown/prose/prepare.test.ts rename to packages/app/src/components/markdown-prose/prepare.test.ts index 62a4293aff..8577890912 100644 --- a/packages/app/src/components/markdown/prose/prepare.test.ts +++ b/packages/app/src/components/markdown-prose/prepare.test.ts @@ -215,20 +215,3 @@ describe("source-preserving prose", () => { expect(opened).toEqual([source]); }); }); - -describe("surface parser policy", () => { - it("leaves plan URLs literal while preserving explicit links", () => { - const prepare = createProsePreparer(noFiles, { linkify: false, allowFileLinks: false }); - expect( - prepare([ - "example.com and https://example.com and `example.com` and [site](https://example.com)", - ]), - ).toBe( - "example\\.com and https\\://example\\.com and `example.com` and [site](https://example.com)", - ); - }); - it("does not promote file URLs outside the assistant parser", () => { - const prepare = createProsePreparer(noFiles, { linkify: true, allowFileLinks: false }); - expect(prepare(["[source](file:///tmp/source.ts)"])).toBe("\\[source](file:///tmp/source.ts)"); - }); -}); diff --git a/packages/app/src/components/markdown/prose/prepare.ts b/packages/app/src/components/markdown-prose/prepare.ts similarity index 94% rename from packages/app/src/components/markdown/prose/prepare.ts rename to packages/app/src/components/markdown-prose/prepare.ts index b428527ccb..b643215489 100644 --- a/packages/app/src/components/markdown/prose/prepare.ts +++ b/packages/app/src/components/markdown-prose/prepare.ts @@ -1,12 +1,6 @@ import type { AssistantFileLinkSource } from "@/assistant-file-links"; -import type { MarkdownReferences } from "../parts"; import type Token from "markdown-it/lib/token.mjs"; -import { - type InlineSpan, - type InlineTrace, - createSourceParser, - type ProseParserOptions, -} from "./source-inline"; +import { type InlineSpan, type InlineTrace, createSourceParser } from "./source-inline"; const settled = createSourceParser(); const INLINE_CODE_SCHEME = "paseo-inline-code:"; @@ -73,11 +67,7 @@ export function prepareProse( } /** One cache per prose row; joining a completed block does not reparse earlier blocks. */ -export function createProsePreparer( - canResolveCode: (text: string) => boolean, - options?: ProseParserOptions, -) { - const sourceParser = options ? createSourceParser(options) : settled; +export function createProsePreparer(canResolveCode: (text: string) => boolean) { let cache = new Map(); return (blocks: readonly string[], visibleLength = Infinity): string => { const retained = new Map(); @@ -86,9 +76,7 @@ export function createProsePreparer( for (const block of blocks) { if (remaining < 0) break; const source = block.slice(0, remaining); - const prepared = - cache.get(source) ?? - prepareBlock(source, canResolveCode, sourceParser, options?.references); + const prepared = cache.get(source) ?? prepareBlock(source, canResolveCode); retained.set(source, prepared); result.push(prepared); remaining -= block.length + 2; @@ -123,7 +111,6 @@ function protect(ranges: Uint8Array, span: { start: number; end: number }) { } function codeUrl(context: InlinePreparation, content: string): string | null { if (context.canResolveCode(content)) return INLINE_CODE_SCHEME + encodeURIComponent(content); - if (!context.parser.options.linkify) return null; const matches = context.parser.linkify.match(content); const match = matches?.length === 1 ? matches[0] : undefined; return match?.index === 0 && match.lastIndex === content.length ? match.url : null; @@ -264,7 +251,6 @@ function prepareText(context: InlinePreparation) { if (contains(context.protectedRanges, i)) continue; const char = context.source[i]!; if ( - (!context.parser.options.linkify && /[:.@]/.test(char)) || char === "|" || (char === "~" && context.source[i - 1] !== "~" && context.source[i + 1] !== "~") ) @@ -297,7 +283,6 @@ function linkifySource(context: InlinePreparation) { return { decoded, positions }; } function prepareLinkifiedLinks(context: InlinePreparation) { - if (!context.parser.options.linkify) return; const links = context.token.children?.filter( (child) => child.type === "link_open" && child.markup === "linkify", @@ -392,12 +377,7 @@ function sourcePositions( } return positions; } -function prepareBlock( - text: string, - canResolveCode: (text: string) => boolean, - sourceParser: ReturnType, - references?: MarkdownReferences, -): string { +function prepareBlock(text: string, canResolveCode: (text: string) => boolean): string { text = text.replace(/\r\n?/g, "\n").replaceAll("\0", "\ufffd"); text = text.replace(/^[ \t>]*/gm, (prefix) => { let expanded = ""; @@ -405,8 +385,8 @@ function prepareBlock( expanded += char === "\t" ? " ".repeat(4 - (expanded.length % 4)) : char; return expanded; }); - const { parser, traces } = sourceParser; - const env = { references }; + const { parser, traces } = settled; + const env = {}; const tokens = parser.parse(text, env); const lines = text.split("\n"); const lineOffsets: number[] = []; diff --git a/packages/app/src/components/markdown-prose/props.ts b/packages/app/src/components/markdown-prose/props.ts new file mode 100644 index 0000000000..19f2d5092e --- /dev/null +++ b/packages/app/src/components/markdown-prose/props.ts @@ -0,0 +1,9 @@ +import type { ReactNode } from "react"; + +export interface MarkdownProseProps { + text: string; + sourceBlocks: readonly string[]; + kind: "prose" | "quote" | "block"; + onLinkPress: (url: string) => boolean; + children: ReactNode; +} diff --git a/packages/app/src/components/markdown/prose/renderer.native.tsx b/packages/app/src/components/markdown-prose/renderer.native.tsx similarity index 65% rename from packages/app/src/components/markdown/prose/renderer.native.tsx rename to packages/app/src/components/markdown-prose/renderer.native.tsx index 6b7b1f1a62..91a2511a83 100644 --- a/packages/app/src/components/markdown/prose/renderer.native.tsx +++ b/packages/app/src/components/markdown-prose/renderer.native.tsx @@ -1,8 +1,9 @@ import { memo, useMemo } from "react"; import { EnrichedMarkdownText, type MarkdownStyle } from "react-native-enriched-markdown"; import { withUnistyles } from "react-native-unistyles"; +import { useAssistantFileLinkActions } from "@/assistant-file-links"; import { useStableEvent } from "@/hooks/use-stable-event"; -import { createCompactMarkdownStyles, createMarkdownStyles } from "@/styles/markdown-styles"; +import { createMarkdownStyles } from "@/styles/markdown-styles"; import type { Theme } from "@/styles/theme"; import type { MarkdownProseProps } from "./props"; import { handleProseLinkPress, createProsePreparer } from "./prepare"; @@ -10,34 +11,32 @@ import { handleProseLinkPress, createProsePreparer } from "./prepare"; const ThemedMarkdown = withUnistyles(EnrichedMarkdownText); const MD4C_FLAGS = { latexMath: false, hardSoftBreaks: true, admonitions: true }; -function proseStyleMapping( - theme: Theme, - compact = false, - blockSpacing = 0, -): { markdownStyle: MarkdownStyle } { - const styles = compact ? createCompactMarkdownStyles(theme) : createMarkdownStyles(theme); +function proseStyleMapping(theme: Theme): { markdownStyle: MarkdownStyle } { + const styles = createMarkdownStyles(theme); const paragraph = { color: styles.body.color, fontSize: styles.body.fontSize, lineHeight: styles.body.lineHeight, marginTop: 0, - marginBottom: styles.paragraph.marginBottom + blockSpacing, + marginBottom: styles.paragraph.marginBottom + theme.spacing[3], }; return { markdownStyle: { paragraph, h1: { ...styles.heading1, - marginBottom: styles.heading1.marginBottom + blockSpacing + styles.heading1.paddingBottom, + marginBottom: + styles.heading1.marginBottom + theme.spacing[3] + styles.heading1.paddingBottom, }, h2: { ...styles.heading2, - marginBottom: styles.heading2.marginBottom + blockSpacing + styles.heading2.paddingBottom, + marginBottom: + styles.heading2.marginBottom + theme.spacing[3] + styles.heading2.paddingBottom, }, - h3: { ...styles.heading3, marginBottom: styles.heading3.marginBottom + blockSpacing }, - h4: { ...styles.heading4, marginBottom: styles.heading4.marginBottom + blockSpacing }, - h5: { ...styles.heading5, marginBottom: styles.heading5.marginBottom + blockSpacing }, - h6: { ...styles.heading6, marginBottom: styles.heading6.marginBottom + blockSpacing }, + h3: { ...styles.heading3, marginBottom: styles.heading3.marginBottom + theme.spacing[3] }, + h4: { ...styles.heading4, marginBottom: styles.heading4.marginBottom + theme.spacing[3] }, + h5: { ...styles.heading5, marginBottom: styles.heading5.marginBottom + theme.spacing[3] }, + h6: { ...styles.heading6, marginBottom: styles.heading6.marginBottom + theme.spacing[3] }, list: { ...paragraph, bulletColor: styles.bullet_list_icon.color, @@ -82,58 +81,40 @@ function proseStyleMapping( }; } +function quoteStyleMapping(theme: Theme): { markdownStyle: MarkdownStyle } { + const mapped = proseStyleMapping(theme); + return { + markdownStyle: { + ...mapped.markdownStyle, + paragraph: { ...mapped.markdownStyle.paragraph, marginBottom: theme.spacing[3] }, + }, + }; +} + +const PROSE_CONTAINER = { marginBottom: -12 }; + const NativeProse = memo(function NativeProse({ text, sourceBlocks, kind, onLinkPress, - compact = false, - blockSpacing = 0, - fileLinks, - references, - linkify = true, - allowFileLinks = false, }: MarkdownProseProps) { - const canResolveCode = useStableEvent( - (content: string) => - fileLinks?.canResolveFile({ href: content, text: content, sourceType: "inline-code" }) ?? - false, - ); - const prepare = useMemo( - () => createProsePreparer(canResolveCode, { linkify, allowFileLinks, references }), - [canResolveCode, linkify, allowFileLinks, references], + const actions = useAssistantFileLinkActions(); + const canResolveCode = useStableEvent((content: string) => + actions.canResolveFile({ href: content, text: content, sourceType: "inline-code" }), ); + const prepare = useMemo(() => createProsePreparer(canResolveCode), [canResolveCode]); const markdown = useMemo( () => prepare(sourceBlocks, text.length), [text.length, sourceBlocks, prepare], ); const handleLinkPress = useStableEvent(({ url }: { url: string }) => { - handleProseLinkPress( - url, - (source) => (fileLinks ? fileLinks.open(source, "preferred") : onLinkPress(source.href)), - onLinkPress, - ); + handleProseLinkPress(url, (source) => actions.open(source, "preferred"), onLinkPress); }); - const uniProps = useMemo( - () => (theme: Theme) => { - const mapped = proseStyleMapping(theme, compact, blockSpacing); - if (kind === "quote") - mapped.markdownStyle.paragraph = { - ...mapped.markdownStyle.paragraph, - marginBottom: theme.spacing[3], - }; - return mapped; - }, - [kind, compact, blockSpacing], - ); - const containerStyle = useMemo( - () => (kind === "prose" ? { marginBottom: -blockSpacing } : undefined), - [kind, blockSpacing], - ); return ( (); + +function splitBlock(block: AssistantMessageItem): AssistantMessageItem[] { + const cached = partsByBlock.get(block); + if (cached) return cached; + // Oversized blocks cannot join capped runs; the existing renderer caps text before parsing. + if (block.text.length > ASSISTANT_MESSAGE_RENDER_CHARACTER_LIMIT) { + return [{ ...block, markdownKind: "block" }]; + } + const environment: { references?: unknown } = {}; + const tokens = parser.parse(block.text, environment); + const preserved = tokens.some( + (token) => + ["fence", "code_block", "table_open", "html_block"].includes(token.type) || + token.children?.some((child) => child.type === "html_inline"), + ); + const isQuote = tokens.some((token) => token.type === "blockquote_open"); + const hasImage = tokens.some((token) => token.children?.some((child) => child.type === "image")); + let parts: AssistantMessageItem[]; + // Retain the container and its definitions when splitting would lose image context. + if (preserved || (hasImage && (environment.references || !canSplitParagraphImages(tokens)))) { + parts = [{ ...block, markdownKind: "block" }]; + } else if (hasImage) { + parts = splitParagraphImages(block); + } else { + parts = [{ ...block, markdownKind: isQuote ? "quote" : "prose" }]; + } + partsByBlock.set(block, parts); + return parts; +} + +function canSplitParagraphImages(tokens: Token[]): boolean { + if (tokens.length !== 3 || tokens[0]?.type !== "paragraph_open") return false; + let depth = 0; + for (const child of tokens[1]!.children ?? []) { + if (child.type === "image" && depth !== 0) return false; + depth += child.nesting; + } + return true; +} + +function splitParagraphImages(block: AssistantMessageItem): AssistantMessageItem[] { + const parts: AssistantMessageItem[] = []; + const state = new parser.inline.State(block.text, parser, {}, []); + let cursor = 0; + function append(text: string, markdownKind: "prose" | "block", offset: number) { + if (!text.trim()) return; + parts.push({ ...block, text: text.trim(), markdownKind, id: `${block.id}:part:${offset}` }); + } + while (state.pos < state.posMax) { + const start = state.pos; + parser.inline.skipToken(state); + const candidate = block.text.slice(start, state.pos); + if (!candidate.includes("![")) continue; + const inline = parser.parseInline(candidate, {})[0]; + if (!inline?.children?.some((child) => child.type === "image")) continue; + append(block.text.slice(cursor, start), "prose", cursor); + append(candidate, "block", start); + cursor = state.pos; + } + append(block.text.slice(cursor), "prose", cursor); + if (parts.length === 1) parts[0] = { ...parts[0]!, id: block.id }; + return parts; +} + +interface GroupedRuns { + growing: boolean; + rows: AssistantMessageItem[]; + starts: number[]; + ends: number[]; +} +const runsByBlocks = new WeakMap(); +const growingRunsByBlocks = new WeakMap(); + +/** Rebuild only the run containing the changed source block; completed runs retain identity. */ +export function groupMarkdownProse( + blocks: readonly AssistantMessageItem[], + previousBlocks?: readonly AssistantMessageItem[], + growing = false, +): AssistantMessageItem[] { + const cache = growing ? growingRunsByBlocks : runsByBlocks; + const cached = cache.get(blocks); + if (cached) return cached.rows; + const previous = previousGroups(previousBlocks); + const retained = retainedRunCount(blocks, previousBlocks, previous, growing); + const start = previous?.starts[retained] ?? 0; + const rows = previous?.rows.slice(0, retained) ?? []; + const starts = previous?.starts.slice(0, retained) ?? []; + const ends = previous?.ends.slice(0, retained) ?? []; + let parts: AssistantMessageItem[] = []; + let length = 0; + let runStart = start; + let runEnd = start; + function flush() { + if (!parts.length) return; + const first = parts[0]!; + const texts = parts.map((part) => part.text); + rows.push({ + ...first, + text: texts.join("\n\n"), + markdownProseBlocks: first.markdownKind === "block" ? undefined : texts, + id: first.markdownKind === "prose" ? first.id.replace(":block:", ":run:") : first.id, + }); + starts.push(runStart); + ends.push(runEnd); + parts = []; + length = 0; + } + for (let index = start; index < blocks.length; index++) { + const block = blocks[index]!; + const partsForBlock = + growing && index === blocks.length - 1 + ? [{ ...block, markdownKind: "block" as const }] + : splitBlock(block); + for (const part of partsForBlock) { + if (parts.length && !canJoinRun(parts[0]!, part, length)) flush(); + if (!parts.length) runStart = index; + length += (parts.length ? 2 : 0) + part.text.length; + parts.push(part); + runEnd = index; + } + } + flush(); + reuseUnchangedRows(blocks, previousBlocks, previous, rows, starts, ends, retained); + cache.set(blocks, { rows, starts, ends, growing }); + return rows; +} + +function retainedRunCount( + blocks: readonly AssistantMessageItem[], + previousBlocks: readonly AssistantMessageItem[] | undefined, + previous: GroupedRuns | undefined, + growing: boolean, +): number { + if (!previous || !previousBlocks) return 0; + let shared = 0; + while (shared < blocks.length && blocks[shared] === previousBlocks[shared]) shared++; + if (previous.growing && (!growing || blocks.length !== previousBlocks.length)) + shared = Math.min(shared, previousBlocks.length - 1); + let retained = 0; + while (retained < previous.rows.length && previous.ends[retained]! < shared) retained++; + // Only completed blocks can merge with the preceding run. The live tail is a + // hard boundary, so appending to it never rebuilds completed prose. + if (!(growing && shared === blocks.length - 1)) retained = Math.max(0, retained - 1); + while (retained > 0 && previous.ends[retained - 1]! >= (previous.starts[retained] ?? shared)) + retained--; + return retained; +} +function reuseUnchangedRows( + blocks: readonly AssistantMessageItem[], + previousBlocks: readonly AssistantMessageItem[] | undefined, + previous: GroupedRuns | undefined, + rows: AssistantMessageItem[], + starts: number[], + ends: number[], + retained: number, +) { + if (!previous || !previousBlocks) return; + for (let index = retained; index < rows.length; index++) { + const old = previous.rows[index]; + const row = rows[index]!; + if (old?.id !== row.id || old.text !== row.text || old.markdownKind !== row.markdownKind) + continue; + const end = ends[index]!; + let identical = true; + for (let block = starts[index]!; block <= end; block++) + if (blocks[block] !== previousBlocks[block]) { + identical = false; + break; + } + if (!identical) continue; + rows[index] = old; + } +} + +function canJoinRun( + first: AssistantMessageItem, + next: AssistantMessageItem, + length: number, +): boolean { + return ( + first.markdownKind === "prose" && + next.markdownKind === "prose" && + length + 2 + next.text.length <= ASSISTANT_MESSAGE_RENDER_CHARACTER_LIMIT + ); +} + +function previousGroups( + blocks: readonly AssistantMessageItem[] | undefined, +): GroupedRuns | undefined { + return blocks && (growingRunsByBlocks.get(blocks) ?? runsByBlocks.get(blocks)); +} diff --git a/packages/app/src/components/markdown/prose/source-inline.ts b/packages/app/src/components/markdown-prose/source-inline.ts similarity index 80% rename from packages/app/src/components/markdown/prose/source-inline.ts rename to packages/app/src/components/markdown-prose/source-inline.ts index 73bbcea617..f311fbfee9 100644 --- a/packages/app/src/components/markdown/prose/source-inline.ts +++ b/packages/app/src/components/markdown-prose/source-inline.ts @@ -1,5 +1,4 @@ import { createAssistantMarkdownParser } from "@/utils/assistant-markdown-parser"; -import { createMarkdownParser } from "@/utils/markdown-parser"; import type Token from "markdown-it/lib/token.mjs"; export interface InlineSpan { @@ -15,20 +14,8 @@ export interface InlineTrace { const TRACE = Symbol("prose source positions"); /** Observe the actual inline rules, including nested labels, without reprinting tokens. */ -import type { MarkdownReferences } from "../parts"; - -export interface ProseParserOptions { - references?: MarkdownReferences; - linkify: boolean; - allowFileLinks: boolean; -} -export function createSourceParser( - options: ProseParserOptions = { linkify: true, allowFileLinks: true }, -) { - const parser = options.allowFileLinks - ? createAssistantMarkdownParser() - : createMarkdownParser({ linkify: options.linkify }); - parser.set({ linkify: options.linkify }); +export function createSourceParser() { + const parser = createAssistantMarkdownParser(); const traces = new WeakMap(); const inlineTraces = new WeakMap(); const parseInline = parser.inline.parse.bind(parser.inline); diff --git a/packages/app/src/components/markdown/part-groups.test.ts b/packages/app/src/components/markdown/part-groups.test.ts index 0f9876e249..3b2b54e044 100644 --- a/packages/app/src/components/markdown/part-groups.test.ts +++ b/packages/app/src/components/markdown/part-groups.test.ts @@ -1,9 +1,6 @@ import { describe, expect, it } from "vitest"; import type { MarkdownDisplayPart } from "./html-ish"; -import { groupMarkdownParts, prepareMarkdownGroups } from "./part-groups"; -import { parseMarkdownPreviewDocument } from "@/file-pane/markdown-preview/document"; -import { createProsePreparer } from "./prose/prepare"; -import { splitMarkdownProse } from "./parts"; +import { groupMarkdownParts } from "./part-groups"; describe("groupMarkdownParts", () => { const flowImage: MarkdownDisplayPart = { @@ -93,62 +90,3 @@ describe("groupMarkdownParts", () => { ]); }); }); - -describe("shared renderer prose parts", () => { - it("groups paragraphs and retains code and quote boundaries for a document", () => { - const parts = splitMarkdownProse( - "First paragraph.\n\nSecond paragraph.\n\n```ts\nconst x = 1;\n```\n\n> Quote one.\n>\n> Quote two.", - ); - expect(parts.map((part) => part.markdownKind)).toEqual(["prose", "block", "quote"]); - expect(parts[0]?.markdownProseBlocks).toEqual(["First paragraph.", "Second paragraph."]); - }); -}); - -describe("Markdown file preview document", () => { - it("uses shared prose parts after removing front matter", () => { - const document = parseMarkdownPreviewDocument( - "---\ntitle: Preview\n---\n\nFirst paragraph.\n\nSecond paragraph.\n\n```js\n42\n```\n\n> Quote", - ); - expect(splitMarkdownProse(document.body).map((part) => part.markdownKind)).toEqual([ - "prose", - "block", - "quote", - ]); - }); - it("retains document-wide references in prose and image blocks", () => { - const parts = splitMarkdownProse( - "[site][ref]\n\n![image][pic]\n\nOther text.\n\n[ref]: https://example.com\n[pic]: image.png", - ); - expect(parts.map((part) => part.markdownKind)).toEqual(["prose", "block", "prose"]); - const first = parts[0]!; - const prepare = createProsePreparer(() => false, { - linkify: true, - allowFileLinks: false, - references: first.references, - }); - expect(prepare(first.markdownProseBlocks!)).toBe("[site]()"); - expect(parts[1]?.references?.PIC?.href).toBe("image.png"); - }); -}); - -describe("renderer container ordering", () => { - it("retains a collapsible before preparing its prose", () => { - const groups = prepareMarkdownGroups({ - text: "Before.\n\n
\nMore\n\nFirst paragraph.\n\nSecond paragraph.\n\n
\n\nAfter.", - enableHtmlish: true, - }); - expect(groups.map((group) => (group.kind === "part" ? group.part.kind : group.kind))).toEqual([ - "markdown", - "details", - "markdown", - ]); - const details = groups[1]; - expect(details?.kind).toBe("part"); - if (details?.kind !== "part" || details.part.kind !== "details") - throw new Error("Missing collapsible"); - expect(details.part.summary).toBe("More"); - expect(splitMarkdownProse(details.part.body).map((part) => part.markdownKind)).toEqual([ - "prose", - ]); - }); -}); diff --git a/packages/app/src/components/markdown/part-groups.ts b/packages/app/src/components/markdown/part-groups.ts index 69b172a60d..016152fb00 100644 --- a/packages/app/src/components/markdown/part-groups.ts +++ b/packages/app/src/components/markdown/part-groups.ts @@ -1,25 +1,9 @@ -import { - splitHtmlishMarkdown, - type MarkdownDisplayPart, - type MarkdownInlineImagePart, -} from "./html-ish"; +import type { MarkdownDisplayPart, MarkdownInlineImagePart } from "./html-ish"; export type MarkdownPartGroup = | { kind: "part"; part: MarkdownDisplayPart } | { kind: "imageText"; images: MarkdownInlineImagePart[]; lead: string; rest: string }; -/** Parse containers before splitting their Markdown text into selectable runs. */ -export function prepareMarkdownGroups({ - text, - enableHtmlish, -}: { - text: string; - enableHtmlish: boolean; -}): MarkdownPartGroup[] { - const parts = enableHtmlish ? splitHtmlishMarkdown(text) : [{ kind: "markdown" as const, text }]; - return groupMarkdownParts(parts); -} - /** * Pairs a run of inline images that flow with text (small status images at the * start of a line, GitHub-style) with the lead paragraph of the following diff --git a/packages/app/src/components/markdown/parts/index.ts b/packages/app/src/components/markdown/parts/index.ts deleted file mode 100644 index 4e09370a84..0000000000 --- a/packages/app/src/components/markdown/parts/index.ts +++ /dev/null @@ -1,229 +0,0 @@ -import { ASSISTANT_MESSAGE_RENDER_CHARACTER_LIMIT } from "@/components/assistant-message-render-limit"; -import { createAssistantMarkdownParser } from "@/utils/assistant-markdown-parser"; -import MarkdownIt from "markdown-it"; -import type Token from "markdown-it/lib/token.mjs"; -import { splitMarkdownBlocks } from "@/utils/split-markdown-blocks"; - -export interface MarkdownReferences { - [label: string]: { href: string; title: string }; -} -export interface MarkdownSource { - id: string; - text: string; -} -export interface MarkdownProsePart extends MarkdownSource { - markdownKind: "prose" | "quote" | "block"; - markdownProseBlocks?: readonly string[]; - references?: MarkdownReferences; -} - -export function createMarkdownProseGrouper( - references?: MarkdownReferences, -) { - type Part = T & MarkdownProsePart; - - const parser = createAssistantMarkdownParser().set({ html: true }); - const partsByBlock = new WeakMap(); - - function splitBlock(block: T): Part[] { - const cached = partsByBlock.get(block); - if (cached) return cached; - // Oversized blocks cannot join capped runs; the existing renderer caps text before parsing. - if (block.text.length > ASSISTANT_MESSAGE_RENDER_CHARACTER_LIMIT) { - return [{ ...block, markdownKind: "block" }]; - } - const environment: { references?: MarkdownReferences } = { references }; - const tokens = parser.parse(block.text, environment); - const preserved = tokens.some( - (token) => - ["fence", "code_block", "table_open", "html_block"].includes(token.type) || - token.children?.some((child) => child.type === "html_inline"), - ); - const isQuote = tokens.some((token) => token.type === "blockquote_open"); - const hasImage = tokens.some((token) => - token.children?.some((child) => child.type === "image"), - ); - let parts: Part[]; - // Retain the container and its definitions when splitting would lose image context. - if (preserved || (hasImage && (environment.references || !canSplitParagraphImages(tokens)))) { - parts = [{ ...block, markdownKind: "block" }]; - } else if (hasImage) { - parts = splitParagraphImages(block); - } else { - parts = [{ ...block, markdownKind: isQuote ? "quote" : "prose" }]; - } - partsByBlock.set(block, parts); - return parts; - } - - function canSplitParagraphImages(tokens: Token[]): boolean { - if (tokens.length !== 3 || tokens[0]?.type !== "paragraph_open") return false; - let depth = 0; - for (const child of tokens[1]!.children ?? []) { - if (child.type === "image" && depth !== 0) return false; - depth += child.nesting; - } - return true; - } - - function splitParagraphImages(block: T): Part[] { - const parts: Part[] = []; - const state = new parser.inline.State(block.text, parser, {}, []); - let cursor = 0; - function append(text: string, markdownKind: "prose" | "block", offset: number) { - if (!text.trim()) return; - parts.push({ ...block, text: text.trim(), markdownKind, id: `${block.id}:part:${offset}` }); - } - while (state.pos < state.posMax) { - const start = state.pos; - parser.inline.skipToken(state); - const candidate = block.text.slice(start, state.pos); - if (!candidate.includes("![")) continue; - const inline = parser.parseInline(candidate, {})[0]; - if (!inline?.children?.some((child) => child.type === "image")) continue; - append(block.text.slice(cursor, start), "prose", cursor); - append(candidate, "block", start); - cursor = state.pos; - } - append(block.text.slice(cursor), "prose", cursor); - if (parts.length === 1) parts[0] = { ...parts[0]!, id: block.id }; - return parts; - } - - interface GroupedRuns { - growing: boolean; - rows: Part[]; - starts: number[]; - ends: number[]; - } - const runsByBlocks = new WeakMap(); - const growingRunsByBlocks = new WeakMap(); - - /** Rebuild only the run containing the changed source block; completed runs retain identity. */ - function groupMarkdownProse( - blocks: readonly T[], - previousBlocks?: readonly T[], - growing = false, - ): Part[] { - const cache = growing ? growingRunsByBlocks : runsByBlocks; - const cached = cache.get(blocks); - if (cached) return cached.rows; - const previous = previousGroups(previousBlocks); - const retained = retainedRunCount(blocks, previousBlocks, previous, growing); - const start = previous?.starts[retained] ?? 0; - const rows = previous?.rows.slice(0, retained) ?? []; - const starts = previous?.starts.slice(0, retained) ?? []; - const ends = previous?.ends.slice(0, retained) ?? []; - let parts: Part[] = []; - let length = 0; - let runStart = start; - let runEnd = start; - function flush() { - if (!parts.length) return; - const first = parts[0]!; - const texts = parts.map((part) => part.text); - rows.push({ - ...first, - text: texts.join("\n\n"), - markdownProseBlocks: first.markdownKind === "block" ? undefined : texts, - id: first.id, - references, - }); - starts.push(runStart); - ends.push(runEnd); - parts = []; - length = 0; - } - for (let index = start; index < blocks.length; index++) { - const block = blocks[index]!; - const partsForBlock = - growing && index === blocks.length - 1 - ? [{ ...block, markdownKind: "block" as const }] - : splitBlock(block); - for (const part of partsForBlock) { - if (parts.length && !canJoinRun(parts[0]!, part, length)) flush(); - if (!parts.length) runStart = index; - length += (parts.length ? 2 : 0) + part.text.length; - parts.push(part); - runEnd = index; - } - } - flush(); - reuseUnchangedRows(blocks, previousBlocks, previous, rows, starts, ends, retained); - cache.set(blocks, { rows, starts, ends, growing }); - return rows; - } - - function retainedRunCount( - blocks: readonly T[], - previousBlocks: readonly T[] | undefined, - previous: GroupedRuns | undefined, - growing: boolean, - ): number { - if (!previous || !previousBlocks) return 0; - let shared = 0; - while (shared < blocks.length && blocks[shared] === previousBlocks[shared]) shared++; - if (previous.growing && (!growing || blocks.length !== previousBlocks.length)) - shared = Math.min(shared, previousBlocks.length - 1); - let retained = 0; - while (retained < previous.rows.length && previous.ends[retained]! < shared) retained++; - // Only completed blocks can merge with the preceding run. The live tail is a - // hard boundary, so appending to it never rebuilds completed prose. - if (!(growing && shared === blocks.length - 1)) retained = Math.max(0, retained - 1); - while (retained > 0 && previous.ends[retained - 1]! >= (previous.starts[retained] ?? shared)) - retained--; - return retained; - } - function reuseUnchangedRows( - blocks: readonly T[], - previousBlocks: readonly T[] | undefined, - previous: GroupedRuns | undefined, - rows: Part[], - starts: number[], - ends: number[], - retained: number, - ) { - if (!previous || !previousBlocks) return; - for (let index = retained; index < rows.length; index++) { - const old = previous.rows[index]; - const row = rows[index]!; - if (old?.id !== row.id || old.text !== row.text || old.markdownKind !== row.markdownKind) - continue; - const end = ends[index]!; - let identical = true; - for (let block = starts[index]!; block <= end; block++) - if (blocks[block] !== previousBlocks[block]) { - identical = false; - break; - } - if (!identical) continue; - rows[index] = old; - } - } - - function canJoinRun(first: Part, next: Part, length: number): boolean { - return ( - first.markdownKind === "prose" && - next.markdownKind === "prose" && - length + 2 + next.text.length <= ASSISTANT_MESSAGE_RENDER_CHARACTER_LIMIT - ); - } - - function previousGroups(blocks: readonly T[] | undefined): GroupedRuns | undefined { - return blocks && (growingRunsByBlocks.get(blocks) ?? runsByBlocks.get(blocks)); - } - - return groupMarkdownProse; -} -const documentParser = new MarkdownIt(); -documentParser.core.ruler.disable("inline"); -export function splitMarkdownProse(text: string): MarkdownProsePart[] { - const environment: { references?: MarkdownReferences } = {}; - documentParser.parse(text, environment); - const groupDocument = createMarkdownProseGrouper(environment.references); - const blocks = splitMarkdownBlocks(text).map((blockText, index) => ({ - id: String(index), - text: blockText, - })); - return groupDocument(blocks); -} diff --git a/packages/app/src/components/markdown/prose/props.ts b/packages/app/src/components/markdown/prose/props.ts deleted file mode 100644 index ee3db840cc..0000000000 --- a/packages/app/src/components/markdown/prose/props.ts +++ /dev/null @@ -1,18 +0,0 @@ -import type { ReactNode } from "react"; -import type { AssistantFileLinkActions } from "@/assistant-file-links"; - -import type { MarkdownReferences } from "../parts"; - -export interface MarkdownProseProps { - text: string; - sourceBlocks: readonly string[]; - kind: "prose" | "quote" | "block"; - onLinkPress: (url: string) => boolean; - children: ReactNode; - references?: MarkdownReferences; - compact?: boolean; - blockSpacing?: number; - fileLinks?: AssistantFileLinkActions; - linkify?: boolean; - allowFileLinks?: boolean; -} diff --git a/packages/app/src/components/markdown/renderer.tsx b/packages/app/src/components/markdown/renderer.tsx index 13e95285ef..83380ea919 100644 --- a/packages/app/src/components/markdown/renderer.tsx +++ b/packages/app/src/components/markdown/renderer.tsx @@ -33,18 +33,19 @@ import { createCompactMarkdownStyles, createMarkdownStyles } from "@/styles/mark import type { Theme } from "@/styles/theme"; import { openExternalUrl } from "@/utils/open-external-url"; import { isNative } from "@/constants/platform"; -import { type MarkdownDisplayPart, type MarkdownInlineImagePart } from "./html-ish"; +import { + splitHtmlishMarkdown, + type MarkdownDisplayPart, + type MarkdownInlineImagePart, +} from "./html-ish"; import { resolveInlineImageSize, type InlineImageDimensions } from "./inline-image-size"; import { getAssistantImageMetadata, setAssistantImageMetadata, } from "@/utils/assistant-image-metadata"; -import { groupMarkdownParts, prepareMarkdownGroups, type MarkdownPartGroup } from "./part-groups"; +import { groupMarkdownParts, type MarkdownPartGroup } from "./part-groups"; import { colorMarkdownLinkChildren } from "./link-children"; import { MarkdownLinkText } from "./link-text"; -import { MarkdownProse } from "./prose/renderer"; -import { splitMarkdownProse, type MarkdownProsePart } from "./parts"; -import type { AssistantFileLinkActions } from "@/assistant-file-links"; export type MarkdownStyles = Record; @@ -83,83 +84,9 @@ export interface MarkdownRendererProps { allowedImageHandlers?: readonly string[]; topLevelMaxExceededItem?: ReactNode; enableHtmlish?: boolean; - /** A virtualized caller supplies an already grouped part, including the live block. */ - part?: Pick; - blockSpacing?: number; - fileLinks?: AssistantFileLinkActions; } -export function MarkdownRenderer(props: MarkdownRendererProps) { - if (isNative && props.part) return ; - return ; -} - -const BLOCK_PART = { markdownKind: "block" as const }; - -function MarkdownRuns(props: MarkdownRendererProps) { - const parts = useMemo(() => { - if (props.part) return [{ ...props.part, text: props.text, id: "part" }]; - return splitMarkdownProse(props.text); - }, [props.part, props.text]); - const parser = props.markdownit ?? defaultMarkdownParser; - return ( - <> - {parts.map((part) => ( - - - - ))} - - ); -} - -function MarkdownScopedBlock({ - references, - ...props -}: MarkdownRendererProps & Pick) { - const parser = props.markdownit ?? defaultMarkdownParser; - // markdown-display supplies its own environment. Preserve the document's reference - // scope while still using the caller's parser rules for each retained block. - const scopedParser = useMemo( - () => - references - ? new Proxy(parser, { - get(target, key, receiver) { - if (key === "parse") - return (source: string, env: object) => - target.parse(source, { ...env, references }); - return Reflect.get(target, key, receiver); - }, - }) - : parser, - [parser, references], - ); - return ; -} - -function openMarkdownLink(url: string): boolean { - void openExternalUrl(url); - return false; -} - -function MarkdownBlockRenderer({ +export function MarkdownRenderer({ text, compact = false, rules, @@ -168,11 +95,10 @@ function MarkdownBlockRenderer({ allowedImageHandlers, topLevelMaxExceededItem, enableHtmlish = true, - part, }: MarkdownRendererProps) { const markdownRules = useMemo(() => rules ?? createSharedMarkdownRules(), [rules]); - const groups = useMemo( - () => prepareMarkdownGroups({ text, enableHtmlish }), + const parts = useMemo( + () => (enableHtmlish ? splitHtmlishMarkdown(text) : [{ kind: "markdown" as const, text }]), [enableHtmlish, text], ); const rendererProps = useMemo( @@ -183,7 +109,6 @@ function MarkdownBlockRenderer({ onLinkPress, allowedImageHandlers, topLevelMaxExceededItem, - part, }), [ allowedImageHandlers, @@ -192,25 +117,24 @@ function MarkdownBlockRenderer({ markdownit, onLinkPress, topLevelMaxExceededItem, - part, ], ); - return ; + return ; } type MarkdownPartRendererProps = Omit & { rules: RenderRules; }; -function MarkdownGroupList({ - groups, +function MarkdownPartList({ + parts, rendererProps, }: { - groups: MarkdownPartGroup[]; + parts: MarkdownDisplayPart[]; rendererProps: MarkdownPartRendererProps; }) { - const keyedGroups = useMemo(() => keyMarkdownGroups(groups), [groups]); + const keyedGroups = useMemo(() => keyMarkdownGroups(groupMarkdownParts(parts)), [parts]); return ( <> {keyedGroups.map(({ key, group }) => @@ -252,12 +176,7 @@ function MarkdownPart({ return ; } -function MarkdownFragment(props: MarkdownRendererProps & { rules: RenderRules }) { - if (isNative && !props.part) return ; - return ; -} - -function MarkdownBlockFragment({ +function MarkdownFragment({ text, compact, rules, @@ -457,8 +376,8 @@ function MarkdownDetails({ }) { const [open, setOpen] = useState(false); const toggle = useCallback(() => setOpen((current) => !current), []); - const bodyGroups = useMemo( - () => groupMarkdownParts(part.bodyParts ?? [{ kind: "markdown" as const, text: part.body }]), + const bodyParts = useMemo( + () => part.bodyParts ?? [{ kind: "markdown" as const, text: part.body }], [part.body, part.bodyParts], ); return ( @@ -473,7 +392,7 @@ function MarkdownDetails({ {open ? ( - + ) : null} diff --git a/packages/app/src/components/message.tsx b/packages/app/src/components/message.tsx index fad33832aa..722f6e7b31 100644 --- a/packages/app/src/components/message.tsx +++ b/packages/app/src/components/message.tsx @@ -1,3 +1,4 @@ +import { MarkdownProse } from "@/components/markdown-prose"; import type { AgentMessage } from "@getpaseo/protocol/agent-message"; import { ASSISTANT_IMAGE_DEFAULT_ASPECT_RATIO } from "@/utils/assistant-image-metadata"; import { LoadingSpinner } from "@/components/ui/loading-spinner"; @@ -1392,8 +1393,6 @@ function AssistantMessageBlockContainer({ ); } -const BLOCK_MARKDOWN_PART = { markdownKind: "block" as const }; - interface MemoizedMarkdownBlockProps { text: string; rules: RenderRules; @@ -1410,7 +1409,6 @@ const MemoizedMarkdownBlock = React.memo(function MemoizedMarkdownBlock({ return ( ({ markdownKind, markdownProseBlocks: markdownProseBlocks ?? [message] }), - [markdownKind, markdownProseBlocks, message], - ); const blocks = useMemo( () => (markdownKind === "block" ? splitMarkdownBlocks(revealedMessage) : []), [revealedMessage, markdownKind], @@ -2003,37 +1997,31 @@ export const AssistantMessage = memo(function AssistantMessage({ return ( - {markdownKind !== "block" ? ( - - ) : ( - <> - {keyedBlocks.map(({ key, block }, index) => ( - - - - ))} - - )} + + {keyedBlocks.map(({ key, block }, index) => ( + + + + ))} + {fullMessageByteLength !== null ? (