mirror of
https://github.com/getpaseo/paseo.git
synced 2026-10-28 04:03:05 +00:00
fix(app): preserve markdown source and file actions in native prose
Prepare original source blocks with source-position edits from the assistant parser. Keep reference scope and live/history projection consistent, and retain link text when dispatching workspace file actions.
This commit is contained in:
Mohamed Boudra
@@ -34,9 +34,10 @@ So arrival sets a _target_ and the reveal rate is derived from the backlog inste
|
||||
display. Native groups adjacent prose into selectable runs; web retains individual blocks.
|
||||
Code, tables, images, HTML-like content and blockquotes end a native run. See
|
||||
[native text selection](native-text-selection.md) for renderer ownership and library constraints.
|
||||
Appends parse only the growing last source block. Completed source blocks retain their parsed
|
||||
classification, completed display rows retain object identity, and a source-item cache avoids
|
||||
re-splitting history. Streamed and fetched messages use the same projection.
|
||||
Completing a reference definition can change an earlier block boundary. Live and history use
|
||||
the same structural parse; unchanged blocks retain their classification and completed rows
|
||||
retain object identity. A source-item cache avoids re-splitting unchanged history. Prose rows
|
||||
retain the original source blocks so native preparation cannot broaden reference scope.
|
||||
- **A row id is never a message id.** Block rows are `${messageId}:block:${n}`; native prose runs use `${messageId}:run:${n}` with
|
||||
`blockGroupId = messageId`, including single-row messages. Inline images split a source block into offset-keyed parts, so nothing can come to rely on the two
|
||||
being equal. `getStreamItemMessageId` in `presentation.ts` is the only way to go from a row to its
|
||||
|
||||
@@ -124,6 +124,44 @@ function rows(result: { tail: StreamItem[]; head: StreamItem[] }): StreamItem[]
|
||||
}
|
||||
|
||||
describe("native prose runs", () => {
|
||||
it("projects a completed streamed reference definition like fresh history", () => {
|
||||
const present = createStreamPresentation({ markdown: "prose-runs" });
|
||||
let state: { tail: StreamItem[]; head: StreamItem[] } = { tail: [], head: [] };
|
||||
for (const text of ["![shot][pic]\n\n[pic]:", " shots/app.png"]) {
|
||||
state = applyStreamEvent({ ...state, event: assistant(text), timestamp: new Date(1000) });
|
||||
present({ ...presentationOptions, ...state, transform: undefined });
|
||||
}
|
||||
const live = present({ ...presentationOptions, ...state, transform: undefined });
|
||||
const history = createStreamPresentation({ markdown: "prose-runs" })({
|
||||
...presentationOptions,
|
||||
tail: hydrateStreamState([
|
||||
{ event: assistant("![shot][pic]\n\n[pic]: shots/app.png"), timestamp: new Date(1000) },
|
||||
]),
|
||||
head: [],
|
||||
transform: undefined,
|
||||
});
|
||||
expect([...live.tail, ...live.head]).toEqual(history.tail);
|
||||
});
|
||||
it("keeps file URL images in Paseo's renderer", () => {
|
||||
const source = hydrateStreamState([
|
||||
{
|
||||
event: assistant("Before  after."),
|
||||
timestamp: new Date(1000),
|
||||
},
|
||||
]);
|
||||
const result = createStreamPresentation({ markdown: "prose-runs" })({
|
||||
...presentationOptions,
|
||||
tail: source,
|
||||
head: [],
|
||||
transform: undefined,
|
||||
});
|
||||
expect(result.tail).toMatchObject([
|
||||
{ text: "Before", markdownKind: "prose" },
|
||||
{ text: "", markdownKind: "block" },
|
||||
{ text: "after.", markdownKind: "prose" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("starts another run before combined paragraphs exceed the render cap", () => {
|
||||
const first = "a".repeat(20_000);
|
||||
const second = "b".repeat(20_000);
|
||||
|
||||
@@ -102,23 +102,13 @@ export function createStreamPresentation({
|
||||
if (cached) return cached;
|
||||
const previousSource = liveSources.get(item.id);
|
||||
const previous = previousSource && blocksBySource.get(previousSource);
|
||||
// Parse only the growing last block on append, as the old live reducer did.
|
||||
// Canonical text replacements are parsed afresh instead of joining fragments.
|
||||
const isAppend = previousSource && previous && item.text.startsWith(previousSource.text);
|
||||
let prefix: AssistantMessageItem[] = [];
|
||||
let growingText = item.text;
|
||||
if (isAppend) {
|
||||
prefix = previous.slice(0, -1);
|
||||
growingText =
|
||||
previous[previous.length - 1]!.text + item.text.slice(previousSource.text.length);
|
||||
}
|
||||
const parsed = splitMarkdownBlocks(growingText);
|
||||
// Whitespace-only text has no block, and a message still owns exactly one row.
|
||||
const textBlocks = parsed.length > 0 || prefix.length > 0 ? parsed : [""];
|
||||
|
||||
const blocks = [...prefix];
|
||||
// Completing a reference definition can join a preceding block. Use the same
|
||||
// structural parse for live and history, then retain unchanged block objects.
|
||||
const parsed = splitMarkdownBlocks(item.text);
|
||||
const textBlocks = parsed.length > 0 ? parsed : [""];
|
||||
const blocks: AssistantMessageItem[] = [];
|
||||
for (const [offset, text] of textBlocks.entries()) {
|
||||
const index = prefix.length + offset;
|
||||
const index = offset;
|
||||
let blockText = text;
|
||||
if (offset === textBlocks.length - 1) {
|
||||
// The split drops trailing blank lines, including a line break followed only
|
||||
|
||||
@@ -730,6 +730,7 @@ const AgentStreamViewComponent = forwardRef<AgentStreamViewHandle, AgentStreamVi
|
||||
{(renderFullContent) => (
|
||||
<AssistantMessage
|
||||
markdownKind={item.markdownKind}
|
||||
markdownProseBlocks={item.markdownProseBlocks}
|
||||
renderFullContent={renderFullContent}
|
||||
occurrenceKey={createAssistantImageOccurrenceKey({ agentId, itemId: item.id })}
|
||||
message={item.text}
|
||||
|
||||
@@ -1,114 +1,111 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createAssistantMarkdownParser } from "@/utils/assistant-markdown-parser";
|
||||
import { inlineCodeLinkContent, prepareProse } from "./prepare";
|
||||
import {
|
||||
prepareProse,
|
||||
inlineCodeLinkContent,
|
||||
proseLinkSource,
|
||||
handleProseLinkPress,
|
||||
} from "./prepare";
|
||||
|
||||
const parser = createAssistantMarkdownParser();
|
||||
const noFiles = () => false;
|
||||
|
||||
describe("native prose dialect", () => {
|
||||
describe("source-preserving prose", () => {
|
||||
it.each([
|
||||
"A & B",
|
||||
"# A & B",
|
||||
"- A & B",
|
||||
"> A & B",
|
||||
"Literal &amp; and &copy;",
|
||||
"Escaped \\* and \\|",
|
||||
"<https://example.org>",
|
||||
"**bold** and *emphasis* and ~~deleted~~",
|
||||
"First \nSecond",
|
||||
"# #",
|
||||
"Setext heading\n==============",
|
||||
"Heading\n=======",
|
||||
'A [titled link](https://example.com "A title")',
|
||||
"- first\n\tsecond",
|
||||
"[linked `code`](https://example.net)",
|
||||
"Before  after.",
|
||||
"Blank ` ` code and ` normal ` code",
|
||||
"# Heading\n\n- **First**\n- Second\n\n~~Deleted~~",
|
||||
"> First paragraph\n>\n> Second paragraph",
|
||||
"A [linked `path.ts`](file:///repo/path.ts)",
|
||||
])("preserves existing rendered inline content: %s", (text) => {
|
||||
expect(parser.render(prepareProse(text, noFiles))).toBe(parser.render(text));
|
||||
"- first\n\tsecond",
|
||||
"\n\nA paragraph.\n\n",
|
||||
"- [ ] Task\n- [x] Done",
|
||||
"> [!NOTE]\n> Note",
|
||||
"> Ordinary quote",
|
||||
])("copies unchanged source byte-for-byte: %s", (source) => {
|
||||
expect(prepareProse([source], noFiles)).toBe(source);
|
||||
});
|
||||
|
||||
it("escapes spoiler delimiters without changing visible text or inline code", () => {
|
||||
const text = "Keep ||literal||, **bold**, ~~deleted~~ and `a || b`.";
|
||||
const prepared = prepareProse(text, noFiles);
|
||||
expect(prepared).toContain("\\|\\|literal\\|\\|");
|
||||
expect(parser.render(prepared)).toBe(parser.render(text));
|
||||
it("changes only prose pipes", () => {
|
||||
expect(prepareProse(["Keep ||literal|| and `a || b` and [a|b](https://x/a|b)."], noFiles)).toBe(
|
||||
"Keep \\|\\|literal\\|\\| and `a || b` and [a\\|b](https://x/a|b).",
|
||||
);
|
||||
});
|
||||
it("wraps inline file code without reprinting its source", () => {
|
||||
const source = "[Docs](https://example.com) and `` src/file.ts:12 ``";
|
||||
expect(prepareProse([source], (text) => text === "src/file.ts:12")).toBe(
|
||||
"[Docs](https://example.com) and [`` src/file.ts:12 ``](<paseo-inline-code:src%2Ffile.ts%3A12>)",
|
||||
);
|
||||
expect(inlineCodeLinkContent("paseo-inline-code:src%2Ffile.ts%3A12")).toBe("src/file.ts:12");
|
||||
});
|
||||
|
||||
it("keeps inline-code URLs tappable", () => {
|
||||
expect(parser.render(prepareProse("`https://example.com`", noFiles))).toBe(
|
||||
'<p><a href="https://example.com"><code>https://example.com</code></a></p>\n',
|
||||
expect(prepareProse(["`https://example.com`"], noFiles)).toBe(
|
||||
"[`https://example.com`](<https://example.com>)",
|
||||
);
|
||||
});
|
||||
|
||||
it("retains links and makes inline workspace paths use the same resolver input", () => {
|
||||
const prepared = prepareProse(
|
||||
"[Docs](https://example.com) and `src/file.ts:12`",
|
||||
(text) => text === "src/file.ts:12",
|
||||
);
|
||||
const tokens = parser.parseInline(prepared, {})[0]!.children!;
|
||||
const hrefs = tokens
|
||||
.filter((token) => token.type === "link_open")
|
||||
.map((token) => token.attrGet("href"));
|
||||
expect(hrefs).toEqual(["https://example.com", "paseo-inline-code:src%2Ffile.ts%3A12"]);
|
||||
expect(inlineCodeLinkContent(hrefs[1]!)).toBe("src/file.ts:12");
|
||||
it("uppercases only h6 text", () => {
|
||||
expect(
|
||||
prepareProse(
|
||||
["###### Small **bold** `lower.ts` [link](https://example.com/lower) &"],
|
||||
noFiles,
|
||||
),
|
||||
).toBe("###### SMALL **BOLD** `lower.ts` [LINK](https://example.com/lower) &");
|
||||
});
|
||||
|
||||
it("preserves multiline quotes, literal pipes and code links", () => {
|
||||
const text = "> Before ||literal||\n> after `src/file.ts`";
|
||||
const prepared = prepareProse(text, () => true);
|
||||
expect(prepared).toContain("\\|\\|literal\\|\\|");
|
||||
expect(prepared).toContain("paseo-inline-code:");
|
||||
});
|
||||
});
|
||||
|
||||
const streamingParser = createAssistantMarkdownParser({ streaming: true });
|
||||
describe("streaming prose", () => {
|
||||
it.each([
|
||||
"Before **bold",
|
||||
"Before `code",
|
||||
"Before [label](https://exa",
|
||||
"### A **heading",
|
||||
"- First\n- **Second",
|
||||
"> First\n> **Second",
|
||||
"Before **",
|
||||
"Before ~~strike",
|
||||
])("retains provisional formatting: %s", (text) => {
|
||||
expect(parser.render(prepareProse(text, noFiles, true))).toBe(streamingParser.render(text));
|
||||
["Before **bold", "Before **bold**"],
|
||||
["Before `code", "Before `code`"],
|
||||
["Before [label](https://exa", "Before label"],
|
||||
["### A **heading", "### A **heading**"],
|
||||
["- First\n- **Second", "- First\n- **Second**"],
|
||||
["> First\n> **Second", "> First\n> **Second**"],
|
||||
["Before **", "Before "],
|
||||
["Before ~~strike", "Before ~~strike~~"],
|
||||
])("minimally completes the streaming tail: %s", (source, expected) => {
|
||||
expect(prepareProse([source], noFiles, true)).toBe(expected);
|
||||
});
|
||||
it("prepares source blocks independently, even beside a reference definition", () => {
|
||||
const blocks = ["![shot][pic]", "Other paragraph.\n\n[pic]: shots/app.png"];
|
||||
expect(prepareProse(blocks, noFiles)).toBe(
|
||||
blocks.map((block) => prepareProse([block], noFiles)).join("\n\n"),
|
||||
);
|
||||
expect(prepareProse(blocks, noFiles)).toBe(
|
||||
"!\\[shot]\\[pic]\n\nOther paragraph.\n\n[pic]: shots/app.png",
|
||||
);
|
||||
});
|
||||
it("localizes resolved references before joining blocks with conflicting definitions", () => {
|
||||
const blocks = [
|
||||
"[foo][ref]\n\n[ref]: https://one.example",
|
||||
"[bar][ref]\n\n[ref]: https://two.example",
|
||||
];
|
||||
expect(prepareProse(blocks, noFiles)).toBe(
|
||||
"[foo](<https://one.example>)\n\n[ref]: https://one.example\n\n[bar](<https://two.example>)\n\n[ref]: https://two.example",
|
||||
);
|
||||
});
|
||||
it("carries a bare filename's text and href through the file action", () => {
|
||||
const prepared = prepareProse(["Open README.md"], (text) => text === "README.md");
|
||||
const source = {
|
||||
href: "http://README.md",
|
||||
text: "README.md",
|
||||
markup: "linkify",
|
||||
sourceInfo: "auto",
|
||||
};
|
||||
const url = `paseo-file-link:${encodeURIComponent(JSON.stringify(source))}`;
|
||||
expect(prepared).toBe(`Open [README.md](<${url}>)`);
|
||||
expect(proseLinkSource(url)).toEqual(source);
|
||||
const opened: unknown[] = [];
|
||||
handleProseLinkPress(
|
||||
url,
|
||||
(value) => opened.push(value),
|
||||
() => {
|
||||
throw new Error("Opened website");
|
||||
},
|
||||
);
|
||||
expect(opened).toEqual([source]);
|
||||
});
|
||||
});
|
||||
|
||||
it("uppercases heading-six text without changing inline code or link destinations", () => {
|
||||
expect(
|
||||
parser.render(
|
||||
prepareProse("###### Small **bold** `lower.ts` [link](https://example.com/lower)", noFiles),
|
||||
),
|
||||
).toBe(
|
||||
'<h6>SMALL <strong>BOLD</strong> <code>lower.ts</code> <a href="https://example.com/lower">LINK</a></h6>\n',
|
||||
);
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
"preserves inline semantics and leaves reference definitions untouched (streaming=%s)",
|
||||
(streaming) => {
|
||||
const text =
|
||||
"[foo][ref] <https://example.org> & \\* **bold** \n[linked `code`](https://example.net)\n\n[ref]: https://example.com";
|
||||
const prepared = prepareProse(text, noFiles, streaming);
|
||||
expect(prepared).toContain("[ref]: https://example.com");
|
||||
expect(parser.render(prepared)).toBe(parser.render(text));
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
"- [ ] Task",
|
||||
"- [x] Done",
|
||||
"1. [X] Done",
|
||||
"- parent\n - [ ] Nested task",
|
||||
"> [!NOTE]\n> Keep this note",
|
||||
"> Ordinary quote",
|
||||
])("preserves native block markers through streaming and settlement: %s", (text) => {
|
||||
const settled = prepareProse(text, noFiles);
|
||||
const streaming = prepareProse(text, noFiles, true);
|
||||
expect(settled).toBe(text);
|
||||
expect(streaming).toBe(settled);
|
||||
expect(parser.render(settled)).toBe(parser.render(text));
|
||||
});
|
||||
|
||||
it.each([false, true])("keeps distant reference images literal (streaming=%s)", (streaming) => {
|
||||
const text = "![shot][pic]\n\nOther paragraph.\n\n[pic]: shots/app.png";
|
||||
const prepared = prepareProse(text, noFiles, streaming);
|
||||
expect(parser.render(prepared)).toContain("![shot][pic]");
|
||||
expect(parser.render(prepared)).not.toContain("<img");
|
||||
});
|
||||
|
||||
@@ -1,127 +1,297 @@
|
||||
import { createAssistantMarkdownParser } from "@/utils/assistant-markdown-parser";
|
||||
import { splitMarkdownBlocks } from "@/utils/split-markdown-blocks";
|
||||
import type { AssistantFileLinkSource } from "@/assistant-file-links";
|
||||
import type Token from "markdown-it/lib/token.mjs";
|
||||
import { type InlineSpan, type InlineTrace, createSourceParser } from "./source-inline";
|
||||
|
||||
const parser = createAssistantMarkdownParser();
|
||||
const streamingParser = createAssistantMarkdownParser({ streaming: true });
|
||||
const settled = createSourceParser(false);
|
||||
const provisional = createSourceParser(true);
|
||||
const INLINE_CODE_SCHEME = "paseo-inline-code:";
|
||||
const FILE_LINK_SCHEME = "paseo-file-link:";
|
||||
|
||||
export function inlineCodeLinkContent(url: string): string | null {
|
||||
return url.startsWith(INLINE_CODE_SCHEME)
|
||||
? decodeURIComponent(url.slice(INLINE_CODE_SCHEME.length))
|
||||
: null;
|
||||
}
|
||||
|
||||
function escapeText(text: string): string {
|
||||
return text.replace(/&/g, "&").replace(/[\\`*_{}[\]<>()#+\-.!|~]/g, "\\$&");
|
||||
export function proseLinkSource(url: string): AssistantFileLinkSource | null {
|
||||
try {
|
||||
const code = inlineCodeLinkContent(url);
|
||||
if (code !== null) return { href: code, text: code, sourceType: "inline-code" };
|
||||
if (!url.startsWith(FILE_LINK_SCHEME)) return null;
|
||||
const source: unknown = JSON.parse(decodeURIComponent(url.slice(FILE_LINK_SCHEME.length)));
|
||||
if (
|
||||
!source ||
|
||||
typeof source !== "object" ||
|
||||
!("href" in source) ||
|
||||
!("text" in source) ||
|
||||
typeof source.href !== "string" ||
|
||||
typeof source.text !== "string"
|
||||
)
|
||||
return null;
|
||||
return { href: source.href, text: source.text, markup: "linkify", sourceInfo: "auto" };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function renderInline(tokens: Token[], canResolveCode: (text: string) => boolean): string {
|
||||
const links: { href: string; title: string | null }[] = [];
|
||||
return tokens
|
||||
.map((token) => {
|
||||
switch (token.type) {
|
||||
case "text":
|
||||
return escapeText(token.content);
|
||||
case "softbreak":
|
||||
return "\n";
|
||||
case "hardbreak":
|
||||
return " \n";
|
||||
case "link_open":
|
||||
links.push({ href: token.attrGet("href") ?? "", title: token.attrGet("title") });
|
||||
return "[";
|
||||
case "link_close": {
|
||||
const link = links.pop()!;
|
||||
const title =
|
||||
link.title === null
|
||||
? ""
|
||||
: ` "${link.title.replace(/&/g, "&").replace(/"/g, """)}"`;
|
||||
return `](<${link.href}>${title})`;
|
||||
}
|
||||
case "code_inline": {
|
||||
const needsPadding =
|
||||
/[^ ]/.test(token.content) &&
|
||||
(/^[ `]/.test(token.content) || /[ `]$/.test(token.content));
|
||||
const padding = needsPadding ? " " : "";
|
||||
const code = `${token.markup}${padding}${token.content}${padding}${token.markup}`;
|
||||
if (links.length > 0) return code;
|
||||
if (canResolveCode(token.content)) {
|
||||
return `[${code}](<${INLINE_CODE_SCHEME}${encodeURIComponent(token.content)}>)`;
|
||||
}
|
||||
const matches = parser.linkify.match(token.content);
|
||||
const match = matches?.length === 1 ? matches[0] : undefined;
|
||||
if (match?.index === 0 && match.lastIndex === token.content.length)
|
||||
return `[${code}](<${match.url}>)`;
|
||||
return code;
|
||||
}
|
||||
default:
|
||||
return token.markup || token.content;
|
||||
}
|
||||
})
|
||||
.join("");
|
||||
export function handleProseLinkPress(
|
||||
url: string,
|
||||
open: (source: AssistantFileLinkSource) => void,
|
||||
onLinkPress: (url: string) => unknown,
|
||||
): void {
|
||||
const source = proseLinkSource(url);
|
||||
if (source) open(source);
|
||||
else onLinkPress(url);
|
||||
}
|
||||
interface Edit {
|
||||
start: number;
|
||||
end: number;
|
||||
text: string;
|
||||
}
|
||||
interface Range {
|
||||
start: number;
|
||||
end: number;
|
||||
}
|
||||
function applyEdits(source: string, edits: Edit[]): string {
|
||||
let cursor = 0;
|
||||
let result = "";
|
||||
for (const edit of edits.sort((a, b) => a.start - b.start || a.end - b.end)) {
|
||||
if (edit.start < cursor) continue;
|
||||
result += source.slice(cursor, edit.start) + edit.text;
|
||||
cursor = edit.end;
|
||||
}
|
||||
return result + source.slice(cursor);
|
||||
}
|
||||
|
||||
function preserveNativeBlockMarker(source: string, rendered: string): string {
|
||||
const marker = source.match(
|
||||
/^\[(?:[ xX]|!(?:NOTE|TIP|IMPORTANT|WARNING|CAUTION))\](?=\s|$)/,
|
||||
)?.[0];
|
||||
if (!marker) return rendered;
|
||||
const escaped = escapeText(marker);
|
||||
return rendered.startsWith(escaped) ? marker + rendered.slice(escaped.length) : rendered;
|
||||
}
|
||||
|
||||
/** Keep the existing inline dialect, including literal pipes and tappable inline paths. */
|
||||
/** Source blocks are presentation's canonical inputs, never reconstructed from a joined run. */
|
||||
export function prepareProse(
|
||||
text: string,
|
||||
blocks: readonly string[],
|
||||
canResolveCode: (text: string) => boolean,
|
||||
streaming = false,
|
||||
visibleLength = Infinity,
|
||||
): string {
|
||||
// Reference definitions have the same scope as Paseo's original block rows.
|
||||
const blocks = splitMarkdownBlocks(text);
|
||||
return blocks
|
||||
const visible: string[] = [];
|
||||
let remaining = visibleLength;
|
||||
for (const block of blocks) {
|
||||
if (remaining < 0) break;
|
||||
visible.push(block.slice(0, remaining));
|
||||
remaining -= block.length + 2;
|
||||
}
|
||||
return visible
|
||||
.map((block, index) =>
|
||||
prepareBlock(block, canResolveCode, streaming && index === blocks.length - 1),
|
||||
prepareBlock(block, canResolveCode, streaming && index === visible.length - 1),
|
||||
)
|
||||
.join("\n\n");
|
||||
}
|
||||
|
||||
interface InlinePreparation {
|
||||
source: string;
|
||||
token: Token;
|
||||
trace: InlineTrace;
|
||||
parser: typeof settled.parser;
|
||||
env: object;
|
||||
streaming: boolean;
|
||||
uppercase: boolean;
|
||||
canResolveCode: (text: string) => boolean;
|
||||
protectedRanges: Range[];
|
||||
referenceRanges: Range[];
|
||||
edits: Edit[];
|
||||
}
|
||||
function insert(context: InlinePreparation, at: number, text: string) {
|
||||
context.edits.push({ start: at, end: at, text });
|
||||
}
|
||||
function contains(ranges: Range[], start: number, end = start + 1): boolean {
|
||||
return ranges.some((range) => start < range.end && end > range.start);
|
||||
}
|
||||
function codeUrl(context: InlinePreparation, content: string): string | null {
|
||||
if (context.canResolveCode(content)) return INLINE_CODE_SCHEME + encodeURIComponent(content);
|
||||
const matches = context.parser.linkify.match(content);
|
||||
const match = matches?.length === 1 ? matches[0] : undefined;
|
||||
return match?.index === 0 && match.lastIndex === content.length ? match.url : null;
|
||||
}
|
||||
function prepareCode(context: InlinePreparation, span: InlineSpan, raw: string, code: Token) {
|
||||
context.protectedRanges.push(span);
|
||||
context.referenceRanges.push(span);
|
||||
const insideLink = context.trace.spans.some(
|
||||
(parent) =>
|
||||
parent.start < span.start &&
|
||||
parent.end >= span.end &&
|
||||
parent.tokens.some((child) => child.type === "link_open"),
|
||||
);
|
||||
const complete = raw.length >= code.markup.length * 2 && raw.endsWith(code.markup);
|
||||
const closing = complete ? "" : code.markup.slice(raw.match(/`+$/)?.[0].length ?? 0);
|
||||
const url = codeUrl(context, code.content);
|
||||
if (!insideLink && url) {
|
||||
insert(context, span.start, "[");
|
||||
insert(context, span.end, `${closing}](<${url}>)`);
|
||||
} else if (closing) insert(context, span.end, closing);
|
||||
}
|
||||
function labelEnd(context: InlinePreparation, source: string, start: number): number {
|
||||
const state = new context.parser.inline.State(source, context.parser, context.env, []);
|
||||
return context.parser.helpers.parseLinkLabel(state, start, false);
|
||||
}
|
||||
function prepareUnfinishedLink(context: InlinePreparation, span: InlineSpan, raw: string) {
|
||||
const start = raw.startsWith("![") ? 1 : 0;
|
||||
const end = labelEnd(context, raw, start);
|
||||
context.edits.push({ start: span.start, end: span.start + start + 1, text: "" });
|
||||
if (start === 1) context.edits.push({ start: span.start + 2, end: span.end, text: "" });
|
||||
else if (end !== -1) context.edits.push({ start: span.start + end, end: span.end, text: "" });
|
||||
context.referenceRanges.push(span);
|
||||
}
|
||||
function preserveLinkScope(context: InlinePreparation, span: InlineSpan, raw: string, link: Token) {
|
||||
const end = labelEnd(context, raw, 0);
|
||||
context.protectedRanges.push({ start: span.start + end, end: span.end });
|
||||
context.referenceRanges.push(span);
|
||||
if (raw[end + 1] === "(") return;
|
||||
// The native parser sees the joined run; bind reference destinations to their source block.
|
||||
const title = link.attrGet("title");
|
||||
const quotedTitle =
|
||||
title === null ? "" : ` "${title.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`;
|
||||
context.edits.push({
|
||||
start: span.start + end,
|
||||
end: span.end,
|
||||
text: `](<${link.attrGet("href")}>${quotedTitle})`,
|
||||
});
|
||||
}
|
||||
function prepareSpan(context: InlinePreparation, span: InlineSpan) {
|
||||
const raw = context.source.slice(span.start, span.end);
|
||||
const link = span.tokens.find((child) => child.type === "link_open");
|
||||
const image = span.tokens.find((child) => child.type === "image");
|
||||
const code = span.tokens.find((child) => child.type === "code_inline");
|
||||
if (
|
||||
image ||
|
||||
raw.startsWith("\\") ||
|
||||
/^&(?:#\w+|\w+);$/.test(raw) ||
|
||||
(link && raw.startsWith("<"))
|
||||
) {
|
||||
context.protectedRanges.push(span);
|
||||
context.referenceRanges.push(span);
|
||||
} else if (link && raw.startsWith("[")) {
|
||||
preserveLinkScope(context, span, raw, link);
|
||||
} else if (code && raw.startsWith("`")) {
|
||||
prepareCode(context, span, raw, code);
|
||||
} else if (context.streaming && (raw.startsWith("[") || raw.startsWith("!["))) {
|
||||
prepareUnfinishedLink(context, span, raw);
|
||||
} else if (
|
||||
context.streaming &&
|
||||
span.tokens.length === 0 &&
|
||||
/^(?:\*{1,3}|_{1,3}|~{1,2}|`+)$/.test(raw)
|
||||
) {
|
||||
context.edits.push({ start: span.start, end: span.end, text: "" });
|
||||
}
|
||||
}
|
||||
function preserveUnresolvedReferences(context: InlinePreparation) {
|
||||
// Unresolved labels stay literal even when another source block defines them.
|
||||
for (const match of context.source.matchAll(/!?\[[^\]\n]+\](?:\[[^\]\n]*\])?/g)) {
|
||||
if (contains(context.referenceRanges, match.index, match.index + match[0].length)) continue;
|
||||
if (/^\[(?:[ xX]|!(?:NOTE|TIP|IMPORTANT|WARNING|CAUTION))\]$/.test(match[0])) continue;
|
||||
for (let i = 0; i < match[0].length; i++)
|
||||
if (match[0][i] === "[") insert(context, match.index + i, "\\");
|
||||
}
|
||||
}
|
||||
function prepareText(context: InlinePreparation) {
|
||||
for (let i = 0; i < context.source.length; i++) {
|
||||
if (contains(context.protectedRanges, i)) continue;
|
||||
const char = context.source[i]!;
|
||||
if (char === "|") insert(context, i, "\\");
|
||||
if (context.uppercase && char.toUpperCase() !== char)
|
||||
context.edits.push({ start: i, end: i + 1, text: char.toUpperCase() });
|
||||
}
|
||||
}
|
||||
function prepareLinkifiedFiles(context: InlinePreparation) {
|
||||
for (const match of context.parser.linkify.match(context.source) ?? []) {
|
||||
if (
|
||||
contains(context.protectedRanges, match.index, match.lastIndex) ||
|
||||
contains(context.referenceRanges, match.index, match.lastIndex) ||
|
||||
!context.canResolveCode(match.text)
|
||||
)
|
||||
continue;
|
||||
const linkified = context.token.children?.some(
|
||||
(child) =>
|
||||
child.type === "link_open" &&
|
||||
child.markup === "linkify" &&
|
||||
child.attrGet("href") === match.url,
|
||||
);
|
||||
if (!linkified) continue;
|
||||
const payload: AssistantFileLinkSource = {
|
||||
href: match.url,
|
||||
text: match.text,
|
||||
markup: "linkify",
|
||||
sourceInfo: "auto",
|
||||
};
|
||||
insert(context, match.index, "[");
|
||||
insert(
|
||||
context,
|
||||
match.lastIndex,
|
||||
`](<${FILE_LINK_SCHEME}${encodeURIComponent(JSON.stringify(payload))}>)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
function sourcePositions(
|
||||
token: Token,
|
||||
heading: Token | undefined,
|
||||
lines: string[],
|
||||
lineOffsets: number[],
|
||||
): number[] {
|
||||
const positions: number[] = [];
|
||||
for (const [index, content] of token.content.split("\n").entries()) {
|
||||
const row = token.map![0] + index;
|
||||
const rawLine = lines[row]!;
|
||||
let headingPrefix = 0;
|
||||
if (heading?.type === "heading_open" && heading.markup.startsWith("#"))
|
||||
headingPrefix = rawLine.match(/^(?:\s*>\s*)*\s*#{1,6}\s+/)?.[0].length ?? 0;
|
||||
const column = rawLine.indexOf(content, headingPrefix);
|
||||
const start = lineOffsets[row]! + Math.max(0, column);
|
||||
for (let i = 0; i <= content.length; i++) positions.push(start + i);
|
||||
}
|
||||
return positions;
|
||||
}
|
||||
function prepareBlock(
|
||||
text: string,
|
||||
canResolveCode: (text: string) => boolean,
|
||||
streaming: boolean,
|
||||
): string {
|
||||
const tokens = (streaming ? streamingParser : parser).parse(text, {});
|
||||
const { parser, traces } = streaming ? provisional : settled;
|
||||
const env = {};
|
||||
const tokens = parser.parse(text, env);
|
||||
const lines = text.split("\n");
|
||||
for (let index = tokens.length - 1; index >= 0; index--) {
|
||||
const token = tokens[index]!;
|
||||
const uppercase = tokens[index - 1]?.type === "heading_open" && tokens[index - 1]?.tag === "h6";
|
||||
if (token.type !== "inline" || !token.map || !token.children) continue;
|
||||
const [start] = token.map;
|
||||
const prefixes = token.content.split("\n").map((content, offset) => {
|
||||
const line = lines[start + offset]!;
|
||||
const heading = tokens[index - 1];
|
||||
const headingPrefix =
|
||||
heading?.type === "heading_open" && heading.markup.startsWith("#")
|
||||
? (line.match(/^(?:\s*>\s*)*\s*#{1,6}\s+/)?.[0].length ?? 0)
|
||||
: 0;
|
||||
const position = line.indexOf(content, headingPrefix);
|
||||
return position < 0 ? "" : line.slice(0, position);
|
||||
});
|
||||
if (uppercase) {
|
||||
for (const child of token.children) {
|
||||
if (child.type === "text") child.content = child.content.toUpperCase();
|
||||
}
|
||||
}
|
||||
const rendered = preserveNativeBlockMarker(
|
||||
token.content,
|
||||
renderInline(token.children, canResolveCode),
|
||||
)
|
||||
.replace(/ $/, " ")
|
||||
.split("\n");
|
||||
const replacement = rendered.map(
|
||||
(line, offset) => `${prefixes[offset] ?? prefixes.at(-1) ?? ""}${line}`,
|
||||
);
|
||||
lines.splice(start, token.content.split("\n").length, ...replacement);
|
||||
const lineOffsets: number[] = [];
|
||||
let offset = 0;
|
||||
for (const line of lines) {
|
||||
lineOffsets.push(offset);
|
||||
offset += line.length + 1;
|
||||
}
|
||||
return lines.join("\n");
|
||||
const edits: Edit[] = [];
|
||||
for (const [index, token] of tokens.entries()) {
|
||||
if (token.type !== "inline" || !token.map) continue;
|
||||
const trace = traces.get(token)!;
|
||||
const heading = tokens[index - 1];
|
||||
const context: InlinePreparation = {
|
||||
source: token.content,
|
||||
token,
|
||||
trace,
|
||||
parser,
|
||||
env,
|
||||
streaming,
|
||||
uppercase: heading?.type === "heading_open" && heading.tag === "h6",
|
||||
canResolveCode,
|
||||
protectedRanges: [],
|
||||
referenceRanges: [],
|
||||
edits: [],
|
||||
};
|
||||
for (const span of trace.spans) prepareSpan(context, span);
|
||||
preserveUnresolvedReferences(context);
|
||||
prepareText(context);
|
||||
prepareLinkifiedFiles(context);
|
||||
if (trace.trim)
|
||||
context.edits.push({
|
||||
start: token.content.length - trace.trim,
|
||||
end: token.content.length,
|
||||
text: "",
|
||||
});
|
||||
if (trace.suffix) insert(context, token.content.length, trace.suffix);
|
||||
const positions = sourcePositions(token, heading, lines, lineOffsets);
|
||||
for (const edit of context.edits)
|
||||
edits.push({ ...edit, start: positions[edit.start]!, end: positions[edit.end]! });
|
||||
}
|
||||
return applyEdits(text, edits);
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ import type { ReactNode } from "react";
|
||||
|
||||
export interface MarkdownProseProps {
|
||||
text: string;
|
||||
sourceBlocks: readonly string[];
|
||||
streaming: boolean;
|
||||
kind: "prose" | "quote" | "block";
|
||||
onLinkPress: (url: string) => boolean;
|
||||
|
||||
@@ -6,7 +6,7 @@ import { useStableEvent } from "@/hooks/use-stable-event";
|
||||
import { createMarkdownStyles } from "@/styles/markdown-styles";
|
||||
import type { Theme } from "@/styles/theme";
|
||||
import type { MarkdownProseProps } from "./props";
|
||||
import { inlineCodeLinkContent, prepareProse } from "./prepare";
|
||||
import { handleProseLinkPress, prepareProse } from "./prepare";
|
||||
|
||||
const ThemedMarkdown = withUnistyles(EnrichedMarkdownText);
|
||||
const MD4C_FLAGS = { latexMath: false, hardSoftBreaks: true, admonitions: true };
|
||||
@@ -95,6 +95,7 @@ const PROSE_CONTAINER = { marginBottom: -12 };
|
||||
|
||||
const NativeProse = memo(function NativeProse({
|
||||
text,
|
||||
sourceBlocks,
|
||||
streaming,
|
||||
kind,
|
||||
onLinkPress,
|
||||
@@ -104,16 +105,11 @@ const NativeProse = memo(function NativeProse({
|
||||
actions.canResolveFile({ href: content, text: content, sourceType: "inline-code" }),
|
||||
);
|
||||
const markdown = useMemo(
|
||||
() => prepareProse(text, canResolveCode, streaming),
|
||||
[text, canResolveCode, streaming],
|
||||
() => prepareProse(sourceBlocks, canResolveCode, streaming, text.length),
|
||||
[text.length, sourceBlocks, canResolveCode, streaming],
|
||||
);
|
||||
const handleLinkPress = useStableEvent(({ url }: { url: string }) => {
|
||||
const content = inlineCodeLinkContent(url);
|
||||
if (content !== null) {
|
||||
actions.open({ href: content, text: content, sourceType: "inline-code" }, "preferred");
|
||||
return;
|
||||
}
|
||||
onLinkPress(url);
|
||||
handleProseLinkPress(url, (source) => actions.open(source, "preferred"), onLinkPress);
|
||||
});
|
||||
return (
|
||||
<ThemedMarkdown
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import { ASSISTANT_MESSAGE_RENDER_CHARACTER_LIMIT } from "@/components/assistant-message-render-limit";
|
||||
import MarkdownIt from "markdown-it";
|
||||
import { createAssistantMarkdownParser } from "@/utils/assistant-markdown-parser";
|
||||
import type Token from "markdown-it/lib/token.mjs";
|
||||
import type { AssistantMessageItem } from "@/types/stream";
|
||||
|
||||
const parser = new MarkdownIt({ html: true });
|
||||
const parser = createAssistantMarkdownParser();
|
||||
const partsByBlock = new WeakMap<AssistantMessageItem, AssistantMessageItem[]>();
|
||||
|
||||
function splitBlock(block: AssistantMessageItem): AssistantMessageItem[] {
|
||||
@@ -14,7 +14,11 @@ function splitBlock(block: AssistantMessageItem): AssistantMessageItem[] {
|
||||
const preserved = tokens.some(
|
||||
(token) =>
|
||||
["fence", "code_block", "table_open", "html_block"].includes(token.type) ||
|
||||
token.children?.some((child) => child.type === "html_inline"),
|
||||
token.children?.some(
|
||||
(child) =>
|
||||
child.type === "html_inline" ||
|
||||
(child.type === "text" && /<\/?[A-Za-z][^>]*>/.test(child.content)),
|
||||
),
|
||||
);
|
||||
const isQuote = tokens.some((token) => token.type === "blockquote_open");
|
||||
const hasImage = tokens.some((token) => token.children?.some((child) => child.type === "image"));
|
||||
@@ -76,10 +80,12 @@ export function groupMarkdownProse(blocks: AssistantMessageItem[]): AssistantMes
|
||||
previous.text.length + 2 + block.text.length <= ASSISTANT_MESSAGE_RENDER_CHARACTER_LIMIT
|
||||
) {
|
||||
previous.text += `\n\n${block.text}`;
|
||||
previous.markdownProseBlocks = [...previous.markdownProseBlocks!, block.text];
|
||||
continue;
|
||||
}
|
||||
rows.push({
|
||||
...block,
|
||||
markdownProseBlocks: block.markdownKind === "block" ? undefined : [block.text],
|
||||
id: block.markdownKind === "prose" ? block.id.replace(":block:", ":run:") : block.id,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
import { createAssistantMarkdownParser } from "@/utils/assistant-markdown-parser";
|
||||
import type Token from "markdown-it/lib/token.mjs";
|
||||
|
||||
export interface InlineSpan {
|
||||
start: number;
|
||||
end: number;
|
||||
tokens: Token[];
|
||||
}
|
||||
export interface InlineTrace {
|
||||
spans: InlineSpan[];
|
||||
suffix: string;
|
||||
trim: number;
|
||||
}
|
||||
const TRACE = Symbol("prose source positions");
|
||||
|
||||
/** Observe the actual inline rules, including nested labels, without reprinting tokens. */
|
||||
export function createSourceParser(streaming: boolean) {
|
||||
const parser = createAssistantMarkdownParser({ streaming });
|
||||
const traces = new WeakMap<Token, InlineTrace>();
|
||||
const inlineTraces = new WeakMap<Token[], InlineTrace>();
|
||||
const parseInline = parser.inline.parse.bind(parser.inline);
|
||||
parser.inline.parse = (source, md, env, output) => {
|
||||
const trace: InlineTrace = { spans: [], suffix: "", trim: 0 };
|
||||
parseInline(source, md, { ...env, [TRACE]: trace }, output);
|
||||
inlineTraces.set(output, trace);
|
||||
};
|
||||
const rules = parser.inline.ruler.getRules("");
|
||||
parser.inline.ruler.before("text", "prose_source_positions", (state, silent) => {
|
||||
if (silent) return false;
|
||||
for (const rule of rules) {
|
||||
const start = state.pos;
|
||||
const count = state.tokens.length;
|
||||
if (rule(state, false)) {
|
||||
const trace: InlineTrace = state.env[TRACE];
|
||||
trace.spans.push({ start, end: state.pos, tokens: state.tokens.slice(count) });
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
});
|
||||
if (streaming) {
|
||||
const counts = new WeakMap<
|
||||
object,
|
||||
{ count: number; last: Token | undefined; content: string }
|
||||
>();
|
||||
parser.inline.ruler2.before("streaming_emphasis", "prose_before_completion", (state) => {
|
||||
counts.set(state, {
|
||||
count: state.tokens.length,
|
||||
last: state.tokens.at(-1),
|
||||
content: state.tokens.at(-1)?.content ?? "",
|
||||
});
|
||||
return true;
|
||||
});
|
||||
parser.inline.ruler2.after("streaming_emphasis", "prose_after_completion", (state) => {
|
||||
const before = counts.get(state)!;
|
||||
const trace: InlineTrace = state.env[TRACE];
|
||||
trace.suffix = state.tokens
|
||||
.slice(before.count)
|
||||
.map((token) => token.content)
|
||||
.join("");
|
||||
trace.trim = before.content.length - (before.last?.content.length ?? 0);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
parser.core.ruler.after("inline", "prose_inline_traces", (state) => {
|
||||
for (const token of state.tokens) {
|
||||
if (token.type === "inline" && token.children)
|
||||
traces.set(token, inlineTraces.get(token.children)!);
|
||||
}
|
||||
});
|
||||
return { parser, traces };
|
||||
}
|
||||
@@ -754,6 +754,7 @@ export const LiveElapsed = memo(function LiveElapsed({
|
||||
|
||||
interface AssistantMessageProps {
|
||||
markdownKind?: "prose" | "quote" | "block";
|
||||
markdownProseBlocks?: readonly string[];
|
||||
renderFullContent?: boolean;
|
||||
occurrenceKey: string;
|
||||
message: string;
|
||||
@@ -1497,6 +1498,7 @@ function MarkdownListView({
|
||||
|
||||
export const AssistantMessage = memo(function AssistantMessage({
|
||||
markdownKind = "block",
|
||||
markdownProseBlocks,
|
||||
renderFullContent = false,
|
||||
occurrenceKey,
|
||||
message,
|
||||
@@ -1999,6 +2001,7 @@ export const AssistantMessage = memo(function AssistantMessage({
|
||||
streaming={phase === "streaming"}
|
||||
kind={markdownKind}
|
||||
text={revealedMessage}
|
||||
sourceBlocks={markdownProseBlocks ?? [message]}
|
||||
onLinkPress={handleMarkdownLinkPress}
|
||||
>
|
||||
{keyedBlocks.map(({ key, block }, index) => (
|
||||
|
||||
@@ -715,6 +715,7 @@ export interface AssistantMessageItem {
|
||||
blockGroupId?: string;
|
||||
blockIndex?: number;
|
||||
markdownKind?: "prose" | "quote" | "block";
|
||||
markdownProseBlocks?: readonly string[];
|
||||
}
|
||||
|
||||
export interface TimelinePosition {
|
||||
|
||||
Reference in New Issue
Block a user