mirror of
https://github.com/getpaseo/paseo.git
synced 2026-10-28 04:03:05 +00:00
fix(app): preserve native prose entities and links while bounding preparation cost
This commit is contained in:
Mohamed Boudra
@@ -34,10 +34,11 @@ So arrival sets a _target_ and the reveal rate is derived from the backlog inste
|
||||
display. Native groups adjacent prose into selectable runs; web retains individual blocks.
|
||||
Code, tables, images, HTML-like content and blockquotes end a native run. See
|
||||
[native text selection](native-text-selection.md) for renderer ownership and library constraints.
|
||||
Completing a reference definition can change an earlier block boundary. Live and history use
|
||||
the same structural parse; unchanged blocks retain their classification and completed rows
|
||||
retain object identity. A source-item cache avoids re-splitting unchanged history. Prose rows
|
||||
retain the original source blocks so native preparation cannot broaden reference scope.
|
||||
Appends reparse the growing last block. A completed reference definition at its start can
|
||||
belong to the preceding block, so that case reparses the full message. Completed rows retain
|
||||
object identity and unchanged history stays cached. Prose rows retain their original source
|
||||
blocks so native preparation cannot broaden reference scope. Paced reveal reuses prepared
|
||||
paragraph prefixes and prepares only the remaining open inline tail.
|
||||
- **A row id is never a message id.** Block rows are `${messageId}:block:${n}`; native prose runs use `${messageId}:run:${n}` with
|
||||
`blockGroupId = messageId`, including single-row messages. Inline images split a source block into offset-keyed parts, so nothing can come to rely on the two
|
||||
being equal. `getStreamItemMessageId` in `presentation.ts` is the only way to go from a row to its
|
||||
|
||||
@@ -14,6 +14,7 @@ Checkbox toggling is disabled because changes would not persist. Admonitions add
|
||||
|
||||
## Compatibility constraints
|
||||
|
||||
- The 1.1.1 native parser drops entity nodes. Decode text entities before rendering, retaining literal entities inside code. Its autolinks also disagree with Paseo about bare domains and trailing punctuation, so preserve Paseo’s recognized destinations as explicit links.
|
||||
- Keep dollar-delimited text literal with `latexMath: false`; dollar amounts must not become math.
|
||||
- Spoilers have no public parser switch in 1.1.1. Escape prose pipes so `||text||` stays visible; code and link destinations retain their pipes.
|
||||
- `lineHeight` is a minimum, so font metrics can increase it. Compare spacing on devices.
|
||||
|
||||
@@ -124,24 +124,42 @@ function rows(result: { tail: StreamItem[]; head: StreamItem[] }): StreamItem[]
|
||||
}
|
||||
|
||||
describe("native prose runs", () => {
|
||||
it("projects a completed streamed reference definition like fresh history", () => {
|
||||
const present = createStreamPresentation({ markdown: "prose-runs" });
|
||||
let state: { tail: StreamItem[]; head: StreamItem[] } = { tail: [], head: [] };
|
||||
for (const text of ["![shot][pic]\n\n[pic]:", " shots/app.png"]) {
|
||||
state = applyStreamEvent({ ...state, event: assistant(text), timestamp: new Date(1000) });
|
||||
present({ ...presentationOptions, ...state, transform: undefined });
|
||||
}
|
||||
const live = present({ ...presentationOptions, ...state, transform: undefined });
|
||||
const history = createStreamPresentation({ markdown: "prose-runs" })({
|
||||
it.each([
|
||||
"<!-- keep this -->\nText.",
|
||||
"<?pi hello?>",
|
||||
"<!DOCTYPE html>",
|
||||
"Text <!-- inline --> after",
|
||||
])("preserves HTML constructs: %s", (text) => {
|
||||
const source = hydrateStreamState([{ event: assistant(text), timestamp: new Date(1000) }]);
|
||||
const result = createStreamPresentation({ markdown: "prose-runs" })({
|
||||
...presentationOptions,
|
||||
tail: hydrateStreamState([
|
||||
{ event: assistant("![shot][pic]\n\n[pic]: shots/app.png"), timestamp: new Date(1000) },
|
||||
]),
|
||||
tail: source,
|
||||
head: [],
|
||||
transform: undefined,
|
||||
});
|
||||
expect([...live.tail, ...live.head]).toEqual(history.tail);
|
||||
expect(result.tail).toMatchObject([{ text, markdownKind: "block" }]);
|
||||
});
|
||||
it.each(["blocks", "prose-runs"] as const)(
|
||||
"projects a completed streamed reference definition like fresh history in %s",
|
||||
(markdown) => {
|
||||
const present = createStreamPresentation({ markdown });
|
||||
let state: { tail: StreamItem[]; head: StreamItem[] } = { tail: [], head: [] };
|
||||
for (const text of ["![shot][pic]\n\n[pic]:", " shots/app.png"]) {
|
||||
state = applyStreamEvent({ ...state, event: assistant(text), timestamp: new Date(1000) });
|
||||
present({ ...presentationOptions, ...state, transform: undefined });
|
||||
}
|
||||
const live = present({ ...presentationOptions, ...state, transform: undefined });
|
||||
const history = createStreamPresentation({ markdown })({
|
||||
...presentationOptions,
|
||||
tail: hydrateStreamState([
|
||||
{ event: assistant("![shot][pic]\n\n[pic]: shots/app.png"), timestamp: new Date(1000) },
|
||||
]),
|
||||
head: [],
|
||||
transform: undefined,
|
||||
});
|
||||
expect([...live.tail, ...live.head]).toEqual(history.tail);
|
||||
},
|
||||
);
|
||||
it("keeps file URL images in Paseo's renderer", () => {
|
||||
const source = hydrateStreamState([
|
||||
{
|
||||
|
||||
@@ -102,13 +102,24 @@ export function createStreamPresentation({
|
||||
if (cached) return cached;
|
||||
const previousSource = liveSources.get(item.id);
|
||||
const previous = previousSource && blocksBySource.get(previousSource);
|
||||
// Completing a reference definition can join a preceding block. Use the same
|
||||
// structural parse for live and history, then retain unchanged block objects.
|
||||
const parsed = splitMarkdownBlocks(item.text);
|
||||
const textBlocks = parsed.length > 0 ? parsed : [""];
|
||||
const blocks: AssistantMessageItem[] = [];
|
||||
const isAppend = previousSource && item.text.startsWith(previousSource.text);
|
||||
let prefix: AssistantMessageItem[] = [];
|
||||
let growingText = item.text;
|
||||
if (isAppend && previous) {
|
||||
prefix = previous.slice(0, -1);
|
||||
growingText =
|
||||
previous[previous.length - 1]!.text + item.text.slice(previousSource.text.length);
|
||||
}
|
||||
let parsed = splitMarkdownBlocks(growingText);
|
||||
// A definition completed in the last block can now belong to the preceding block.
|
||||
if (prefix.length && /^\s{0,3}\[[^\]]+\]:/.test(parsed[0] ?? growingText)) {
|
||||
prefix = [];
|
||||
parsed = splitMarkdownBlocks(item.text);
|
||||
}
|
||||
const textBlocks = parsed.length + prefix.length > 0 ? parsed : [""];
|
||||
const blocks = [...prefix];
|
||||
for (const [offset, text] of textBlocks.entries()) {
|
||||
const index = offset;
|
||||
const index = prefix.length + offset;
|
||||
let blockText = text;
|
||||
if (offset === textBlocks.length - 1) {
|
||||
// The split drops trailing blank lines, including a line break followed only
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
prepareProse,
|
||||
createProsePreparer,
|
||||
inlineCodeLinkContent,
|
||||
proseLinkSource,
|
||||
handleProseLinkPress,
|
||||
@@ -13,7 +14,6 @@ describe("source-preserving prose", () => {
|
||||
"# A & B",
|
||||
"- A & B",
|
||||
"> A & B",
|
||||
"Literal &amp; and &copy;",
|
||||
"Escaped \\* and \\|",
|
||||
"<https://example.org>",
|
||||
"**bold** and *emphasis* and ~~deleted~~",
|
||||
@@ -32,6 +32,51 @@ describe("source-preserving prose", () => {
|
||||
])("copies unchanged source byte-for-byte: %s", (source) => {
|
||||
expect(prepareProse([source], noFiles)).toBe(source);
|
||||
});
|
||||
it("decodes entities outside code", () => {
|
||||
expect(prepareProse(["A & B A A © `&`"], noFiles)).toBe(
|
||||
"A & B A A © `&`",
|
||||
);
|
||||
});
|
||||
it("decodes entities once, including encoded entity syntax", () => {
|
||||
expect(prepareProse(["Literal &amp; and &copy;"], noFiles)).toBe(
|
||||
"Literal \\& and \\©",
|
||||
);
|
||||
});
|
||||
it.each(["\r", "\r\n"])("normalizes line endings %j", (ending) => {
|
||||
expect(prepareProse([`First${ending}Second ||text||`], noFiles)).toBe(
|
||||
"First\nSecond \\|\\|text\\|\\|",
|
||||
);
|
||||
});
|
||||
it.each([
|
||||
["See example.com.", "See [example.com](<http://example.com>)."],
|
||||
["https://example.com/a_(b).", "[https://example.com/a_(b)](<https://example.com/a_(b)>)."],
|
||||
["https://example.com/a.", "[https://example.com/a](<https://example.com/a>)."],
|
||||
])("makes linkify destinations explicit: %s", (source, expected) => {
|
||||
expect(prepareProse([source], noFiles)).toBe(expected);
|
||||
});
|
||||
it("only exempts task markers inside list items", () => {
|
||||
expect(prepareProse(["[x]", "Other\n\n[x]: https://example.com"], noFiles)).toBe(
|
||||
"\\[x]\n\nOther\n\n[x]: https://example.com",
|
||||
);
|
||||
const tasks = "- [x] Task\n- [ ] Pending\n1. [x] Ordered";
|
||||
expect(prepareProse([tasks], noFiles)).toBe(tasks);
|
||||
});
|
||||
it("prepares 30k of inline code in under 10ms warm", () => {
|
||||
const text = "Use `foo` and `bar` here. ".repeat(1300).slice(0, 30000);
|
||||
for (let i = 0; i < 10; i++) prepareProse([text], noFiles);
|
||||
const start = performance.now();
|
||||
for (let i = 0; i < 5; i++) expect(prepareProse([text], noFiles)).toBe(text);
|
||||
expect((performance.now() - start) / 5).toBeLessThan(10);
|
||||
});
|
||||
it("reuses prepared prefixes while completing the visible inline tail", () => {
|
||||
const prepare = createProsePreparer(noFiles);
|
||||
const blocks = ["First & paragraph.", "Use `foo` and **bold** here. ".repeat(30)];
|
||||
const source = blocks.join("\n\n");
|
||||
prepare(blocks, true, source.length);
|
||||
for (let length = 0; length < source.length; length += 7) {
|
||||
expect(prepare(blocks, true, length)).toBe(prepareProse(blocks, noFiles, true, length));
|
||||
}
|
||||
});
|
||||
it("changes only prose pipes", () => {
|
||||
expect(prepareProse(["Keep ||literal|| and `a || b` and [a|b](https://x/a|b)."], noFiles)).toBe(
|
||||
"Keep \\|\\|literal\\|\\| and `a || b` and [a\\|b](https://x/a|b).",
|
||||
@@ -55,7 +100,7 @@ describe("source-preserving prose", () => {
|
||||
["###### Small **bold** `lower.ts` [link](https://example.com/lower) &"],
|
||||
noFiles,
|
||||
),
|
||||
).toBe("###### SMALL **BOLD** `lower.ts` [LINK](https://example.com/lower) &");
|
||||
).toBe("###### SMALL **BOLD** `lower.ts` [LINK](https://example.com/lower) &");
|
||||
});
|
||||
it.each([
|
||||
["Before **bold", "Before **bold**"],
|
||||
|
||||
@@ -47,10 +47,6 @@ interface Edit {
|
||||
end: number;
|
||||
text: string;
|
||||
}
|
||||
interface Range {
|
||||
start: number;
|
||||
end: number;
|
||||
}
|
||||
function applyEdits(source: string, edits: Edit[]): string {
|
||||
let cursor = 0;
|
||||
let result = "";
|
||||
@@ -83,6 +79,57 @@ export function prepareProse(
|
||||
.join("\n\n");
|
||||
}
|
||||
|
||||
interface PreparedBlock {
|
||||
text: string;
|
||||
cuts: number[];
|
||||
offsets: number[];
|
||||
}
|
||||
|
||||
/** One cache per mounted prose row. Reveal reuses the prepared prefix and parses only its open tail. */
|
||||
export function createProsePreparer(canResolveCode: (text: string) => boolean) {
|
||||
let cache = new Map<string, PreparedBlock>();
|
||||
return (blocks: readonly string[], streaming: boolean, visibleLength: number): string => {
|
||||
const retained = new Map<string, PreparedBlock>();
|
||||
const result: string[] = [];
|
||||
let remaining = visibleLength;
|
||||
for (const [index, block] of blocks.entries()) {
|
||||
const live = streaming && index === blocks.length - 1;
|
||||
const key = `${live}:${block}`;
|
||||
const prepared = cache.get(key) ?? compileBlock(block, canResolveCode, live);
|
||||
retained.set(key, prepared);
|
||||
if (remaining >= 0) {
|
||||
result.push(revealBlock(block, prepared, remaining, canResolveCode, streaming));
|
||||
remaining -= block.length + 2;
|
||||
}
|
||||
}
|
||||
cache = retained;
|
||||
return result.join("\n\n");
|
||||
};
|
||||
}
|
||||
function revealBlock(
|
||||
source: string,
|
||||
prepared: PreparedBlock,
|
||||
length: number,
|
||||
canResolveCode: (text: string) => boolean,
|
||||
streaming: boolean,
|
||||
): string {
|
||||
if (length >= source.length) return prepared.text;
|
||||
// Checkpoints are outside inline containers, at whitespace in a plain paragraph.
|
||||
let low = 0;
|
||||
let high = prepared.cuts.length;
|
||||
while (low < high) {
|
||||
const mid = (low + high) >>> 1;
|
||||
if (prepared.cuts[mid]! <= length) low = mid + 1;
|
||||
else high = mid;
|
||||
}
|
||||
const index = low - 1;
|
||||
if (index < 0) return prepareBlock(source.slice(0, length), canResolveCode, streaming);
|
||||
const cut = prepared.cuts[index]!;
|
||||
// Retain paragraph context if the tail happens to begin with a block marker.
|
||||
const tail = prepareBlock(`x ${source.slice(cut, length)}`, canResolveCode, streaming).slice(2);
|
||||
return prepared.text.slice(0, prepared.offsets[index]) + tail;
|
||||
}
|
||||
|
||||
interface InlinePreparation {
|
||||
source: string;
|
||||
token: Token;
|
||||
@@ -92,15 +139,20 @@ interface InlinePreparation {
|
||||
streaming: boolean;
|
||||
uppercase: boolean;
|
||||
canResolveCode: (text: string) => boolean;
|
||||
protectedRanges: Range[];
|
||||
referenceRanges: Range[];
|
||||
protectedRanges: Uint8Array;
|
||||
referenceRanges: Uint8Array;
|
||||
task: boolean;
|
||||
edits: Edit[];
|
||||
}
|
||||
function insert(context: InlinePreparation, at: number, text: string) {
|
||||
context.edits.push({ start: at, end: at, text });
|
||||
}
|
||||
function contains(ranges: Range[], start: number, end = start + 1): boolean {
|
||||
return ranges.some((range) => start < range.end && end > range.start);
|
||||
function contains(ranges: Uint8Array, start: number, end = start + 1): boolean {
|
||||
for (let i = start; i < end; i++) if (ranges[i]) return true;
|
||||
return false;
|
||||
}
|
||||
function protect(ranges: Uint8Array, span: { start: number; end: number }) {
|
||||
ranges.fill(1, span.start, span.end);
|
||||
}
|
||||
function codeUrl(context: InlinePreparation, content: string): string | null {
|
||||
if (context.canResolveCode(content)) return INLINE_CODE_SCHEME + encodeURIComponent(content);
|
||||
@@ -109,18 +161,12 @@ function codeUrl(context: InlinePreparation, content: string): string | null {
|
||||
return match?.index === 0 && match.lastIndex === content.length ? match.url : null;
|
||||
}
|
||||
function prepareCode(context: InlinePreparation, span: InlineSpan, raw: string, code: Token) {
|
||||
context.protectedRanges.push(span);
|
||||
context.referenceRanges.push(span);
|
||||
const insideLink = context.trace.spans.some(
|
||||
(parent) =>
|
||||
parent.start < span.start &&
|
||||
parent.end >= span.end &&
|
||||
parent.tokens.some((child) => child.type === "link_open"),
|
||||
);
|
||||
protect(context.protectedRanges, span);
|
||||
protect(context.referenceRanges, span);
|
||||
const complete = raw.length >= code.markup.length * 2 && raw.endsWith(code.markup);
|
||||
const closing = complete ? "" : code.markup.slice(raw.match(/`+$/)?.[0].length ?? 0);
|
||||
const url = codeUrl(context, code.content);
|
||||
if (!insideLink && url) {
|
||||
if (!span.insideLink && url) {
|
||||
insert(context, span.start, "[");
|
||||
insert(context, span.end, `${closing}](<${url}>)`);
|
||||
} else if (closing) insert(context, span.end, closing);
|
||||
@@ -135,12 +181,12 @@ function prepareUnfinishedLink(context: InlinePreparation, span: InlineSpan, raw
|
||||
context.edits.push({ start: span.start, end: span.start + start + 1, text: "" });
|
||||
if (start === 1) context.edits.push({ start: span.start + 2, end: span.end, text: "" });
|
||||
else if (end !== -1) context.edits.push({ start: span.start + end, end: span.end, text: "" });
|
||||
context.referenceRanges.push(span);
|
||||
protect(context.referenceRanges, span);
|
||||
}
|
||||
function preserveLinkScope(context: InlinePreparation, span: InlineSpan, raw: string, link: Token) {
|
||||
const end = labelEnd(context, raw, 0);
|
||||
context.protectedRanges.push({ start: span.start + end, end: span.end });
|
||||
context.referenceRanges.push(span);
|
||||
protect(context.protectedRanges, { start: span.start + end, end: span.end });
|
||||
protect(context.referenceRanges, span);
|
||||
if (raw[end + 1] === "(") return;
|
||||
// The native parser sees the joined run; bind reference destinations to their source block.
|
||||
const title = link.attrGet("title");
|
||||
@@ -152,19 +198,25 @@ function preserveLinkScope(context: InlinePreparation, span: InlineSpan, raw: st
|
||||
text: `](<${link.attrGet("href")}>${quotedTitle})`,
|
||||
});
|
||||
}
|
||||
function decodeEntity(context: InlinePreparation, span: InlineSpan, raw: string) {
|
||||
// MD4C 1.1.1 drops entity nodes. Decode once without introducing Markdown syntax.
|
||||
const decoded = context.parser.utils.unescapeAll(raw);
|
||||
let text = decoded.replace(/([\\`*{}[\]()#+.!_<>|~-])/g, "\\$1");
|
||||
if (decoded.endsWith("&") && /^(?:#\w+|\w+);/.test(context.source.slice(span.end)))
|
||||
text = text.slice(0, -1) + "\\&";
|
||||
context.edits.push({ start: span.start, end: span.end, text });
|
||||
protect(context.protectedRanges, span);
|
||||
}
|
||||
function prepareSpan(context: InlinePreparation, span: InlineSpan) {
|
||||
const raw = context.source.slice(span.start, span.end);
|
||||
const link = span.tokens.find((child) => child.type === "link_open");
|
||||
const image = span.tokens.find((child) => child.type === "image");
|
||||
const code = span.tokens.find((child) => child.type === "code_inline");
|
||||
if (
|
||||
image ||
|
||||
raw.startsWith("\\") ||
|
||||
/^&(?:#\w+|\w+);$/.test(raw) ||
|
||||
(link && raw.startsWith("<"))
|
||||
) {
|
||||
context.protectedRanges.push(span);
|
||||
context.referenceRanges.push(span);
|
||||
if (image || raw.startsWith("\\") || (link && raw.startsWith("<"))) {
|
||||
protect(context.protectedRanges, span);
|
||||
protect(context.referenceRanges, span);
|
||||
} else if (/^&(?:#\w+|\w+);$/.test(raw)) {
|
||||
decodeEntity(context, span, raw);
|
||||
} else if (link && raw.startsWith("[")) {
|
||||
preserveLinkScope(context, span, raw, link);
|
||||
} else if (code && raw.startsWith("`")) {
|
||||
@@ -183,7 +235,14 @@ function preserveUnresolvedReferences(context: InlinePreparation) {
|
||||
// Unresolved labels stay literal even when another source block defines them.
|
||||
for (const match of context.source.matchAll(/!?\[[^\]\n]+\](?:\[[^\]\n]*\])?/g)) {
|
||||
if (contains(context.referenceRanges, match.index, match.index + match[0].length)) continue;
|
||||
if (/^\[(?:[ xX]|!(?:NOTE|TIP|IMPORTANT|WARNING|CAUTION))\]$/.test(match[0])) continue;
|
||||
if (/^\[!(?:NOTE|TIP|IMPORTANT|WARNING|CAUTION)\]$/.test(match[0])) continue;
|
||||
if (
|
||||
context.task &&
|
||||
match.index === 0 &&
|
||||
/^\[[ xX]\]$/.test(match[0]) &&
|
||||
/\s/.test(context.source[match[0].length] ?? "")
|
||||
)
|
||||
continue;
|
||||
for (let i = 0; i < match[0].length; i++)
|
||||
if (match[0][i] === "[") insert(context, match.index + i, "\\");
|
||||
}
|
||||
@@ -197,33 +256,30 @@ function prepareText(context: InlinePreparation) {
|
||||
context.edits.push({ start: i, end: i + 1, text: char.toUpperCase() });
|
||||
}
|
||||
}
|
||||
function prepareLinkifiedFiles(context: InlinePreparation) {
|
||||
function prepareLinkifiedLinks(context: InlinePreparation) {
|
||||
const destinations = new Set(
|
||||
context.token.children
|
||||
?.filter((child) => child.type === "link_open" && child.markup === "linkify")
|
||||
.map((child) => child.attrGet("href")),
|
||||
);
|
||||
for (const match of context.parser.linkify.match(context.source) ?? []) {
|
||||
if (
|
||||
contains(context.protectedRanges, match.index, match.lastIndex) ||
|
||||
contains(context.referenceRanges, match.index, match.lastIndex) ||
|
||||
!context.canResolveCode(match.text)
|
||||
contains(context.referenceRanges, match.index, match.lastIndex)
|
||||
)
|
||||
continue;
|
||||
const linkified = context.token.children?.some(
|
||||
(child) =>
|
||||
child.type === "link_open" &&
|
||||
child.markup === "linkify" &&
|
||||
child.attrGet("href") === match.url,
|
||||
);
|
||||
if (!linkified) continue;
|
||||
if (!destinations.has(match.url)) continue;
|
||||
const payload: AssistantFileLinkSource = {
|
||||
href: match.url,
|
||||
text: match.text,
|
||||
markup: "linkify",
|
||||
sourceInfo: "auto",
|
||||
};
|
||||
const href = context.canResolveCode(match.text)
|
||||
? FILE_LINK_SCHEME + encodeURIComponent(JSON.stringify(payload))
|
||||
: match.url;
|
||||
insert(context, match.index, "[");
|
||||
insert(
|
||||
context,
|
||||
match.lastIndex,
|
||||
`](<${FILE_LINK_SCHEME}${encodeURIComponent(JSON.stringify(payload))}>)`,
|
||||
);
|
||||
insert(context, match.lastIndex, `](<${href}>)`);
|
||||
}
|
||||
}
|
||||
function sourcePositions(
|
||||
@@ -250,6 +306,15 @@ function prepareBlock(
|
||||
canResolveCode: (text: string) => boolean,
|
||||
streaming: boolean,
|
||||
): string {
|
||||
return compileBlock(text, canResolveCode, streaming).text;
|
||||
}
|
||||
function compileBlock(
|
||||
text: string,
|
||||
canResolveCode: (text: string) => boolean,
|
||||
streaming: boolean,
|
||||
): PreparedBlock {
|
||||
const normalized = !text.includes("\r");
|
||||
text = text.replace(/\r\n?/g, "\n").replaceAll("\0", "\ufffd");
|
||||
const { parser, traces } = streaming ? provisional : settled;
|
||||
const env = {};
|
||||
const tokens = parser.parse(text, env);
|
||||
@@ -261,9 +326,17 @@ function prepareBlock(
|
||||
offset += line.length + 1;
|
||||
}
|
||||
const edits: Edit[] = [];
|
||||
let cuts: number[] = [];
|
||||
for (const [index, token] of tokens.entries()) {
|
||||
if (token.type !== "inline" || !token.map) continue;
|
||||
const trace = traces.get(token)!;
|
||||
if (
|
||||
normalized &&
|
||||
text === token.content &&
|
||||
tokens.length === 3 &&
|
||||
tokens[0]!.type === "paragraph_open"
|
||||
)
|
||||
cuts = paragraphCuts(token.content, trace);
|
||||
const heading = tokens[index - 1];
|
||||
const context: InlinePreparation = {
|
||||
source: token.content,
|
||||
@@ -274,14 +347,15 @@ function prepareBlock(
|
||||
streaming,
|
||||
uppercase: heading?.type === "heading_open" && heading.tag === "h6",
|
||||
canResolveCode,
|
||||
protectedRanges: [],
|
||||
referenceRanges: [],
|
||||
protectedRanges: new Uint8Array(token.content.length),
|
||||
referenceRanges: new Uint8Array(token.content.length),
|
||||
task: tokens[index - 2]?.type === "list_item_open",
|
||||
edits: [],
|
||||
};
|
||||
for (const span of trace.spans) prepareSpan(context, span);
|
||||
preserveUnresolvedReferences(context);
|
||||
prepareText(context);
|
||||
prepareLinkifiedFiles(context);
|
||||
prepareLinkifiedLinks(context);
|
||||
if (trace.trim)
|
||||
context.edits.push({
|
||||
start: token.content.length - trace.trim,
|
||||
@@ -293,5 +367,28 @@ function prepareBlock(
|
||||
for (const edit of context.edits)
|
||||
edits.push({ ...edit, start: positions[edit.start]!, end: positions[edit.end]! });
|
||||
}
|
||||
return applyEdits(text, edits);
|
||||
const rendered = applyEdits(text, edits);
|
||||
const offsets: number[] = [];
|
||||
let editIndex = 0;
|
||||
let delta = 0;
|
||||
for (const cut of cuts) {
|
||||
while (editIndex < edits.length && edits[editIndex]!.end <= cut) {
|
||||
const edit = edits[editIndex++]!;
|
||||
delta += edit.text.length - (edit.end - edit.start);
|
||||
}
|
||||
offsets.push(cut + delta);
|
||||
}
|
||||
return { text: rendered, cuts, offsets };
|
||||
}
|
||||
function paragraphCuts(source: string, trace: InlineTrace): number[] {
|
||||
const cuts: number[] = [];
|
||||
let depth = 0;
|
||||
for (const span of trace.spans) {
|
||||
if (span.insideLink) continue;
|
||||
if (depth === 0 && span.tokens.length === 0 && source[span.start] !== "&") {
|
||||
for (let i = span.start; i < span.end; i++) if (source[i] === " ") cuts.push(i + 1);
|
||||
}
|
||||
for (const token of span.tokens) depth += token.nesting;
|
||||
}
|
||||
return cuts;
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ import { useStableEvent } from "@/hooks/use-stable-event";
|
||||
import { createMarkdownStyles } from "@/styles/markdown-styles";
|
||||
import type { Theme } from "@/styles/theme";
|
||||
import type { MarkdownProseProps } from "./props";
|
||||
import { handleProseLinkPress, prepareProse } from "./prepare";
|
||||
import { handleProseLinkPress, createProsePreparer } from "./prepare";
|
||||
|
||||
const ThemedMarkdown = withUnistyles(EnrichedMarkdownText);
|
||||
const MD4C_FLAGS = { latexMath: false, hardSoftBreaks: true, admonitions: true };
|
||||
@@ -104,9 +104,10 @@ const NativeProse = memo(function NativeProse({
|
||||
const canResolveCode = useStableEvent((content: string) =>
|
||||
actions.canResolveFile({ href: content, text: content, sourceType: "inline-code" }),
|
||||
);
|
||||
const prepare = useMemo(() => createProsePreparer(canResolveCode), [canResolveCode]);
|
||||
const markdown = useMemo(
|
||||
() => prepareProse(sourceBlocks, canResolveCode, streaming, text.length),
|
||||
[text.length, sourceBlocks, canResolveCode, streaming],
|
||||
() => prepare(sourceBlocks, streaming, text.length),
|
||||
[text.length, sourceBlocks, prepare, streaming],
|
||||
);
|
||||
const handleLinkPress = useStableEvent(({ url }: { url: string }) => {
|
||||
handleProseLinkPress(url, (source) => actions.open(source, "preferred"), onLinkPress);
|
||||
|
||||
@@ -3,7 +3,7 @@ import { createAssistantMarkdownParser } from "@/utils/assistant-markdown-parser
|
||||
import type Token from "markdown-it/lib/token.mjs";
|
||||
import type { AssistantMessageItem } from "@/types/stream";
|
||||
|
||||
const parser = createAssistantMarkdownParser();
|
||||
const parser = createAssistantMarkdownParser().set({ html: true });
|
||||
const partsByBlock = new WeakMap<AssistantMessageItem, AssistantMessageItem[]>();
|
||||
|
||||
function splitBlock(block: AssistantMessageItem): AssistantMessageItem[] {
|
||||
@@ -14,11 +14,7 @@ function splitBlock(block: AssistantMessageItem): AssistantMessageItem[] {
|
||||
const preserved = tokens.some(
|
||||
(token) =>
|
||||
["fence", "code_block", "table_open", "html_block"].includes(token.type) ||
|
||||
token.children?.some(
|
||||
(child) =>
|
||||
child.type === "html_inline" ||
|
||||
(child.type === "text" && /<\/?[A-Za-z][^>]*>/.test(child.content)),
|
||||
),
|
||||
token.children?.some((child) => child.type === "html_inline"),
|
||||
);
|
||||
const isQuote = tokens.some((token) => token.type === "blockquote_open");
|
||||
const hasImage = tokens.some((token) => token.children?.some((child) => child.type === "image"));
|
||||
|
||||
@@ -5,6 +5,7 @@ export interface InlineSpan {
|
||||
start: number;
|
||||
end: number;
|
||||
tokens: Token[];
|
||||
insideLink: boolean;
|
||||
}
|
||||
export interface InlineTrace {
|
||||
spans: InlineSpan[];
|
||||
@@ -30,9 +31,14 @@ export function createSourceParser(streaming: boolean) {
|
||||
for (const rule of rules) {
|
||||
const start = state.pos;
|
||||
const count = state.tokens.length;
|
||||
const trace: InlineTrace = state.env[TRACE];
|
||||
const firstNested = trace.spans.length;
|
||||
if (rule(state, false)) {
|
||||
const trace: InlineTrace = state.env[TRACE];
|
||||
trace.spans.push({ start, end: state.pos, tokens: state.tokens.slice(count) });
|
||||
const tokens = state.tokens.slice(count);
|
||||
if (tokens.some((token) => token.type === "link_open")) {
|
||||
for (let i = firstNested; i < trace.spans.length; i++) trace.spans[i]!.insideLink = true;
|
||||
}
|
||||
trace.spans.push({ start, end: state.pos, tokens, insideLink: false });
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user