mirror of
https://github.com/getpaseo/paseo.git
synced 2026-10-27 11:42:52 +00:00
fix(server): keep image materialization responsive
Decode and hash attachments in bounded yielding units, deduplicate before writing, and publish complete private files before provider timeline events. Await mapping through existing replay boundaries and serialize notification dispatch so turn completion cannot overtake an image.
This commit is contained in:
@@ -13,8 +13,8 @@ const MATERIALIZED_PNG_PATH_PATTERN =
|
||||
/paseo-attachments(?:-[^\\/]+)?[\\/](?:[^\\/]+[\\/])?[0-9a-f]{64}\.png$/;
|
||||
|
||||
interface ClaudeImageTestSession {
|
||||
translateMessageToEvents(message: SDKMessage): AgentStreamEvent[];
|
||||
convertHistoryEntry(entry: unknown): AgentTimelineItem[];
|
||||
translateMessageToEvents(message: SDKMessage): Promise<AgentStreamEvent[]>;
|
||||
convertHistoryEntry(entry: unknown): Promise<AgentTimelineItem[]>;
|
||||
}
|
||||
|
||||
async function createSession(): Promise<ClaudeImageTestSession> {
|
||||
@@ -157,7 +157,7 @@ describe("Claude tool_result image rendering", () => {
|
||||
test("emits the image as assistant markdown and keeps base64 out of the live tool output", async () => {
|
||||
const session = await createSession();
|
||||
|
||||
const events = session.translateMessageToEvents(imageToolResultUserMessage());
|
||||
const events = await session.translateMessageToEvents(imageToolResultUserMessage());
|
||||
|
||||
const timelineItems = events
|
||||
.filter((event) => event.type === "timeline")
|
||||
@@ -181,7 +181,7 @@ describe("Claude tool_result image rendering", () => {
|
||||
test("replays the image as assistant markdown through history conversion", async () => {
|
||||
const session = await createSession();
|
||||
|
||||
const items = session.convertHistoryEntry(imageToolResultHistoryEntry());
|
||||
const items = await session.convertHistoryEntry(imageToolResultHistoryEntry());
|
||||
|
||||
const [imageMessage, ...extraImages] = imageMessages(items);
|
||||
expect(extraImages).toEqual([]);
|
||||
@@ -202,7 +202,7 @@ describe("Claude tool_result image rendering", () => {
|
||||
test("keeps base64 out of an errored tool_result that carries an image", async () => {
|
||||
const session = await createSession();
|
||||
|
||||
const events = session.translateMessageToEvents(erroredImageToolResultUserMessage());
|
||||
const events = await session.translateMessageToEvents(erroredImageToolResultUserMessage());
|
||||
|
||||
const timelineItems = events
|
||||
.filter((event) => event.type === "timeline")
|
||||
@@ -226,7 +226,7 @@ describe("Claude tool_result image rendering", () => {
|
||||
test("emits one image message per image block in a multi-image tool_result", async () => {
|
||||
const session = await createSession();
|
||||
|
||||
const events = session.translateMessageToEvents(multiImageToolResultUserMessage());
|
||||
const events = await session.translateMessageToEvents(multiImageToolResultUserMessage());
|
||||
|
||||
const timelineItems = events
|
||||
.filter((event) => event.type === "timeline")
|
||||
|
||||
@@ -32,7 +32,7 @@ import type { AgentAttachment } from "@getpaseo/protocol/messages";
|
||||
import { buildAgentPrompt, renderPromptAttachmentAsText } from "../../prompt-attachments.js";
|
||||
|
||||
interface TestClaudeSession {
|
||||
translateMessageToEvents(message: SDKMessage): AgentStreamEvent[];
|
||||
translateMessageToEvents(message: SDKMessage): Promise<AgentStreamEvent[]>;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
@@ -53,7 +53,7 @@ afterEach(() => {
|
||||
});
|
||||
|
||||
describe("convertClaudeHistoryEntry", () => {
|
||||
test("maps user tool results to timeline items", () => {
|
||||
test("maps user tool results to timeline items", async () => {
|
||||
const toolUseId = "toolu_test";
|
||||
const entry = {
|
||||
type: "user",
|
||||
@@ -79,14 +79,14 @@ describe("convertClaudeHistoryEntry", () => {
|
||||
];
|
||||
|
||||
const mapBlocks = vi.fn().mockReturnValue(stubTimeline);
|
||||
const result = convertClaudeHistoryEntry(entry, mapBlocks);
|
||||
const result = await convertClaudeHistoryEntry(entry, mapBlocks);
|
||||
|
||||
expect(result).toEqual(stubTimeline);
|
||||
expect(mapBlocks).toHaveBeenCalledTimes(1);
|
||||
expect(Array.isArray(mapBlocks.mock.calls[0][0])).toBe(true);
|
||||
});
|
||||
|
||||
test("replays persisted Claude tool results as completed tool calls", () => {
|
||||
test("replays persisted Claude tool results as completed tool calls", async () => {
|
||||
const entry = {
|
||||
type: "user",
|
||||
message: {
|
||||
@@ -124,12 +124,12 @@ describe("convertClaudeHistoryEntry", () => {
|
||||
|
||||
const mapPersistedToolResultBlocks = (): AgentTimelineItem[] => completedToolCall;
|
||||
|
||||
expect(convertClaudeHistoryEntry(entry, mapPersistedToolResultBlocks)).toEqual(
|
||||
expect(await convertClaudeHistoryEntry(entry, mapPersistedToolResultBlocks)).toEqual(
|
||||
completedToolCall,
|
||||
);
|
||||
});
|
||||
|
||||
test("returns user messages when no tool blocks exist", () => {
|
||||
test("returns user messages when no tool blocks exist", async () => {
|
||||
const entry = {
|
||||
type: "user",
|
||||
message: {
|
||||
@@ -138,7 +138,7 @@ describe("convertClaudeHistoryEntry", () => {
|
||||
},
|
||||
};
|
||||
|
||||
expect(convertClaudeHistoryEntry(entry, () => [])).toEqual([
|
||||
expect(await convertClaudeHistoryEntry(entry, () => [])).toEqual([
|
||||
{
|
||||
type: "user_message",
|
||||
text: "Run npm test",
|
||||
@@ -146,7 +146,7 @@ describe("convertClaudeHistoryEntry", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
test("converts compact boundary metadata variants", () => {
|
||||
test("converts compact boundary metadata variants", async () => {
|
||||
const fixtures = [
|
||||
{
|
||||
entry: {
|
||||
@@ -175,7 +175,7 @@ describe("convertClaudeHistoryEntry", () => {
|
||||
] as const;
|
||||
|
||||
for (const fixture of fixtures) {
|
||||
expect(convertClaudeHistoryEntry(fixture.entry, () => [])).toEqual([
|
||||
expect(await convertClaudeHistoryEntry(fixture.entry, () => [])).toEqual([
|
||||
{
|
||||
type: "compaction",
|
||||
status: "completed",
|
||||
@@ -186,7 +186,7 @@ describe("convertClaudeHistoryEntry", () => {
|
||||
}
|
||||
});
|
||||
|
||||
test("skips synthetic user entries", () => {
|
||||
test("skips synthetic user entries", async () => {
|
||||
const entry = {
|
||||
type: "user",
|
||||
isSynthetic: true,
|
||||
@@ -197,13 +197,13 @@ describe("convertClaudeHistoryEntry", () => {
|
||||
};
|
||||
|
||||
const mapBlocks = vi.fn().mockReturnValue([]);
|
||||
const result = convertClaudeHistoryEntry(entry, mapBlocks);
|
||||
const result = await convertClaudeHistoryEntry(entry, mapBlocks);
|
||||
|
||||
expect(result).toEqual([]);
|
||||
expect(mapBlocks).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("skips meta user entries from Claude skill loading", () => {
|
||||
test("skips meta user entries from Claude skill loading", async () => {
|
||||
const entry = {
|
||||
type: "user",
|
||||
isMeta: true,
|
||||
@@ -219,13 +219,13 @@ describe("convertClaudeHistoryEntry", () => {
|
||||
};
|
||||
|
||||
const mapBlocks = vi.fn().mockReturnValue([]);
|
||||
const result = convertClaudeHistoryEntry(entry, mapBlocks);
|
||||
const result = await convertClaudeHistoryEntry(entry, mapBlocks);
|
||||
|
||||
expect(result).toEqual([]);
|
||||
expect(mapBlocks).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("skips interrupt placeholder transcript noise", () => {
|
||||
test("skips interrupt placeholder transcript noise", async () => {
|
||||
const interruptEntry = {
|
||||
type: "user",
|
||||
message: {
|
||||
@@ -246,11 +246,11 @@ describe("convertClaudeHistoryEntry", () => {
|
||||
.fn()
|
||||
.mockReturnValue([{ type: "assistant_message", text: "No response requested." }]);
|
||||
|
||||
expect(convertClaudeHistoryEntry(interruptEntry, mapBlocks)).toEqual([]);
|
||||
expect(convertClaudeHistoryEntry(assistantNoiseEntry, mapBlocks)).toEqual([]);
|
||||
expect(await convertClaudeHistoryEntry(interruptEntry, mapBlocks)).toEqual([]);
|
||||
expect(await convertClaudeHistoryEntry(assistantNoiseEntry, mapBlocks)).toEqual([]);
|
||||
});
|
||||
|
||||
test("skips <local-command-stdout> messages (model switch, /context, etc.)", () => {
|
||||
test("skips <local-command-stdout> messages (model switch, /context, etc.)", async () => {
|
||||
// Real entries from Claude Code JSONL history files
|
||||
const modelSwitch = {
|
||||
type: "user",
|
||||
@@ -305,24 +305,24 @@ describe("convertClaudeHistoryEntry", () => {
|
||||
|
||||
const mapBlocks = vi.fn().mockReturnValue([]);
|
||||
|
||||
expect(convertClaudeHistoryEntry(modelSwitch, mapBlocks)).toEqual([]);
|
||||
expect(convertClaudeHistoryEntry(modelSwitchWithAnsi, mapBlocks)).toEqual([]);
|
||||
expect(convertClaudeHistoryEntry(contextDump, mapBlocks)).toEqual([]);
|
||||
expect(convertClaudeHistoryEntry(planMode, mapBlocks)).toEqual([]);
|
||||
expect(convertClaudeHistoryEntry(goodbye, mapBlocks)).toEqual([]);
|
||||
expect(convertClaudeHistoryEntry(empty, mapBlocks)).toEqual([]);
|
||||
expect(await convertClaudeHistoryEntry(modelSwitch, mapBlocks)).toEqual([]);
|
||||
expect(await convertClaudeHistoryEntry(modelSwitchWithAnsi, mapBlocks)).toEqual([]);
|
||||
expect(await convertClaudeHistoryEntry(contextDump, mapBlocks)).toEqual([]);
|
||||
expect(await convertClaudeHistoryEntry(planMode, mapBlocks)).toEqual([]);
|
||||
expect(await convertClaudeHistoryEntry(goodbye, mapBlocks)).toEqual([]);
|
||||
expect(await convertClaudeHistoryEntry(empty, mapBlocks)).toEqual([]);
|
||||
|
||||
// Real user messages must NOT be filtered
|
||||
const realMessage = {
|
||||
type: "user",
|
||||
message: { role: "user", content: "fix the bug in auth.ts" },
|
||||
};
|
||||
expect(convertClaudeHistoryEntry(realMessage, mapBlocks)).toEqual([
|
||||
expect(await convertClaudeHistoryEntry(realMessage, mapBlocks)).toEqual([
|
||||
{ type: "user_message", text: "fix the bug in auth.ts" },
|
||||
]);
|
||||
});
|
||||
|
||||
test("maps task notifications to synthetic tool calls", () => {
|
||||
test("maps task notifications to synthetic tool calls", async () => {
|
||||
const entry = {
|
||||
type: "system",
|
||||
subtype: "task_notification",
|
||||
@@ -333,7 +333,7 @@ describe("convertClaudeHistoryEntry", () => {
|
||||
output_file: "/tmp/bg-fail-1.txt",
|
||||
};
|
||||
|
||||
expect(convertClaudeHistoryEntry(entry, () => [])).toEqual([
|
||||
expect(await convertClaudeHistoryEntry(entry, () => [])).toEqual([
|
||||
{
|
||||
type: "tool_call",
|
||||
callId: "task_notification_task-note-system-1",
|
||||
@@ -357,7 +357,7 @@ describe("convertClaudeHistoryEntry", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
test("maps queue-operation task notifications to synthetic tool calls", () => {
|
||||
test("maps queue-operation task notifications to synthetic tool calls", async () => {
|
||||
const entry = {
|
||||
type: "queue-operation",
|
||||
operation: "enqueue",
|
||||
@@ -372,7 +372,7 @@ describe("convertClaudeHistoryEntry", () => {
|
||||
].join("\n"),
|
||||
};
|
||||
|
||||
expect(convertClaudeHistoryEntry(entry, () => [])).toEqual([
|
||||
expect(await convertClaudeHistoryEntry(entry, () => [])).toEqual([
|
||||
{
|
||||
type: "tool_call",
|
||||
callId: "task_notification_task-note-queue-1",
|
||||
@@ -396,7 +396,7 @@ describe("convertClaudeHistoryEntry", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
test("passes assistant content blocks through to the mapper", () => {
|
||||
test("passes assistant content blocks through to the mapper", async () => {
|
||||
const entry = {
|
||||
type: "assistant",
|
||||
message: {
|
||||
@@ -414,7 +414,7 @@ describe("convertClaudeHistoryEntry", () => {
|
||||
];
|
||||
const mapBlocks = vi.fn().mockReturnValue(mappedTimeline);
|
||||
|
||||
expect(convertClaudeHistoryEntry(entry, mapBlocks)).toEqual(mappedTimeline);
|
||||
expect(await convertClaudeHistoryEntry(entry, mapBlocks)).toEqual(mappedTimeline);
|
||||
expect(mapBlocks).toHaveBeenCalledWith(entry.message.content);
|
||||
});
|
||||
});
|
||||
@@ -1208,7 +1208,7 @@ describe("ClaudeAgentSession features", () => {
|
||||
input: Record<string, unknown>,
|
||||
options: Record<string, unknown>,
|
||||
): Promise<PermissionResult>;
|
||||
translateMessageToEvents(message: SDKMessage): AgentStreamEvent[];
|
||||
translateMessageToEvents(message: SDKMessage): Promise<AgentStreamEvent[]>;
|
||||
};
|
||||
|
||||
try {
|
||||
@@ -1248,7 +1248,7 @@ describe("ClaudeAgentSession features", () => {
|
||||
),
|
||||
).resolves.toMatchObject({ behavior: "deny", interrupt: undefined });
|
||||
|
||||
internal.translateMessageToEvents({
|
||||
await internal.translateMessageToEvents({
|
||||
type: "command_lifecycle",
|
||||
command_uuid: steerUuid,
|
||||
state: "started",
|
||||
@@ -2395,7 +2395,7 @@ describe("ClaudeAgentSession context window usage", () => {
|
||||
test("emits canonical task snapshots from Claude TaskCreate results", async () => {
|
||||
const session = await createSessionForTest();
|
||||
try {
|
||||
session.translateMessageToEvents({
|
||||
await session.translateMessageToEvents({
|
||||
type: "assistant",
|
||||
message: {
|
||||
content: [
|
||||
@@ -2409,7 +2409,7 @@ describe("ClaudeAgentSession context window usage", () => {
|
||||
},
|
||||
} as unknown as SDKMessage);
|
||||
|
||||
const events = session.translateMessageToEvents({
|
||||
const events = await session.translateMessageToEvents({
|
||||
type: "user",
|
||||
message: {
|
||||
content: [{ type: "tool_result", tool_use_id: "task-create-1", content: "created" }],
|
||||
@@ -3611,9 +3611,9 @@ describe("ClaudeAgentSession context window usage", () => {
|
||||
]);
|
||||
|
||||
try {
|
||||
const compactEvents = (session as unknown as TestClaudeSession).translateMessageToEvents(
|
||||
createCompactBoundary(),
|
||||
);
|
||||
const compactEvents = await (
|
||||
session as unknown as TestClaudeSession
|
||||
).translateMessageToEvents(createCompactBoundary());
|
||||
expect(compactEvents).toContainEqual(
|
||||
expect.objectContaining({
|
||||
type: "usage_updated",
|
||||
@@ -3799,7 +3799,7 @@ describe("ClaudeAgentSession context window usage", () => {
|
||||
test("result.result is surfaced as an assistant message when no model output was produced", async () => {
|
||||
const session = await createSessionForTest();
|
||||
|
||||
const events = session.translateMessageToEvents({
|
||||
const events = await session.translateMessageToEvents({
|
||||
type: "result",
|
||||
subtype: "success",
|
||||
result: "Unknown command: /foo-doesnt-exist",
|
||||
@@ -3834,7 +3834,7 @@ describe("ClaudeAgentSession context window usage", () => {
|
||||
test("result.result is not duplicated when the model produced output during the turn", async () => {
|
||||
const session = await createSessionForTest();
|
||||
|
||||
const events = session.translateMessageToEvents({
|
||||
const events = await session.translateMessageToEvents({
|
||||
type: "result",
|
||||
subtype: "success",
|
||||
result: "Here is the answer.",
|
||||
|
||||
@@ -2546,7 +2546,11 @@ class ClaudeAgentSession implements AgentSession {
|
||||
this.input.push(sdkMessage);
|
||||
setTimeout(() => {
|
||||
if (this.activeForegroundTurnId === turnId) {
|
||||
this.emitSubmittedUserMessage(sdkMessage, turnId, options?.clientMessageId);
|
||||
void this.emitSubmittedUserMessage(sdkMessage, turnId, options?.clientMessageId).catch(
|
||||
(error: unknown) => {
|
||||
this.logger.warn({ err: error }, "Failed to emit submitted Claude message");
|
||||
},
|
||||
);
|
||||
}
|
||||
}, 0);
|
||||
} catch (error) {
|
||||
@@ -4133,7 +4137,7 @@ class ClaudeAgentSession implements AgentSession {
|
||||
consecutiveInterruptAbortRecoveries = 0;
|
||||
if (await this.handleMissingResumedConversation(message, activeQuery)) return true;
|
||||
if (!this.ownsQuery(activeQuery)) return true;
|
||||
this.routeSdkMessageFromPump(message);
|
||||
await this.routeSdkMessageFromPump(message);
|
||||
return false;
|
||||
} finally {
|
||||
this.processingSdkMessages--;
|
||||
@@ -4257,7 +4261,7 @@ class ClaudeAgentSession implements AgentSession {
|
||||
return !this.closed && this.query === query;
|
||||
}
|
||||
|
||||
private routeSdkMessageFromPump(message: SDKMessage): void {
|
||||
private async routeSdkMessageFromPump(message: SDKMessage): Promise<void> {
|
||||
if (this.consumeSessionStateChange(message)) {
|
||||
return;
|
||||
}
|
||||
@@ -4291,7 +4295,7 @@ class ClaudeAgentSession implements AgentSession {
|
||||
"provider.claude.parsed_event",
|
||||
);
|
||||
|
||||
const events = this.buildPumpedMessageEvents(message, identifiers.messageId, turnId);
|
||||
const events = await this.buildPumpedMessageEvents(message, identifiers.messageId, turnId);
|
||||
|
||||
if (events.length === 0) {
|
||||
return;
|
||||
@@ -4326,15 +4330,22 @@ class ClaudeAgentSession implements AgentSession {
|
||||
this.dispatchEvents(events);
|
||||
}
|
||||
|
||||
private buildPumpedMessageEvents(
|
||||
private async buildPumpedMessageEvents(
|
||||
message: SDKMessage,
|
||||
messageIdHint: string | null,
|
||||
turnId: string | null,
|
||||
): AgentStreamEvent[] {
|
||||
const messageEvents = this.translateMessageToEvents(message, {
|
||||
): Promise<AgentStreamEvent[]> {
|
||||
const query = this.query;
|
||||
const messageEvents = await this.translateMessageToEvents(message, {
|
||||
suppressAssistantText: true,
|
||||
suppressReasoning: true,
|
||||
});
|
||||
if (
|
||||
this.closed ||
|
||||
this.query !== query ||
|
||||
(this.activeForegroundTurnId ?? this.autonomousTurn?.id ?? null) !== turnId
|
||||
)
|
||||
return [];
|
||||
const assistantTimelineEvents = readClaudeParentToolUseId(message)
|
||||
? []
|
||||
: this.timelineAssembler
|
||||
@@ -4479,13 +4490,13 @@ class ClaudeAgentSession implements AgentSession {
|
||||
return withdrewAll;
|
||||
}
|
||||
|
||||
private translateMessageToEvents(
|
||||
private async translateMessageToEvents(
|
||||
message: SDKMessage,
|
||||
options?: {
|
||||
suppressAssistantText?: boolean;
|
||||
suppressReasoning?: boolean;
|
||||
},
|
||||
): AgentStreamEvent[] {
|
||||
): Promise<AgentStreamEvent[]> {
|
||||
const parentToolUseId = readClaudeParentToolUseId(message);
|
||||
if (parentToolUseId) {
|
||||
return this.translateSidechainFrameToEvents(message, parentToolUseId);
|
||||
@@ -4533,11 +4544,11 @@ class ClaudeAgentSession implements AgentSession {
|
||||
this.appendSystemMessageEvents(message, events);
|
||||
break;
|
||||
case "user":
|
||||
this.appendUserMessageEvents(message, events);
|
||||
await this.appendUserMessageEvents(message, events);
|
||||
this.appendSidechainResultEvents(message, events);
|
||||
break;
|
||||
case "assistant": {
|
||||
const timelineItems = this.mapBlocksToTimeline(message.message.content, {
|
||||
const timelineItems = await this.mapBlocksToTimeline(message.message.content, {
|
||||
suppressAssistantText: options?.suppressAssistantText ?? false,
|
||||
suppressReasoning: options?.suppressReasoning ?? false,
|
||||
});
|
||||
@@ -4548,7 +4559,7 @@ class ClaudeAgentSession implements AgentSession {
|
||||
break;
|
||||
}
|
||||
case "stream_event":
|
||||
this.appendStreamEventEvents(message, events, options);
|
||||
await this.appendStreamEventEvents(message, events, options);
|
||||
break;
|
||||
case "result":
|
||||
this.appendResultEvents(message, events);
|
||||
@@ -4650,13 +4661,14 @@ class ClaudeAgentSession implements AgentSession {
|
||||
}
|
||||
}
|
||||
|
||||
private emitSubmittedUserMessage(
|
||||
private async emitSubmittedUserMessage(
|
||||
message: Extract<SDKMessage, { type: "user" }>,
|
||||
turnId: string,
|
||||
clientMessageId?: string,
|
||||
): void {
|
||||
): Promise<void> {
|
||||
const events: AgentStreamEvent[] = [];
|
||||
this.appendUserMessageEvents(message, events);
|
||||
await this.appendUserMessageEvents(message, events);
|
||||
if (this.activeForegroundTurnId !== turnId) return;
|
||||
if (events.length === 0) {
|
||||
return;
|
||||
}
|
||||
@@ -4786,10 +4798,10 @@ class ClaudeAgentSession implements AgentSession {
|
||||
}
|
||||
}
|
||||
|
||||
private appendUserMessageEvents(
|
||||
private async appendUserMessageEvents(
|
||||
message: Extract<SDKMessage, { type: "user" }>,
|
||||
events: AgentStreamEvent[],
|
||||
): void {
|
||||
): Promise<void> {
|
||||
if (isSyntheticUserEntry(message)) {
|
||||
return;
|
||||
}
|
||||
@@ -4828,7 +4840,7 @@ class ClaudeAgentSession implements AgentSession {
|
||||
return;
|
||||
}
|
||||
if (Array.isArray(content)) {
|
||||
this.appendUserContentArrayEvents(content, messageId, events);
|
||||
await this.appendUserContentArrayEvents(content, messageId, events);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4854,12 +4866,12 @@ class ClaudeAgentSession implements AgentSession {
|
||||
events.push({ type: "timeline", item, provider: "claude" });
|
||||
}
|
||||
|
||||
private appendUserContentArrayEvents(
|
||||
private async appendUserContentArrayEvents(
|
||||
content: ReadonlyArray<unknown>,
|
||||
messageId: string | undefined,
|
||||
events: AgentStreamEvent[],
|
||||
): void {
|
||||
const timelineItems = this.mapBlocksToTimeline(content, {
|
||||
): Promise<void> {
|
||||
const timelineItems = await this.mapBlocksToTimeline(content, {
|
||||
textMessageType: "user_message",
|
||||
});
|
||||
for (const item of timelineItems) {
|
||||
@@ -4875,16 +4887,16 @@ class ClaudeAgentSession implements AgentSession {
|
||||
}
|
||||
}
|
||||
|
||||
private appendStreamEventEvents(
|
||||
private async appendStreamEventEvents(
|
||||
message: Extract<SDKMessage, { type: "stream_event" }>,
|
||||
events: AgentStreamEvent[],
|
||||
options: { suppressAssistantText?: boolean; suppressReasoning?: boolean } | undefined,
|
||||
): void {
|
||||
): Promise<void> {
|
||||
const usageUpdatedEvent = this.contextUsage.buildStreamUsageEvent(message.event);
|
||||
if (usageUpdatedEvent) {
|
||||
events.push(usageUpdatedEvent);
|
||||
}
|
||||
const timelineItems = this.mapPartialEvent(message.event, {
|
||||
const timelineItems = await this.mapPartialEvent(message.event, {
|
||||
suppressAssistantText: options?.suppressAssistantText ?? false,
|
||||
suppressReasoning: options?.suppressReasoning ?? false,
|
||||
});
|
||||
@@ -5350,7 +5362,7 @@ class ClaudeAgentSession implements AgentSession {
|
||||
): Promise<void> {
|
||||
const timeline: PersistedTimelineEntry[] = [];
|
||||
for await (const entry of entries) {
|
||||
this.ingestPersistedHistoryEntry(entry, timeline, replay);
|
||||
await this.ingestPersistedHistoryEntry(entry, timeline, replay);
|
||||
}
|
||||
|
||||
if (timeline.length > 0) {
|
||||
@@ -5410,11 +5422,11 @@ class ClaudeAgentSession implements AgentSession {
|
||||
return replay;
|
||||
}
|
||||
|
||||
private ingestPersistedHistoryEntry(
|
||||
private async ingestPersistedHistoryEntry(
|
||||
entry: ClaudeHistoryEntry,
|
||||
timeline: PersistedTimelineEntry[],
|
||||
replay: ClaudeReplayOwnership,
|
||||
): void {
|
||||
): Promise<void> {
|
||||
if (entry.isSidechain) {
|
||||
return;
|
||||
}
|
||||
@@ -5428,7 +5440,7 @@ class ClaudeAgentSession implements AgentSession {
|
||||
? replay.toolOwners.get(notificationToolUseId)
|
||||
: undefined;
|
||||
if (notificationOwner) {
|
||||
for (const item of this.convertHistoryEntry(entry)) {
|
||||
for (const item of await this.convertHistoryEntry(entry)) {
|
||||
this.persistedProviderSubagentEvents.push({
|
||||
type: "provider_subagent",
|
||||
provider: "claude",
|
||||
@@ -5445,7 +5457,7 @@ class ClaudeAgentSession implements AgentSession {
|
||||
const taskSnapshot = this.taskState.observe(entry);
|
||||
const items = [
|
||||
...(taskSnapshot ? [taskSnapshot] : []),
|
||||
...this.convertHistoryEntry(entry).map((item) =>
|
||||
...(await this.convertHistoryEntry(entry)).map((item) =>
|
||||
labelReplayedSubagentCard(item, replay.subagentToolCalls),
|
||||
),
|
||||
];
|
||||
@@ -5482,10 +5494,10 @@ class ClaudeAgentSession implements AgentSession {
|
||||
});
|
||||
}
|
||||
|
||||
private convertHistoryEntry(entry: ClaudeHistoryEntry): AgentTimelineItem[] {
|
||||
return convertClaudeHistoryEntry(entry, (content) => this.mapBlocksToTimeline(content)).map(
|
||||
limitAgentTimelineItemContent,
|
||||
);
|
||||
private async convertHistoryEntry(entry: ClaudeHistoryEntry): Promise<AgentTimelineItem[]> {
|
||||
return (
|
||||
await convertClaudeHistoryEntry(entry, (content) => this.mapBlocksToTimeline(content))
|
||||
).map(limitAgentTimelineItemContent);
|
||||
}
|
||||
|
||||
// Maps Claude content blocks into AgentTimelineItems.
|
||||
@@ -5500,14 +5512,14 @@ class ClaudeAgentSession implements AgentSession {
|
||||
//
|
||||
// NOTE: convertClaudeHistoryEntry uses extractUserMessageText directly instead of this function
|
||||
// for user entries. Both paths must produce equivalent user_message items.
|
||||
private mapBlocksToTimeline(
|
||||
private async mapBlocksToTimeline(
|
||||
content: string | ReadonlyArray<unknown>,
|
||||
options?: {
|
||||
textMessageType?: "assistant_message" | "user_message";
|
||||
suppressAssistantText?: boolean;
|
||||
suppressReasoning?: boolean;
|
||||
},
|
||||
): AgentTimelineItem[] {
|
||||
): Promise<AgentTimelineItem[]> {
|
||||
const textMessageType = options?.textMessageType ?? "assistant_message";
|
||||
const suppressText =
|
||||
textMessageType === "assistant_message" && (options?.suppressAssistantText ?? false);
|
||||
@@ -5534,7 +5546,7 @@ class ClaudeAgentSession implements AgentSession {
|
||||
if (!isClaudeContentChunk(block) || this.mapSubagentHandbackBlock(block, items)) {
|
||||
continue;
|
||||
}
|
||||
this.mapBlockToTimeline(block, {
|
||||
await this.mapBlockToTimeline(block, {
|
||||
items,
|
||||
userTextParts,
|
||||
textMessageType,
|
||||
@@ -5579,7 +5591,7 @@ class ClaudeAgentSession implements AgentSession {
|
||||
}
|
||||
}
|
||||
|
||||
private mapBlockToTimeline(
|
||||
private async mapBlockToTimeline(
|
||||
block: ClaudeContentChunk,
|
||||
context: {
|
||||
items: AgentTimelineItem[];
|
||||
@@ -5588,7 +5600,7 @@ class ClaudeAgentSession implements AgentSession {
|
||||
suppressText: boolean;
|
||||
suppressReasoning: boolean;
|
||||
},
|
||||
): void {
|
||||
): Promise<void> {
|
||||
switch (block.type) {
|
||||
case "text":
|
||||
case "text_delta":
|
||||
@@ -5612,7 +5624,7 @@ class ClaudeAgentSession implements AgentSession {
|
||||
case "code_execution_tool_result":
|
||||
case "bash_code_execution_tool_result":
|
||||
case "text_editor_code_execution_tool_result":
|
||||
this.handleToolResult(block, context.items);
|
||||
await this.handleToolResult(block, context.items);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
@@ -5658,7 +5670,10 @@ class ClaudeAgentSession implements AgentSession {
|
||||
);
|
||||
}
|
||||
|
||||
private handleToolResult(block: ClaudeContentChunk, items: AgentTimelineItem[]): void {
|
||||
private async handleToolResult(
|
||||
block: ClaudeContentChunk,
|
||||
items: AgentTimelineItem[],
|
||||
): Promise<void> {
|
||||
const entry =
|
||||
typeof block.tool_use_id === "string" ? this.toolUseCache.get(block.tool_use_id) : undefined;
|
||||
const blockToolName = typeof block.tool_name === "string" ? block.tool_name : undefined;
|
||||
@@ -5691,7 +5706,7 @@ class ClaudeAgentSession implements AgentSession {
|
||||
this.rememberSettledSubagentToolCall(settledToolCall, entry);
|
||||
|
||||
for (const image of images) {
|
||||
const imageItem = renderProviderImageOutputAsAssistantMarkdown(image, {
|
||||
const imageItem = await renderProviderImageOutputAsAssistantMarkdown(image, {
|
||||
materialize: materializeProviderImage,
|
||||
});
|
||||
if (imageItem) {
|
||||
@@ -5915,13 +5930,13 @@ class ClaudeAgentSession implements AgentSession {
|
||||
return false;
|
||||
}
|
||||
|
||||
private mapPartialEvent(
|
||||
private async mapPartialEvent(
|
||||
event: SDKPartialAssistantMessage["event"],
|
||||
options?: {
|
||||
suppressAssistantText?: boolean;
|
||||
suppressReasoning?: boolean;
|
||||
},
|
||||
): AgentTimelineItem[] {
|
||||
): Promise<AgentTimelineItem[]> {
|
||||
if (this.updatePartialEventToolState(event)) {
|
||||
return [];
|
||||
}
|
||||
@@ -5929,14 +5944,14 @@ class ClaudeAgentSession implements AgentSession {
|
||||
switch (event.type) {
|
||||
case "content_block_start":
|
||||
return isClaudeContentChunk(event.content_block)
|
||||
? this.mapBlocksToTimeline([event.content_block], {
|
||||
? await this.mapBlocksToTimeline([event.content_block], {
|
||||
suppressAssistantText: options?.suppressAssistantText,
|
||||
suppressReasoning: options?.suppressReasoning,
|
||||
})
|
||||
: [];
|
||||
case "content_block_delta":
|
||||
return isClaudeContentChunk(event.delta)
|
||||
? this.mapBlocksToTimeline([event.delta], {
|
||||
? await this.mapBlocksToTimeline([event.delta], {
|
||||
suppressAssistantText: options?.suppressAssistantText,
|
||||
suppressReasoning: options?.suppressReasoning,
|
||||
})
|
||||
@@ -6299,12 +6314,14 @@ interface ClaudeHistoryEntry {
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
function mapAssistantHistoryBlocksWithMessageId(
|
||||
async function mapAssistantHistoryBlocksWithMessageId(
|
||||
entry: ClaudeHistoryEntry,
|
||||
content: string | ClaudeContentChunk[],
|
||||
mapBlocks: (content: string | ClaudeContentChunk[]) => AgentTimelineItem[],
|
||||
): AgentTimelineItem[] {
|
||||
const items = mapBlocks(content);
|
||||
mapBlocks: (
|
||||
content: string | ClaudeContentChunk[],
|
||||
) => AgentTimelineItem[] | Promise<AgentTimelineItem[]>,
|
||||
): Promise<AgentTimelineItem[]> {
|
||||
const items = await mapBlocks(content);
|
||||
const assistantMessageId =
|
||||
typeof entry.uuid === "string" && entry.uuid.length > 0 ? entry.uuid : null;
|
||||
if (!assistantMessageId) {
|
||||
@@ -6372,10 +6389,12 @@ function isProviderImageMessage(item: AgentTimelineItem): boolean {
|
||||
return item.type === "assistant_message" && isProviderImageMarkdown(item.text);
|
||||
}
|
||||
|
||||
export function convertClaudeHistoryEntry(
|
||||
export async function convertClaudeHistoryEntry(
|
||||
entry: ClaudeHistoryEntry,
|
||||
mapBlocks: (content: string | ClaudeContentChunk[]) => AgentTimelineItem[],
|
||||
): AgentTimelineItem[] {
|
||||
mapBlocks: (
|
||||
content: string | ClaudeContentChunk[],
|
||||
) => AgentTimelineItem[] | Promise<AgentTimelineItem[]>,
|
||||
): Promise<AgentTimelineItem[]> {
|
||||
const preamble = convertClaudeHistoryEntryPreamble(entry);
|
||||
if ("shortCircuit" in preamble) {
|
||||
return preamble.shortCircuit;
|
||||
@@ -6414,7 +6433,7 @@ export function convertClaudeHistoryEntry(
|
||||
}
|
||||
|
||||
if (hasToolBlock && normalizedBlocks) {
|
||||
const mapped = mapBlocks(normalizedBlocks);
|
||||
const mapped = await mapBlocks(normalizedBlocks);
|
||||
if (entry.type === "user") {
|
||||
// tool_result handling (handleToolResult) emits image markdown as an assistant_message
|
||||
// alongside the tool_call. User-entry text blocks also map to assistant_message in this path
|
||||
@@ -6428,7 +6447,7 @@ export function convertClaudeHistoryEntry(
|
||||
}
|
||||
|
||||
if (entry.type === "assistant" && contentValue) {
|
||||
return mapAssistantHistoryBlocksWithMessageId(entry, contentValue, mapBlocks);
|
||||
return await mapAssistantHistoryBlocksWithMessageId(entry, contentValue, mapBlocks);
|
||||
}
|
||||
|
||||
return timeline;
|
||||
|
||||
@@ -1,3 +1,9 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import {
|
||||
materializeProviderImage,
|
||||
renderProviderImageOutputAsAssistantMarkdown,
|
||||
} from "../../provider-image-output.js";
|
||||
import type { SDKMessage } from "@anthropic-ai/claude-agent-sdk";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
@@ -660,3 +666,34 @@ describe("live and replay agree", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("awaits image materialization before publishing replay observations", async () => {
|
||||
const bytes = Buffer.from("async replay image fixture");
|
||||
const convertEntry = async () => {
|
||||
const item = await renderProviderImageOutputAsAssistantMarkdown(
|
||||
{ data: bytes.toString("base64"), mimeType: "image/png" },
|
||||
{ materialize: materializeProviderImage },
|
||||
);
|
||||
return item ? [item] : [];
|
||||
};
|
||||
const { observations } = await observeReplaySubagents({
|
||||
parent: parentWithTaskCall(),
|
||||
subagents: [
|
||||
{
|
||||
agentId: AGENT_ID,
|
||||
meta: { toolUseId: TOOL_USE_ID },
|
||||
entries: [{ type: "assistant", message: { content: [] } }],
|
||||
},
|
||||
],
|
||||
convertEntry,
|
||||
});
|
||||
const image = observations.find(
|
||||
(observation) =>
|
||||
observation.kind === "timeline" && observation.item.type === "assistant_message",
|
||||
);
|
||||
if (!image || image.kind !== "timeline" || image.item.type !== "assistant_message")
|
||||
throw new Error("Expected replay image");
|
||||
const source = image.item.text.match(/!\[Image\]\(([^)]+)\)/)?.[1];
|
||||
expect(source).toBeTruthy();
|
||||
expect(await readFile(fileURLToPath(source!))).toEqual(bytes);
|
||||
});
|
||||
|
||||
@@ -253,7 +253,7 @@ function observeSubtitle(
|
||||
async function observeSubagent(
|
||||
subagent: ClaudeReplaySubagentInput,
|
||||
parent: ClaudeReplayParentFacts,
|
||||
convertEntry: (entry: ClaudeReplayEntry) => AgentTimelineItem[],
|
||||
convertEntry: (entry: ClaudeReplayEntry) => AgentTimelineItem[] | Promise<AgentTimelineItem[]>,
|
||||
toolOwners: Map<string, string>,
|
||||
parentSubagentId?: string,
|
||||
): Promise<SubagentObservation[]> {
|
||||
@@ -280,7 +280,7 @@ async function observeSubagent(
|
||||
childTerminalStatus = entry.message?.stop_reason === "end_turn" ? "completed" : null;
|
||||
}
|
||||
recordReplayToolOwners(toolOwners, entry, link.id);
|
||||
for (const item of convertEntry(entry)) {
|
||||
for (const item of await convertEntry(entry)) {
|
||||
timeline.push({ kind: "timeline", id: link.id, item, ...(timestamp ? { timestamp } : {}) });
|
||||
}
|
||||
}
|
||||
@@ -323,7 +323,7 @@ function recordReplayToolOwners(
|
||||
export async function observeReplaySubagents(input: {
|
||||
subagents: readonly ClaudeReplaySubagentInput[];
|
||||
parent: ClaudeReplayParentFacts;
|
||||
convertEntry: (entry: ClaudeReplayEntry) => AgentTimelineItem[];
|
||||
convertEntry: (entry: ClaudeReplayEntry) => AgentTimelineItem[] | Promise<AgentTimelineItem[]>;
|
||||
}): Promise<{ observations: SubagentObservation[]; toolOwners: ReadonlyMap<string, string> }> {
|
||||
const observations: SubagentObservation[] = [];
|
||||
const toolOwners = new Map<string, string>();
|
||||
|
||||
+32
@@ -1,3 +1,9 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import {
|
||||
materializeProviderImage,
|
||||
renderProviderImageOutputAsAssistantMarkdown,
|
||||
} from "../../provider-image-output.js";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { ProviderSubagentStore } from "../../../provider-subagents/store.js";
|
||||
@@ -195,3 +201,29 @@ describe("Claude workflow replay", () => {
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("awaits image materialization before publishing replay observations", async () => {
|
||||
const bytes = Buffer.from("async replay image fixture");
|
||||
const convertEntry = async () => {
|
||||
const item = await renderProviderImageOutputAsAssistantMarkdown(
|
||||
{ data: bytes.toString("base64"), mimeType: "image/png" },
|
||||
{ materialize: materializeProviderImage },
|
||||
);
|
||||
return item ? [item] : [];
|
||||
};
|
||||
const observations = await observeReplayWorkflows({
|
||||
workflows: [{ runId: RUN_ID, status: "completed" }],
|
||||
parentEntries: parentEntries(),
|
||||
entriesByRunId: new Map([[RUN_ID, [{ type: "assistant", message: { content: [] } }]]]),
|
||||
convertEntry,
|
||||
});
|
||||
const image = observations.find(
|
||||
(observation) =>
|
||||
observation.kind === "timeline" && observation.item.type === "assistant_message",
|
||||
);
|
||||
if (!image || image.kind !== "timeline" || image.item.type !== "assistant_message")
|
||||
throw new Error("Expected replay image");
|
||||
const source = image.item.text.match(/!\[Image\]\(([^)]+)\)/)?.[1];
|
||||
expect(source).toBeTruthy();
|
||||
expect(await readFile(fileURLToPath(source!))).toEqual(bytes);
|
||||
});
|
||||
|
||||
+3
-3
@@ -53,7 +53,7 @@ export async function observeReplayWorkflows(input: {
|
||||
string,
|
||||
Iterable<ClaudeReplayEntry> | AsyncIterable<ClaudeReplayEntry>
|
||||
>;
|
||||
convertEntry?: (entry: ClaudeReplayEntry) => AgentTimelineItem[];
|
||||
convertEntry?: (entry: ClaudeReplayEntry) => AgentTimelineItem[] | Promise<AgentTimelineItem[]>;
|
||||
}): Promise<SubagentObservation[]> {
|
||||
const toolCallIdByRunId = await readWorkflowLinks(input.parentEntries);
|
||||
const observations: SubagentObservation[] = [];
|
||||
@@ -122,7 +122,7 @@ async function observeWorkflowTimeline(input: {
|
||||
id: string;
|
||||
result: unknown;
|
||||
entries: Iterable<ClaudeReplayEntry> | AsyncIterable<ClaudeReplayEntry>;
|
||||
convertEntry?: (entry: ClaudeReplayEntry) => AgentTimelineItem[];
|
||||
convertEntry?: (entry: ClaudeReplayEntry) => AgentTimelineItem[] | Promise<AgentTimelineItem[]>;
|
||||
finishedAt?: string;
|
||||
}): Promise<SubagentObservation[]> {
|
||||
const observations: SubagentObservation[] = [];
|
||||
@@ -130,7 +130,7 @@ async function observeWorkflowTimeline(input: {
|
||||
|
||||
for await (const entry of input.entries) {
|
||||
const timestamp = normalizeProviderReplayTimestamp(entry.timestamp);
|
||||
const items = input.convertEntry?.(entry) ?? [];
|
||||
const items = (await input.convertEntry?.(entry)) ?? [];
|
||||
for (const item of items) {
|
||||
if (item.type === "assistant_message") replayedAssistantText.add(item.text.trim());
|
||||
observations.push({
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1939,12 +1939,12 @@ function codexImageOutputFromResult(result: unknown): ProviderImageOutput | null
|
||||
};
|
||||
}
|
||||
|
||||
function mapCodexThreadImageItem(
|
||||
async function mapCodexThreadImageItem(
|
||||
normalizedType: string,
|
||||
normalizedItem: Record<string, unknown>,
|
||||
): AgentTimelineItem | null {
|
||||
): Promise<AgentTimelineItem | null> {
|
||||
if (normalizedType === "imageView") {
|
||||
return renderProviderImageOutputAsAssistantMarkdown({
|
||||
return await renderProviderImageOutputAsAssistantMarkdown({
|
||||
path: firstStringField(normalizedItem, ["path"]),
|
||||
});
|
||||
}
|
||||
@@ -1960,7 +1960,7 @@ function mapCodexThreadImageItem(
|
||||
|
||||
const savedPath = firstStringField(normalizedItem, ["savedPath", "saved_path"]);
|
||||
const result = codexImageOutputFromResult(normalizedItem.result);
|
||||
return renderProviderImageOutputAsAssistantMarkdown(
|
||||
return await renderProviderImageOutputAsAssistantMarkdown(
|
||||
{
|
||||
path: savedPath ?? result?.path ?? null,
|
||||
url: result?.url ?? null,
|
||||
@@ -1982,10 +1982,10 @@ function mapCodexAgentMessage(item: Record<string, unknown>): AgentTimelineItem
|
||||
};
|
||||
}
|
||||
|
||||
export function threadItemToTimeline(
|
||||
export async function threadItemToTimeline(
|
||||
item: unknown,
|
||||
options?: { includeUserMessage?: boolean; cwd?: string | null },
|
||||
): AgentTimelineItem | null {
|
||||
): Promise<AgentTimelineItem | null> {
|
||||
const itemRecord = toObjectRecord(item);
|
||||
if (!itemRecord) return null;
|
||||
const includeUserMessage = options?.includeUserMessage ?? true;
|
||||
@@ -1999,7 +1999,7 @@ export function threadItemToTimeline(
|
||||
: itemRecord;
|
||||
|
||||
if (normalizedType === "imageView" || normalizedType === "imageGeneration") {
|
||||
return mapCodexThreadImageItem(normalizedType, normalizedItem);
|
||||
return await mapCodexThreadImageItem(normalizedType, normalizedItem);
|
||||
}
|
||||
if (normalizedType && CODEX_TOOL_THREAD_ITEM_TYPES.has(normalizedType)) {
|
||||
return mapCodexToolCallFromThreadItem(normalizedItem, { cwd });
|
||||
@@ -2024,7 +2024,7 @@ export function threadItemToTimeline(
|
||||
}
|
||||
}
|
||||
|
||||
function mcpToolResultImagesToTimeline(item: unknown): AgentTimelineItem[] {
|
||||
async function mcpToolResultImagesToTimeline(item: unknown): Promise<AgentTimelineItem[]> {
|
||||
const itemRecord = toObjectRecord(item);
|
||||
if (!itemRecord) {
|
||||
return [];
|
||||
@@ -2037,24 +2037,25 @@ function mcpToolResultImagesToTimeline(item: unknown): AgentTimelineItem[] {
|
||||
}
|
||||
|
||||
const { images } = splitCodexMcpToolResultImages(itemRecord.result);
|
||||
return images
|
||||
.map((image) =>
|
||||
renderProviderImageOutputAsAssistantMarkdown(image, {
|
||||
materialize: materializeProviderImage,
|
||||
}),
|
||||
)
|
||||
.filter((timelineItem): timelineItem is AgentTimelineItem => timelineItem !== null);
|
||||
const items: AgentTimelineItem[] = [];
|
||||
for (const image of images) {
|
||||
const imageItem = await renderProviderImageOutputAsAssistantMarkdown(image, {
|
||||
materialize: materializeProviderImage,
|
||||
});
|
||||
if (imageItem) items.push(imageItem);
|
||||
}
|
||||
return items;
|
||||
}
|
||||
|
||||
function threadItemToTimelineEntries(
|
||||
async function threadItemToTimelineEntries(
|
||||
item: unknown,
|
||||
options?: { includeUserMessage?: boolean; cwd?: string | null },
|
||||
): AgentTimelineItem[] {
|
||||
const timelineItem = threadItemToTimeline(item, options);
|
||||
): Promise<AgentTimelineItem[]> {
|
||||
const timelineItem = await threadItemToTimeline(item, options);
|
||||
if (!timelineItem) {
|
||||
return [];
|
||||
}
|
||||
return [timelineItem, ...mcpToolResultImagesToTimeline(item)];
|
||||
return [timelineItem, ...(await mcpToolResultImagesToTimeline(item))];
|
||||
}
|
||||
|
||||
const CodexThreadReadResponseSchema = z
|
||||
@@ -2103,7 +2104,7 @@ async function loadCodexThreadHistoryTimeline(params: {
|
||||
historicalSubAgentActivity.agentThreadId,
|
||||
);
|
||||
if (existingIndex !== undefined) {
|
||||
const activityTimelineItem = threadItemToTimeline(item, { cwd: params.cwd });
|
||||
const activityTimelineItem = await threadItemToTimeline(item, { cwd: params.cwd });
|
||||
updateHistoricalSubAgentActivity(
|
||||
timeline,
|
||||
existingIndex,
|
||||
@@ -2116,7 +2117,7 @@ async function loadCodexThreadHistoryTimeline(params: {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
for (const timelineItem of threadItemToTimelineEntries(item, { cwd: params.cwd })) {
|
||||
for (const timelineItem of await threadItemToTimelineEntries(item, { cwd: params.cwd })) {
|
||||
const timestamp =
|
||||
readCodexHistoryTimestamp(item) ?? readCodexTurnHistoryTimestamp(turn, timelineItem);
|
||||
const settledTimelineItem =
|
||||
@@ -3290,10 +3291,12 @@ export async function codexAppServerTurnInputFromPrompt(
|
||||
}
|
||||
if (block.type === "image") {
|
||||
try {
|
||||
const filePath = materializeProviderImage({
|
||||
data: block.data,
|
||||
mimeType: block.mimeType,
|
||||
}).path;
|
||||
const filePath = (
|
||||
await materializeProviderImage({
|
||||
data: block.data,
|
||||
mimeType: block.mimeType,
|
||||
})
|
||||
).path;
|
||||
output.push({ type: "localImage", path: filePath });
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
@@ -4370,6 +4373,7 @@ export class CodexAppServerAgentSession implements AgentSession {
|
||||
this.dismissPendingPlanApprovals("Dismissed by a new prompt");
|
||||
|
||||
try {
|
||||
await this.pendingNotifications;
|
||||
await this.connect();
|
||||
if (!this.client) {
|
||||
throw new Error("Codex client not initialized");
|
||||
@@ -5437,7 +5441,21 @@ export class CodexAppServerAgentSession implements AgentSession {
|
||||
return `codex-turn-${this.nextTurnOrdinal++}`;
|
||||
}
|
||||
|
||||
private handleNotification(method: string, params: unknown): void {
|
||||
private pendingNotifications: Promise<void> = Promise.resolve();
|
||||
|
||||
private handleNotification(method: string, params: unknown): Promise<void> {
|
||||
// An image must exist before its event, and completion must follow that event.
|
||||
// Transport callbacks cannot await: the session owns their ordered drain.
|
||||
const next = this.pendingNotifications.then(() => {
|
||||
return this.closed ? undefined : this.processNotification(method, params);
|
||||
});
|
||||
this.pendingNotifications = next.catch((error: unknown) => {
|
||||
this.handleUnexpectedTermination(error instanceof Error ? error : new Error(String(error)));
|
||||
});
|
||||
return this.pendingNotifications;
|
||||
}
|
||||
|
||||
private async processNotification(method: string, params: unknown): Promise<void> {
|
||||
const notificationParams = toObjectRecord(params);
|
||||
if (method === "serverRequest/resolved" && typeof notificationParams?.requestId === "number") {
|
||||
const requestId = this.mcpElicitationPermissionIds.get(notificationParams.requestId);
|
||||
@@ -5467,13 +5485,16 @@ export class CodexAppServerAgentSession implements AgentSession {
|
||||
return;
|
||||
}
|
||||
if (route.kind === "sub_agent") {
|
||||
this.dispatchSubAgentNotification(parsed, route.callId);
|
||||
await this.dispatchSubAgentNotification(parsed, route.callId);
|
||||
return;
|
||||
}
|
||||
this.dispatchParsedNotification(parsed);
|
||||
await this.dispatchParsedNotification(parsed);
|
||||
}
|
||||
|
||||
private dispatchSubAgentNotification(parsed: ParsedCodexNotification, callId: string): void {
|
||||
private async dispatchSubAgentNotification(
|
||||
parsed: ParsedCodexNotification,
|
||||
callId: string,
|
||||
): Promise<void> {
|
||||
switch (parsed.kind) {
|
||||
case "thread_started":
|
||||
this.emitSubAgentActivityUpdate(callId, "running", { reopen: true });
|
||||
@@ -5484,7 +5505,7 @@ export class CodexAppServerAgentSession implements AgentSession {
|
||||
case "reasoning_delta":
|
||||
case "item_started":
|
||||
case "item_completed":
|
||||
this.dispatchParsedNotification(parsed);
|
||||
await this.dispatchParsedNotification(parsed);
|
||||
return;
|
||||
case "exec_command_output_delta":
|
||||
case "file_change_output_delta":
|
||||
@@ -5510,7 +5531,7 @@ export class CodexAppServerAgentSession implements AgentSession {
|
||||
}
|
||||
}
|
||||
|
||||
private dispatchParsedNotification(parsed: ParsedCodexNotification): void {
|
||||
private async dispatchParsedNotification(parsed: ParsedCodexNotification): Promise<void> {
|
||||
if (isCodexDeltaNotification(parsed)) {
|
||||
this.handleCodexDeltaNotification(parsed);
|
||||
return;
|
||||
@@ -5555,10 +5576,10 @@ export class CodexAppServerAgentSession implements AgentSession {
|
||||
this.handlePatchApplyCompletedNotification(parsed);
|
||||
return;
|
||||
case "item_completed":
|
||||
this.handleItemCompletedNotification(parsed);
|
||||
await this.handleItemCompletedNotification(parsed);
|
||||
return;
|
||||
case "item_started":
|
||||
this.handleItemStartedNotification(parsed);
|
||||
await this.handleItemStartedNotification(parsed);
|
||||
return;
|
||||
case "invalid_payload":
|
||||
this.warnInvalidNotificationPayload(parsed.method, parsed.params);
|
||||
@@ -5600,7 +5621,7 @@ export class CodexAppServerAgentSession implements AgentSession {
|
||||
pending.push(parsed);
|
||||
}
|
||||
|
||||
private replayPendingSubAgentNotifications(threadIds: readonly string[]): void {
|
||||
private async replayPendingSubAgentNotifications(threadIds: readonly string[]): Promise<void> {
|
||||
for (const threadId of threadIds) {
|
||||
const pending = this.pendingSubAgentNotificationsByThreadId.get(threadId);
|
||||
if (!pending) {
|
||||
@@ -5612,7 +5633,7 @@ export class CodexAppServerAgentSession implements AgentSession {
|
||||
continue;
|
||||
}
|
||||
for (const parsed of pending) {
|
||||
this.dispatchSubAgentNotification(parsed, callId);
|
||||
await this.dispatchSubAgentNotification(parsed, callId);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6578,9 +6599,13 @@ export class CodexAppServerAgentSession implements AgentSession {
|
||||
}
|
||||
}
|
||||
|
||||
private handleItemCompletedNotification(
|
||||
private ownsNotificationTurn(turnId: string | null): boolean {
|
||||
return !this.closed && this.activeForegroundTurnId === turnId;
|
||||
}
|
||||
|
||||
private async handleItemCompletedNotification(
|
||||
parsed: Extract<ParsedCodexNotification, { kind: "item_completed" }>,
|
||||
): void {
|
||||
): Promise<void> {
|
||||
// Codex emits mirrored lifecycle notifications via both `codex/event/item_*`
|
||||
// and canonical `item/*`. Render ordinary items only from the canonical
|
||||
// channel, but accept a legacy-only child announcement so it can establish
|
||||
@@ -6590,20 +6615,23 @@ export class CodexAppServerAgentSession implements AgentSession {
|
||||
}
|
||||
this.receiveAsyncQuestion(parsed.threadId, parsed.item);
|
||||
if (this.isUserMessageItem(parsed.item)) {
|
||||
this.handleUserMessageItem(parsed);
|
||||
await this.handleUserMessageItem(parsed);
|
||||
return;
|
||||
}
|
||||
const childSubAgentCallId = this.getSubAgentCallIdForThread(parsed.threadId);
|
||||
if (this.handleCompletedSpecialItem(parsed, childSubAgentCallId)) {
|
||||
return;
|
||||
}
|
||||
const timelineItem = threadItemToTimeline(parsed.item, {
|
||||
const foregroundTurnId = this.activeForegroundTurnId;
|
||||
const timelineItem = await threadItemToTimeline(parsed.item, {
|
||||
includeUserMessage: false,
|
||||
cwd: this.config.cwd ?? null,
|
||||
});
|
||||
if (!timelineItem) {
|
||||
return;
|
||||
}
|
||||
const imageItems = await mcpToolResultImagesToTimeline(parsed.item);
|
||||
if (!this.ownsNotificationTurn(foregroundTurnId)) return;
|
||||
const registeredChildThreadIds =
|
||||
timelineItem.type === "tool_call"
|
||||
? this.registerSubAgentToolCall({
|
||||
@@ -6613,12 +6641,11 @@ export class CodexAppServerAgentSession implements AgentSession {
|
||||
parentSubagentId: resolveCodexParentSubagentId(childSubAgentCallId, parsed.threadId),
|
||||
})
|
||||
: [];
|
||||
const imageItems = mcpToolResultImagesToTimeline(parsed.item);
|
||||
if (childSubAgentCallId) {
|
||||
this.emitCompletedProviderSubagentItem(parsed, timelineItem);
|
||||
this.handleSubAgentChildItemCompleted(childSubAgentCallId, parsed.item.id, timelineItem);
|
||||
this.emitProviderSubagentTimelineItems(parsed.threadId, imageItems);
|
||||
this.replayPendingSubAgentNotifications(registeredChildThreadIds);
|
||||
await this.replayPendingSubAgentNotifications(registeredChildThreadIds);
|
||||
return;
|
||||
}
|
||||
const normalizedItemType = normalizeCodexThreadItemType(
|
||||
@@ -6626,7 +6653,7 @@ export class CodexAppServerAgentSession implements AgentSession {
|
||||
);
|
||||
const itemId = parsed.item.id;
|
||||
if (this.shouldSkipCompletedThreadItem(timelineItem, normalizedItemType, itemId)) {
|
||||
this.replayPendingSubAgentNotifications(registeredChildThreadIds);
|
||||
await this.replayPendingSubAgentNotifications(registeredChildThreadIds);
|
||||
return;
|
||||
}
|
||||
if (this.consumeStreamedTextCompletion(timelineItem, itemId)) {
|
||||
@@ -6637,7 +6664,7 @@ export class CodexAppServerAgentSession implements AgentSession {
|
||||
this.emittedItemCompletedIds.add(itemId);
|
||||
this.emittedItemStartedIds.delete(itemId);
|
||||
}
|
||||
this.replayPendingSubAgentNotifications(registeredChildThreadIds);
|
||||
await this.replayPendingSubAgentNotifications(registeredChildThreadIds);
|
||||
return;
|
||||
}
|
||||
this.applyBufferedDeltaTextToTimelineItem(timelineItem, itemId);
|
||||
@@ -6666,7 +6693,7 @@ export class CodexAppServerAgentSession implements AgentSession {
|
||||
this.pendingCommandOutputDeltas.delete(itemId);
|
||||
this.pendingFileChangeOutputDeltas.delete(itemId);
|
||||
}
|
||||
this.replayPendingSubAgentNotifications(registeredChildThreadIds);
|
||||
await this.replayPendingSubAgentNotifications(registeredChildThreadIds);
|
||||
}
|
||||
|
||||
private consumeStreamedTextCompletion(
|
||||
@@ -6738,14 +6765,14 @@ export class CodexAppServerAgentSession implements AgentSession {
|
||||
}
|
||||
}
|
||||
|
||||
private handleItemStartedNotification(
|
||||
private async handleItemStartedNotification(
|
||||
parsed: Extract<ParsedCodexNotification, { kind: "item_started" }>,
|
||||
): void {
|
||||
): Promise<void> {
|
||||
if (shouldIgnoreMirroredLifecycleItem(parsed.source, parsed.item)) {
|
||||
return;
|
||||
}
|
||||
if (this.isUserMessageItem(parsed.item)) {
|
||||
this.handleUserMessageItem(parsed);
|
||||
await this.handleUserMessageItem(parsed);
|
||||
return;
|
||||
}
|
||||
const childSubAgentCallId = this.getSubAgentCallIdForThread(parsed.threadId);
|
||||
@@ -6767,7 +6794,7 @@ export class CodexAppServerAgentSession implements AgentSession {
|
||||
if (this.handleRegisteredSubAgentActivity(parsed.item)) {
|
||||
return;
|
||||
}
|
||||
const timelineItem = threadItemToTimeline(parsed.item, {
|
||||
const timelineItem = await threadItemToTimeline(parsed.item, {
|
||||
includeUserMessage: false,
|
||||
cwd: this.config.cwd ?? null,
|
||||
});
|
||||
@@ -6786,7 +6813,7 @@ export class CodexAppServerAgentSession implements AgentSession {
|
||||
this.upsertSubAgentChildItem(childSubAgentCallId, parsed.item.id, timelineItem);
|
||||
}
|
||||
this.emitSubAgentActivityUpdate(childSubAgentCallId, "running");
|
||||
this.replayPendingSubAgentNotifications(registeredChildThreadIds);
|
||||
await this.replayPendingSubAgentNotifications(registeredChildThreadIds);
|
||||
return;
|
||||
}
|
||||
const normalizedItemType = normalizeCodexThreadItemType(
|
||||
@@ -6810,14 +6837,14 @@ export class CodexAppServerAgentSession implements AgentSession {
|
||||
this.pendingCommandOutputDeltas.delete(itemId);
|
||||
this.pendingFileChangeOutputDeltas.delete(itemId);
|
||||
}
|
||||
this.replayPendingSubAgentNotifications(registeredChildThreadIds);
|
||||
await this.replayPendingSubAgentNotifications(registeredChildThreadIds);
|
||||
}
|
||||
|
||||
private handleUserMessageItem(
|
||||
private async handleUserMessageItem(
|
||||
parsed: Extract<ParsedCodexNotification, { kind: "item_started" | "item_completed" }>,
|
||||
): void {
|
||||
): Promise<void> {
|
||||
const itemId = parsed.item.id;
|
||||
const timelineItem = threadItemToTimeline(parsed.item, {
|
||||
const timelineItem = await threadItemToTimeline(parsed.item, {
|
||||
includeUserMessage: true,
|
||||
cwd: this.config.cwd ?? null,
|
||||
});
|
||||
|
||||
@@ -1627,3 +1627,40 @@ test.each([
|
||||
expect(await session.canStopRuntime!()).toBe(expected);
|
||||
},
|
||||
);
|
||||
|
||||
test.each([false, true])(
|
||||
"image prompt keeps OMP's vision capability contract (vision: %s)",
|
||||
async (vision) => {
|
||||
const runtime = new FakeOmp();
|
||||
runtime.queueSessionSetup((session) => {
|
||||
session.state = {
|
||||
...session.state,
|
||||
model: {
|
||||
provider: "test",
|
||||
id: "image-model",
|
||||
input: vision ? ["text", "image"] : ["text"],
|
||||
},
|
||||
};
|
||||
});
|
||||
const client = new OmpAgentClient({ logger: createTestLogger(), runtime });
|
||||
const session = await client.createSession({ provider: "omp", cwd: process.cwd() });
|
||||
onTestFinished(() => session.close());
|
||||
const bytes = Buffer.from("OMP attachment fixture");
|
||||
const first = session.startTurn([
|
||||
{ type: "text", text: "Describe this image." },
|
||||
{ type: "image", data: bytes.toString("base64"), mimeType: "image/png" },
|
||||
]);
|
||||
await expect(session.startTurn("overlapping prompt")).rejects.toThrow("already active");
|
||||
await first;
|
||||
const [prompt] = runtime.latestSession().prompts;
|
||||
expect(prompt.imageCount).toBe(vision ? 1 : 0);
|
||||
if (vision) expect(prompt.message).toBe("Describe this image.");
|
||||
else {
|
||||
const imagePath = prompt.message.match(/\[Image available at: (.+)\]/)?.[1];
|
||||
expect(imagePath).toBeTruthy();
|
||||
const { readFile, rm } = await import("node:fs/promises");
|
||||
expect(await readFile(imagePath!)).toEqual(bytes);
|
||||
await rm(imagePath!);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
@@ -317,9 +317,9 @@ function ompModelSupportsImageInput(model: OmpModel | null | undefined): boolean
|
||||
return model?.input?.includes("image") === true;
|
||||
}
|
||||
|
||||
function renderTextOnlyImageHint(image: { data: string; mimeType: string }): string {
|
||||
async function renderTextOnlyImageHint(image: { data: string; mimeType: string }): Promise<string> {
|
||||
try {
|
||||
const materialized = materializeProviderImage({
|
||||
const materialized = await materializeProviderImage({
|
||||
data: image.data,
|
||||
mimeType: image.mimeType,
|
||||
});
|
||||
@@ -329,10 +329,10 @@ function renderTextOnlyImageHint(image: { data: string; mimeType: string }): str
|
||||
}
|
||||
}
|
||||
|
||||
function convertPromptInput(
|
||||
async function convertPromptInput(
|
||||
prompt: AgentPromptInput,
|
||||
options: { model: OmpModel | null | undefined },
|
||||
): OmpPromptPayload {
|
||||
): Promise<OmpPromptPayload> {
|
||||
if (typeof prompt === "string") {
|
||||
return { text: prompt };
|
||||
}
|
||||
@@ -355,7 +355,7 @@ function convertPromptInput(
|
||||
mimeType: block.mimeType,
|
||||
});
|
||||
} else {
|
||||
textParts.push(renderTextOnlyImageHint(block));
|
||||
textParts.push(await renderTextOnlyImageHint(block));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
@@ -868,11 +868,9 @@ export class OmpAgentSession implements AgentSession {
|
||||
throw new Error("An OMP turn is already active");
|
||||
}
|
||||
|
||||
const payload = convertPromptInput(prompt, { model: this.state.model });
|
||||
const turnId = randomUUID();
|
||||
this.live = true;
|
||||
this.activeTurnId = turnId;
|
||||
this.rememberClientMessage(options?.clientMessageId ?? null, payload.text);
|
||||
this.activeAssistantMessageId = null;
|
||||
this.activeTurnTerminalAssistantMessage = null;
|
||||
this.activeTurnStarted = false;
|
||||
@@ -880,6 +878,9 @@ export class OmpAgentSession implements AgentSession {
|
||||
this.activeTurnHasUserMessage = false;
|
||||
this.activePromptRequestId = null;
|
||||
this.clearNoTurnBuffers();
|
||||
const payload = await convertPromptInput(prompt, { model: this.state.model });
|
||||
if (this.closed || this.activeTurnId !== turnId) return { turnId };
|
||||
this.rememberClientMessage(options?.clientMessageId ?? null, payload.text);
|
||||
this.activeNoTurnPromptText = payload.text;
|
||||
this.usagePoller.startTurn();
|
||||
if (announceTurn) {
|
||||
@@ -946,7 +947,9 @@ export class OmpAgentSession implements AgentSession {
|
||||
): Promise<SteerResult> {
|
||||
if (this.closed || this.activeTurnId !== options.expectedTurnId)
|
||||
return { status: "unavailable" };
|
||||
const payload = convertPromptInput(prompt, { model: this.state.model });
|
||||
const payload = await convertPromptInput(prompt, { model: this.state.model });
|
||||
if (this.closed || this.activeTurnId !== options.expectedTurnId)
|
||||
return { status: "unavailable" };
|
||||
if (this.parseSlashCommandInput(payload.text)) return { status: "unavailable" };
|
||||
const submission = this.rememberClientMessage(options.clientMessageId ?? null, payload.text);
|
||||
try {
|
||||
|
||||
+12
-10
@@ -21,16 +21,18 @@ for (let index = 0; index < chunks; index++) {
|
||||
const event =
|
||||
mode === "delta"
|
||||
? timeline.delta({ ...part, delta })
|
||||
: timeline.messages([
|
||||
{
|
||||
id: "answer",
|
||||
type: "assistant",
|
||||
agent: "build",
|
||||
model: { providerID: "test", id: "model" },
|
||||
time: { created: 2 },
|
||||
content: [{ type: "reasoning", text }],
|
||||
},
|
||||
])[0];
|
||||
: (
|
||||
await timeline.messages([
|
||||
{
|
||||
id: "answer",
|
||||
type: "assistant",
|
||||
agent: "build",
|
||||
model: { providerID: "test", id: "model" },
|
||||
time: { created: 2 },
|
||||
content: [{ type: "reasoning", text }],
|
||||
},
|
||||
])
|
||||
)[0];
|
||||
if (event?.type !== "timeline") throw new Error("Expected a timeline event");
|
||||
projection.append({
|
||||
seq: index + 1,
|
||||
|
||||
@@ -92,7 +92,7 @@ export class SessionChildren {
|
||||
this.childStates.set(info.id, signature);
|
||||
this.options.emit({ type: "provider_subagent", provider: "opencode", event: presentation });
|
||||
}
|
||||
for (const event of timeline.messages(await messages(this.options.client(), info.id))) {
|
||||
for (const event of await timeline.messages(await messages(this.options.client(), info.id))) {
|
||||
if (event.type === "timeline")
|
||||
this.options.emit({
|
||||
type: "provider_subagent",
|
||||
|
||||
@@ -131,7 +131,7 @@ export class OpenCodeV2Session implements AgentSession {
|
||||
await this.configureConnection();
|
||||
const location = { directory: this.config.cwd };
|
||||
this.modes = modesFromV2((await this.client.agent.list({ location })).data);
|
||||
this.timeline.messages(await messages(this.client, this.id));
|
||||
await this.timeline.messages(await messages(this.client, this.id));
|
||||
await this.startStream();
|
||||
await this.children.reconcile(this.id);
|
||||
}
|
||||
@@ -261,7 +261,7 @@ export class OpenCodeV2Session implements AgentSession {
|
||||
async *streamHistory() {
|
||||
await this.reconnectIfExited();
|
||||
const history = new V2Timeline(false);
|
||||
yield* history.messages(await messages(this.client, this.id));
|
||||
yield* await history.messages(await messages(this.client, this.id));
|
||||
}
|
||||
async getRuntimeInfo() {
|
||||
await this.reconnectIfExited();
|
||||
@@ -417,7 +417,7 @@ export class OpenCodeV2Session implements AgentSession {
|
||||
]);
|
||||
this.info = info;
|
||||
this.history = history;
|
||||
for (const event of this.timeline.messages(history)) this.emitTimeline(event);
|
||||
for (const event of await this.timeline.messages(history)) this.emitTimeline(event);
|
||||
await this.permissions.reconcile(this.id);
|
||||
}
|
||||
private async reconcile() {
|
||||
|
||||
@@ -80,7 +80,7 @@ describe("OpenCode v2 token streaming", () => {
|
||||
}
|
||||
});
|
||||
|
||||
test("deduplicates snapshots before and after deltas using per-type ordinals", () => {
|
||||
test("deduplicates snapshots before and after deltas using per-type ordinals", async () => {
|
||||
const timeline = new V2Timeline();
|
||||
const text = { assistantMessageID: "answer", type: "text", ordinal: 0 } as const;
|
||||
const reasoning = { ...text, type: "reasoning" } as const;
|
||||
@@ -99,24 +99,24 @@ describe("OpenCode v2 token streaming", () => {
|
||||
},
|
||||
{ type: "text", text: "Hello" },
|
||||
]);
|
||||
expect(timeline.messages([snapshot])).toMatchObject([{ item: { text: "lo" } }]);
|
||||
expect(await timeline.messages([snapshot])).toMatchObject([{ item: { text: "lo" } }]);
|
||||
expect(timeline.delta({ ...text, delta: "lo" })).toBeNull();
|
||||
expect(timeline.messages([snapshot])).toEqual([]);
|
||||
expect(await timeline.messages([snapshot])).toEqual([]);
|
||||
expect(timeline.delta({ ...text, delta: "!" })).toMatchObject({ item: { text: "!" } });
|
||||
expect(timeline.messages([snapshot])).toEqual([]);
|
||||
expect(await timeline.messages([snapshot])).toEqual([]);
|
||||
});
|
||||
|
||||
test("recovers missed fragments from snapshots after reconnect", () => {
|
||||
test("recovers missed fragments from snapshots after reconnect", async () => {
|
||||
const timeline = new V2Timeline();
|
||||
const part = { assistantMessageID: "answer", type: "text", ordinal: 0 } as const;
|
||||
timeline.startPart(part);
|
||||
expect(timeline.delta({ ...part, delta: "Hel" })).toMatchObject({ item: { text: "Hel" } });
|
||||
timeline.resetStreams();
|
||||
expect(timeline.delta({ ...part, delta: "!" })).toBeNull();
|
||||
expect(timeline.messages([assistant([{ type: "text", text: "Hello!" }])])).toMatchObject([
|
||||
expect(await timeline.messages([assistant([{ type: "text", text: "Hello!" }])])).toMatchObject([
|
||||
{ item: { text: "lo!" } },
|
||||
]);
|
||||
expect(timeline.messages([assistant([{ type: "text", text: "Hello!" }])])).toEqual([]);
|
||||
expect(await timeline.messages([assistant([{ type: "text", text: "Hello!" }])])).toEqual([]);
|
||||
});
|
||||
|
||||
test("withholds streamed prose during a structured-output turn", async () => {
|
||||
|
||||
@@ -15,7 +15,7 @@ function assistant(content: SessionMessageAssistant["content"]): SessionMessageA
|
||||
};
|
||||
}
|
||||
|
||||
test("a completed edit carries the replaced and replacement text", () => {
|
||||
test("a completed edit carries the replaced and replacement text", async () => {
|
||||
// Captured from OpenCode 2.0.18, whose edit tool takes `path` instead of `filePath`.
|
||||
const edit = assistant([
|
||||
{
|
||||
@@ -45,7 +45,7 @@ test("a completed edit carries the replaced and replacement text", () => {
|
||||
},
|
||||
]);
|
||||
|
||||
expect(new V2Timeline().messages([edit])).toMatchObject([
|
||||
expect(await new V2Timeline().messages([edit])).toMatchObject([
|
||||
{
|
||||
item: {
|
||||
type: "tool_call",
|
||||
@@ -57,7 +57,7 @@ test("a completed edit carries the replaced and replacement text", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
test("a tool's image output is shown as an image, not as base64 in the tool output", () => {
|
||||
test("a tool's image output is shown as an image, not as base64 in the tool output", async () => {
|
||||
const png =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==";
|
||||
const screenshot = assistant([
|
||||
@@ -80,7 +80,7 @@ test("a tool's image output is shown as an image, not as base64 in the tool outp
|
||||
]);
|
||||
const timeline = new V2Timeline();
|
||||
|
||||
const events = timeline.messages([screenshot]);
|
||||
const events = await timeline.messages([screenshot]);
|
||||
|
||||
expect(JSON.stringify(events)).not.toContain(png);
|
||||
expect(events.map((event) => event.type === "timeline" && event.item)).toMatchObject([
|
||||
@@ -95,7 +95,7 @@ test("a tool's image output is shown as an image, not as base64 in the tool outp
|
||||
text: expect.stringMatching(/^\n\n!\[Image\]\(file:\/\/.*paseo-attachments.*\.png\)\n\n$/),
|
||||
},
|
||||
]);
|
||||
expect(timeline.messages([screenshot])).toEqual([]);
|
||||
expect(await timeline.messages([screenshot])).toEqual([]);
|
||||
});
|
||||
|
||||
test("a surrogate pair split across deltas streams back intact", () => {
|
||||
|
||||
@@ -55,7 +55,15 @@ export class V2Timeline {
|
||||
return { type: "timeline", provider: "opencode", item, timestamp: new Date().toISOString() };
|
||||
}
|
||||
|
||||
messages(messages: SessionMessageInfo[]): AgentStreamEvent[] {
|
||||
private pendingMessages: Promise<unknown> = Promise.resolve();
|
||||
|
||||
messages(messages: SessionMessageInfo[]): Promise<AgentStreamEvent[]> {
|
||||
const result = this.pendingMessages.then(() => this.projectMessages(messages));
|
||||
this.pendingMessages = result.catch(() => {});
|
||||
return result;
|
||||
}
|
||||
|
||||
private async projectMessages(messages: SessionMessageInfo[]): Promise<AgentStreamEvent[]> {
|
||||
const events: AgentStreamEvent[] = [];
|
||||
const state: TimelineState = { structured: false, accepted: false };
|
||||
for (const message of messages) {
|
||||
@@ -63,7 +71,7 @@ export class V2Timeline {
|
||||
const push = (item: AgentTimelineItem) =>
|
||||
events.push({ type: "timeline", provider: "opencode", item, timestamp });
|
||||
if (message.type === "user") this.userMessage(message, state, push);
|
||||
else if (message.type === "assistant") this.assistantMessage(message, state, push);
|
||||
else if (message.type === "assistant") await this.assistantMessage(message, state, push);
|
||||
else if (message.type === "compaction") this.compactionMessage(message, push);
|
||||
}
|
||||
return events;
|
||||
@@ -89,7 +97,7 @@ export class V2Timeline {
|
||||
});
|
||||
}
|
||||
|
||||
private assistantMessage(
|
||||
private async assistantMessage(
|
||||
message: Extract<SessionMessageInfo, { type: "assistant" }>,
|
||||
state: TimelineState,
|
||||
push: (item: AgentTimelineItem) => void,
|
||||
@@ -99,16 +107,16 @@ export class V2Timeline {
|
||||
// paths address the same cursor entry.
|
||||
let textOrdinal = 0;
|
||||
let reasoningOrdinal = 0;
|
||||
message.content.forEach((part, partIndex) => {
|
||||
for (const [partIndex, part] of message.content.entries()) {
|
||||
if (part.type === "tool" && part.name === STRUCTURED_OUTPUT_TOOL) {
|
||||
this.structuredOutputPart(message.id, partIndex, part, state, push);
|
||||
} else if (part.type === "text" || part.type === "reasoning") {
|
||||
const ordinal = part.type === "text" ? textOrdinal++ : reasoningOrdinal++;
|
||||
this.textPart(message.id, part, ordinal, state, push);
|
||||
} else {
|
||||
this.toolPart(message.id, partIndex, part, push);
|
||||
await this.toolPart(message.id, partIndex, part, push);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private structuredOutputPart(
|
||||
@@ -156,7 +164,7 @@ export class V2Timeline {
|
||||
else push({ type: "reasoning", text: suffix });
|
||||
}
|
||||
|
||||
private toolPart(
|
||||
private async toolPart(
|
||||
messageID: string,
|
||||
partIndex: number,
|
||||
part: Extract<
|
||||
@@ -174,16 +182,16 @@ export class V2Timeline {
|
||||
if (item) push(item);
|
||||
// Images follow their tool call as markdown, as Claude and Codex emit them,
|
||||
// so base64 never reaches the tool output.
|
||||
images.forEach((image, index) => {
|
||||
for (const [index, image] of images.entries()) {
|
||||
const imageKey = `${key}:image:${index}`;
|
||||
if (this.content.has(imageKey)) return;
|
||||
if (this.content.has(imageKey)) continue;
|
||||
this.content.set(imageKey, "");
|
||||
const imageItem = renderProviderImageOutputAsAssistantMarkdown(
|
||||
const imageItem = await renderProviderImageOutputAsAssistantMarkdown(
|
||||
{ url: image.uri, mimeType: image.mime },
|
||||
{ materialize: materializeProviderImage },
|
||||
);
|
||||
if (imageItem) push(imageItem);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private compactionMessage(
|
||||
|
||||
@@ -3583,3 +3583,14 @@ test("pi runtime safety protects queued replies and runtime-owned activity", asy
|
||||
native.getStateError = new Error("Unable to prove idle");
|
||||
await expect(session.canStopRuntime!()).rejects.toThrow("Unable to prove idle");
|
||||
});
|
||||
|
||||
test("reserves the Pi turn while an image prompt is materialized", async () => {
|
||||
const { pi, session } = await createSession();
|
||||
onTestFinished(() => session.close());
|
||||
const first = session.startTurn([
|
||||
{ type: "image", data: ONE_BY_ONE_PNG_BASE64, mimeType: "image/png" },
|
||||
]);
|
||||
await expect(session.startTurn("overlapping prompt")).rejects.toThrow("already active");
|
||||
await first;
|
||||
expect(pi.latestSession().prompts).toHaveLength(1);
|
||||
});
|
||||
|
||||
@@ -367,9 +367,9 @@ function piModelSupportsImageInput(model: PiModel | null | undefined): boolean {
|
||||
return model?.input?.includes("image") === true;
|
||||
}
|
||||
|
||||
function renderTextOnlyImageHint(image: { data: string; mimeType: string }): string {
|
||||
async function renderTextOnlyImageHint(image: { data: string; mimeType: string }): Promise<string> {
|
||||
try {
|
||||
const materialized = materializeProviderImage({
|
||||
const materialized = await materializeProviderImage({
|
||||
data: image.data,
|
||||
mimeType: image.mimeType,
|
||||
});
|
||||
@@ -379,10 +379,10 @@ function renderTextOnlyImageHint(image: { data: string; mimeType: string }): str
|
||||
}
|
||||
}
|
||||
|
||||
function convertPromptInput(
|
||||
async function convertPromptInput(
|
||||
prompt: AgentPromptInput,
|
||||
options: { model: PiModel | null | undefined },
|
||||
): PiPromptPayload {
|
||||
): Promise<PiPromptPayload> {
|
||||
if (typeof prompt === "string") {
|
||||
return { text: prompt };
|
||||
}
|
||||
@@ -405,7 +405,7 @@ function convertPromptInput(
|
||||
mimeType: block.mimeType,
|
||||
});
|
||||
} else {
|
||||
textParts.push(renderTextOnlyImageHint(block));
|
||||
textParts.push(await renderTextOnlyImageHint(block));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
@@ -1280,7 +1280,6 @@ export class PiRpcAgentSession implements AgentSession {
|
||||
throw new Error("A Pi turn is already active");
|
||||
}
|
||||
|
||||
const payload = convertPromptInput(prompt, { model: this.state.model });
|
||||
const turnId = randomUUID();
|
||||
this.activeTurnId = turnId;
|
||||
this.usagePoller.startTurn();
|
||||
@@ -1292,6 +1291,8 @@ export class PiRpcAgentSession implements AgentSession {
|
||||
this.activePromptRequestId = null;
|
||||
this.pendingSteerSubmissions.length = 0;
|
||||
this.clearNoTurnBuffers();
|
||||
const payload = await convertPromptInput(prompt, { model: this.state.model });
|
||||
if (this.closed || this.activeTurnId !== turnId) return { turnId };
|
||||
this.activeNoTurnPromptText = payload.text;
|
||||
const shouldProbeForNoTurnPrompt = this.parseSlashCommandInput(payload.text) !== null;
|
||||
|
||||
@@ -1363,7 +1364,9 @@ export class PiRpcAgentSession implements AgentSession {
|
||||
if (this.closed || this.activeTurnId !== options.expectedTurnId) {
|
||||
return { status: "unavailable" };
|
||||
}
|
||||
const payload = convertPromptInput(prompt, { model: this.state.model });
|
||||
const payload = await convertPromptInput(prompt, { model: this.state.model });
|
||||
if (this.closed || this.activeTurnId !== options.expectedTurnId)
|
||||
return { status: "unavailable" };
|
||||
// Pi rejects steer RPCs that are extension commands, so slash inputs keep the
|
||||
// interrupt-and-replace fallback where they can run directly.
|
||||
if (this.parseSlashCommandInput(payload.text)) {
|
||||
|
||||
@@ -5,8 +5,8 @@ import { describe, expect, test } from "vitest";
|
||||
import { materializeProviderImage } from "./provider-image-output.js";
|
||||
|
||||
describe.skipIf(process.platform === "win32")("materializeProviderImage", () => {
|
||||
test("writes image attachments under a private temp directory", () => {
|
||||
const materialized = materializeProviderImage({
|
||||
test("writes image attachments under a private temp directory", async () => {
|
||||
const materialized = await materializeProviderImage({
|
||||
data: "YWJjMTIz",
|
||||
mimeType: "image/png",
|
||||
});
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
import { existsSync, rmSync } from "node:fs";
|
||||
import { existsSync, rmSync, statSync, readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import MarkdownIt from "markdown-it";
|
||||
import { projectTimelineRows } from "../timeline-projection.js";
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
import {
|
||||
MAX_PROVIDER_IMAGE_BYTES,
|
||||
isProviderImageMarkdown,
|
||||
materializeProviderImage,
|
||||
renderProviderImageOutputAsAssistantMarkdown,
|
||||
@@ -12,8 +13,8 @@ import {
|
||||
|
||||
const HASH = "a".repeat(64);
|
||||
|
||||
function renderImageMarkdown(imagePath: string): string {
|
||||
const item = renderProviderImageOutputAsAssistantMarkdown({ path: imagePath });
|
||||
async function renderImageMarkdown(imagePath: string): Promise<string> {
|
||||
const item = await renderProviderImageOutputAsAssistantMarkdown({ path: imagePath });
|
||||
if (!item || item.type !== "assistant_message") {
|
||||
throw new Error("Expected provider image output to render as assistant markdown.");
|
||||
}
|
||||
@@ -40,8 +41,8 @@ describe("isProviderImageMarkdown", () => {
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test("emits Windows file paths as file URIs", () => {
|
||||
const markdown = renderImageMarkdown(
|
||||
test("emits Windows file paths as file URIs", async () => {
|
||||
const markdown = await renderImageMarkdown(
|
||||
`C:\\Users\\me\\AppData\\Local\\Temp\\paseo-attachments\\${HASH}.png`,
|
||||
);
|
||||
|
||||
@@ -51,22 +52,24 @@ describe("isProviderImageMarkdown", () => {
|
||||
expect(isProviderImageMarkdown(markdown)).toBe(true);
|
||||
});
|
||||
|
||||
test("emits POSIX file paths with spaces as valid file URI markdown", () => {
|
||||
const markdown = renderImageMarkdown("/home/user/Projects/Project With Spaces/screenshot.png");
|
||||
test("emits POSIX file paths with spaces as valid file URI markdown", async () => {
|
||||
const markdown = await renderImageMarkdown(
|
||||
"/home/user/Projects/Project With Spaces/screenshot.png",
|
||||
);
|
||||
|
||||
expect(markdown).toBe(
|
||||
"\n\n\n\n",
|
||||
);
|
||||
});
|
||||
|
||||
test("encodes URI-significant characters in POSIX file paths", () => {
|
||||
const markdown = renderImageMarkdown("/tmp/screenshot#1?draft.png");
|
||||
test("encodes URI-significant characters in POSIX file paths", async () => {
|
||||
const markdown = await renderImageMarkdown("/tmp/screenshot#1?draft.png");
|
||||
|
||||
expect(markdown).toBe("\n\n\n\n");
|
||||
});
|
||||
|
||||
test("preserves double-leading slashes in POSIX file paths", () => {
|
||||
const markdown = renderImageMarkdown("//tmp/screenshot#1.png");
|
||||
test("preserves double-leading slashes in POSIX file paths", async () => {
|
||||
const markdown = await renderImageMarkdown("//tmp/screenshot#1.png");
|
||||
|
||||
expect(markdown).toBe("\n\n\n\n");
|
||||
});
|
||||
@@ -78,8 +81,8 @@ describe("isProviderImageMarkdown", () => {
|
||||
"\\\\?\\UNC\\server\\share\\shot?draft.png",
|
||||
"file://server/share/shot%3Fdraft.png",
|
||||
],
|
||||
])("encodes %s image paths as file URIs", (_label, imagePath, expectedSource) => {
|
||||
expect(renderImageMarkdown(imagePath)).toBe(`\n\n\n\n`);
|
||||
])("encodes %s image paths as file URIs", async (_label, imagePath, expectedSource) => {
|
||||
expect(await renderImageMarkdown(imagePath)).toBe(`\n\n\n\n`);
|
||||
});
|
||||
|
||||
test("rejects user-authored markdown that is not a materialized attachment", () => {
|
||||
@@ -92,8 +95,8 @@ describe("isProviderImageMarkdown", () => {
|
||||
});
|
||||
|
||||
describe("materializeProviderImage", () => {
|
||||
test("recreates the private temp directory if the cached directory is removed", () => {
|
||||
const first = materializeProviderImage({
|
||||
test("recreates the private temp directory if the cached directory is removed", async () => {
|
||||
const first = await materializeProviderImage({
|
||||
data: "YWJjMTIz",
|
||||
mimeType: "image/png",
|
||||
});
|
||||
@@ -102,7 +105,7 @@ describe("materializeProviderImage", () => {
|
||||
|
||||
rmSync(firstDir, { recursive: true, force: true });
|
||||
|
||||
const second = materializeProviderImage({
|
||||
const second = await materializeProviderImage({
|
||||
data: "ZGVmNDU2",
|
||||
mimeType: "image/png",
|
||||
});
|
||||
@@ -117,8 +120,8 @@ describe("materializeProviderImage", () => {
|
||||
});
|
||||
|
||||
describe("provider image paragraph boundaries", () => {
|
||||
test.each(["", "\n"])("keeps image outputs separate from prose ending in %j", (ending) => {
|
||||
const image = renderProviderImageOutputAsAssistantMarkdown({
|
||||
test.each(["", "\n"])("keeps image outputs separate from prose ending in %j", async (ending) => {
|
||||
const image = await renderProviderImageOutputAsAssistantMarkdown({
|
||||
url: "https://example.com/chart.png",
|
||||
});
|
||||
if (!image) throw new Error("Expected image output");
|
||||
@@ -152,10 +155,10 @@ describe("provider image paragraph boundaries", () => {
|
||||
});
|
||||
});
|
||||
|
||||
test("materialized image output retains paragraph boundaries and attachment recognition", () => {
|
||||
const item = renderProviderImageOutputAsAssistantMarkdown(
|
||||
test("materialized image output retains paragraph boundaries and attachment recognition", async () => {
|
||||
const item = await renderProviderImageOutputAsAssistantMarkdown(
|
||||
{ data: "YWJjMTIz", mimeType: "image/png" },
|
||||
{ materialize: () => ({ path: `/tmp/paseo-attachments/${HASH}.png` }) },
|
||||
{ materialize: async () => ({ path: `/tmp/paseo-attachments/${HASH}.png` }) },
|
||||
);
|
||||
expect(item).toEqual({
|
||||
type: "assistant_message",
|
||||
@@ -164,3 +167,76 @@ test("materialized image output retains paragraph boundaries and attachment reco
|
||||
if (!item || item.type !== "assistant_message") throw new Error("Expected image output");
|
||||
expect(isProviderImageMarkdown(item.text)).toBe(true);
|
||||
});
|
||||
|
||||
test("repeated content is reused without rewriting the attachment", async () => {
|
||||
const image = {
|
||||
data: Buffer.from("dedupe regression").toString("base64"),
|
||||
mimeType: "image/png",
|
||||
};
|
||||
const first = await materializeProviderImage(image);
|
||||
const before = statSync(first.path, { bigint: true });
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
const second = await materializeProviderImage(image);
|
||||
expect(second.path).toBe(first.path);
|
||||
expect(statSync(second.path, { bigint: true }).mtimeNs).toBe(before.mtimeNs);
|
||||
});
|
||||
|
||||
test("64 MiB materialization leaves the event loop responsive and preserves bytes", async () => {
|
||||
const bytes = Buffer.alloc(64 * 1024 * 1024, 37);
|
||||
const data = bytes.toString("base64");
|
||||
let previous = performance.now();
|
||||
let longest = 0;
|
||||
const probe = setInterval(() => {
|
||||
const now = performance.now();
|
||||
longest = Math.max(longest, now - previous);
|
||||
previous = now;
|
||||
}, 1);
|
||||
try {
|
||||
const result = await materializeProviderImage({ data, mimeType: "image/png" });
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
console.log(`64 MiB attachment longest loop gap: ${longest.toFixed(2)} ms`);
|
||||
expect(longest).toBeLessThan(50);
|
||||
expect(readFileSync(result.path).equals(bytes)).toBe(true);
|
||||
rmSync(result.path, { force: true });
|
||||
} finally {
|
||||
clearInterval(probe);
|
||||
}
|
||||
});
|
||||
|
||||
test("preserves base64 decoding across whitespace and chunk boundaries", async () => {
|
||||
const bytes = Buffer.alloc(512 * 1024, 123);
|
||||
const base64 = bytes.toString("base64");
|
||||
const data = base64.match(/.{1,77}/g)!.join("\n");
|
||||
const result = await materializeProviderImage({ data, mimeType: "image/png" });
|
||||
expect(readFileSync(result.path).equals(bytes)).toBe(true);
|
||||
});
|
||||
|
||||
test("rejects oversized materialization before work and leaves the queue usable", async () => {
|
||||
const data = "A".repeat(Math.ceil(MAX_PROVIDER_IMAGE_BYTES / 3) * 4 + 260);
|
||||
await expect(materializeProviderImage({ data, mimeType: "image/png" })).rejects.toThrow(
|
||||
"exceeds",
|
||||
);
|
||||
const omitted = await renderProviderImageOutputAsAssistantMarkdown(
|
||||
{ data, mimeType: "image/png" },
|
||||
{ materialize: materializeProviderImage },
|
||||
);
|
||||
expect(omitted).toEqual({
|
||||
type: "assistant_message",
|
||||
text: "Image output was omitted because it was not available as a file path or URL.",
|
||||
});
|
||||
const image = await materializeProviderImage({ data: "YWJjMTIz", mimeType: "image/png" });
|
||||
expect(readFileSync(image.path).toString()).toBe("abc123");
|
||||
});
|
||||
|
||||
test("simultaneous requests for the same content share the published file", async () => {
|
||||
const image = {
|
||||
data: Buffer.from("concurrent dedupe fixture").toString("base64"),
|
||||
mimeType: "image/png",
|
||||
};
|
||||
const results = await Promise.all([
|
||||
materializeProviderImage(image),
|
||||
materializeProviderImage(image),
|
||||
]);
|
||||
expect(results[0]).toEqual(results[1]);
|
||||
expect(readFileSync(results[0].path).toString()).toBe("concurrent dedupe fixture");
|
||||
});
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import * as fsSync from "node:fs";
|
||||
import * as fs from "node:fs/promises";
|
||||
import { setImmediate } from "node:timers/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
@@ -23,32 +24,26 @@ const PRIVATE_ATTACHMENT_DIR_MODE = 0o700;
|
||||
const MATERIALIZED_IMAGE_FILE_MODE = 0o600;
|
||||
|
||||
let materializedImageAttachmentDir: string | null = null;
|
||||
// Serialize materializations: one decoder and one file operation at a time, including
|
||||
// concurrent history replay and live turns. A rejected operation must not poison the queue.
|
||||
let pendingMaterialization: Promise<unknown> = Promise.resolve();
|
||||
|
||||
function canReuseMaterializedImageAttachmentDir(dir: string): boolean {
|
||||
try {
|
||||
const stats = fsSync.lstatSync(dir);
|
||||
if (!stats.isDirectory()) {
|
||||
return false;
|
||||
async function getMaterializedImageAttachmentDir(): Promise<string> {
|
||||
if (materializedImageAttachmentDir) {
|
||||
try {
|
||||
const stats = await fs.lstat(materializedImageAttachmentDir);
|
||||
if (stats.isDirectory()) {
|
||||
await fs.chmod(materializedImageAttachmentDir, PRIVATE_ATTACHMENT_DIR_MODE);
|
||||
return materializedImageAttachmentDir;
|
||||
}
|
||||
} catch {
|
||||
/* Temp storage may have been removed between turns. */
|
||||
}
|
||||
fsSync.chmodSync(dir, PRIVATE_ATTACHMENT_DIR_MODE);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function getMaterializedImageAttachmentDir(): string {
|
||||
if (
|
||||
materializedImageAttachmentDir &&
|
||||
canReuseMaterializedImageAttachmentDir(materializedImageAttachmentDir)
|
||||
) {
|
||||
return materializedImageAttachmentDir;
|
||||
}
|
||||
|
||||
materializedImageAttachmentDir = fsSync.mkdtempSync(
|
||||
materializedImageAttachmentDir = await fs.mkdtemp(
|
||||
path.join(os.tmpdir(), PROVIDER_IMAGE_ATTACHMENT_DIR_PREFIX),
|
||||
);
|
||||
fsSync.chmodSync(materializedImageAttachmentDir, PRIVATE_ATTACHMENT_DIR_MODE);
|
||||
await fs.chmod(materializedImageAttachmentDir, PRIVATE_ATTACHMENT_DIR_MODE);
|
||||
return materializedImageAttachmentDir;
|
||||
}
|
||||
|
||||
@@ -71,32 +66,92 @@ function getImageExtension(mimeType: string): string {
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeImageData(mimeType: string, data: string): { mimeType: string; data: string } {
|
||||
if (data.startsWith("data:")) {
|
||||
const match = data.match(/^data:([^;]+);base64,(.*)$/);
|
||||
if (match) {
|
||||
return { mimeType: match[1], data: match[2] };
|
||||
}
|
||||
}
|
||||
return { mimeType, data };
|
||||
}
|
||||
// 128 MiB admits the measured 64 MiB fixture with headroom while bounding disk
|
||||
// and decoding per image. This is materialization admission, not a wire schema limit.
|
||||
export const MAX_PROVIDER_IMAGE_BYTES = 128 * 1024 * 1024;
|
||||
const BASE64_CHUNK_CHARACTERS = 256 * 1024; // Multiple of four; decode units stay aligned.
|
||||
|
||||
// Filenames are a content hash of the bytes so re-materializing the same image
|
||||
// within a process reuses the existing temp file instead of leaking a fresh one
|
||||
// for repeated image blocks or history replay.
|
||||
export function materializeProviderImage(image: {
|
||||
data: string;
|
||||
mimeType: string | null;
|
||||
}): MaterializedProviderImage {
|
||||
const attachmentsDir = getMaterializedImageAttachmentDir();
|
||||
const normalized = normalizeImageData(image.mimeType ?? "image/png", image.data);
|
||||
const bytes = Buffer.from(normalized.data, "base64");
|
||||
const extension = getImageExtension(normalized.mimeType);
|
||||
const hash = createHash("sha256").update(bytes).digest("hex");
|
||||
const filePath = path.join(attachmentsDir, `${hash}.${extension}`);
|
||||
fsSync.writeFileSync(filePath, bytes, { mode: MATERIALIZED_IMAGE_FILE_MODE });
|
||||
fsSync.chmodSync(filePath, MATERIALIZED_IMAGE_FILE_MODE);
|
||||
return { path: filePath };
|
||||
}): Promise<MaterializedProviderImage> {
|
||||
const result = pendingMaterialization.then(() => materializeImage(image));
|
||||
pendingMaterialization = result.catch(() => {});
|
||||
return result;
|
||||
}
|
||||
|
||||
async function materializeImage(image: {
|
||||
data: string;
|
||||
mimeType: string | null;
|
||||
}): Promise<MaterializedProviderImage> {
|
||||
const maxEncodedCharacters = Math.ceil(MAX_PROVIDER_IMAGE_BYTES / 3) * 4;
|
||||
// Check length before inspecting/copying the payload, including a bounded data URI header.
|
||||
if (image.data.length > maxEncodedCharacters + 256) {
|
||||
throw new Error(`Image attachment exceeds ${MAX_PROVIDER_IMAGE_BYTES} bytes`);
|
||||
}
|
||||
let data = image.data;
|
||||
let mimeType = image.mimeType ?? "image/png";
|
||||
if (data.startsWith("data:")) {
|
||||
const separator = data.indexOf(";base64,");
|
||||
if (separator >= 0 && separator < 256) {
|
||||
mimeType = data.slice(5, separator);
|
||||
data = data.slice(separator + 8);
|
||||
}
|
||||
}
|
||||
if (data.length > maxEncodedCharacters) {
|
||||
throw new Error(`Image attachment exceeds ${MAX_PROVIDER_IMAGE_BYTES} bytes`);
|
||||
}
|
||||
const hash = createHash("sha256");
|
||||
for await (const bytes of decodeImage(data)) hash.update(bytes);
|
||||
const dir = await getMaterializedImageAttachmentDir();
|
||||
const filePath = path.join(dir, `${hash.digest("hex")}.${getImageExtension(mimeType)}`);
|
||||
try {
|
||||
await fs.stat(filePath);
|
||||
return { path: filePath };
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
||||
}
|
||||
|
||||
const staging = await fs.mkdtemp(path.join(dir, ".pending-"));
|
||||
const stagingPath = path.join(staging, "image");
|
||||
try {
|
||||
await fs.chmod(staging, PRIVATE_ATTACHMENT_DIR_MODE);
|
||||
const file = await fs.open(stagingPath, "wx", MATERIALIZED_IMAGE_FILE_MODE);
|
||||
try {
|
||||
await file.chmod(MATERIALIZED_IMAGE_FILE_MODE);
|
||||
// Decode again only for a new hash. This bounds retained memory to one chunk
|
||||
// and performs no writes at all for repeated content, including history replay.
|
||||
for await (const bytes of decodeImage(data)) await file.writeFile(bytes);
|
||||
} finally {
|
||||
await file.close();
|
||||
}
|
||||
await fs.rename(stagingPath, filePath);
|
||||
return { path: filePath };
|
||||
} finally {
|
||||
await fs.rm(staging, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function* decodeImage(data: string): AsyncGenerator<Buffer> {
|
||||
let carry = "";
|
||||
let decodedBytes = 0;
|
||||
for (let offset = 0; offset < data.length; offset += BASE64_CHUNK_CHARACTERS) {
|
||||
// Buffer.from ignores non-base64 characters. Carry incomplete quartets so
|
||||
// line-wrapped payloads decode identically across chunk boundaries.
|
||||
const chunk =
|
||||
carry +
|
||||
data.slice(offset, offset + BASE64_CHUNK_CHARACTERS).replace(/[^A-Za-z0-9+/_=-]/g, "");
|
||||
const finalChunk = offset + BASE64_CHUNK_CHARACTERS >= data.length || chunk.includes("=");
|
||||
const end = finalChunk ? chunk.length : chunk.length - (chunk.length % 4);
|
||||
const bytes = Buffer.from(chunk.slice(0, end), "base64");
|
||||
carry = chunk.slice(end);
|
||||
decodedBytes += bytes.length;
|
||||
if (decodedBytes > MAX_PROVIDER_IMAGE_BYTES)
|
||||
throw new Error("Image attachment exceeds size limit");
|
||||
yield bytes;
|
||||
await setImmediate();
|
||||
if (finalChunk) break;
|
||||
}
|
||||
}
|
||||
|
||||
// Recognizes markdown rendered for a materialized provider image: its source is a content-hashed
|
||||
@@ -112,7 +167,10 @@ export function isProviderImageMarkdown(text: string): boolean {
|
||||
}
|
||||
|
||||
interface RenderProviderImageOutputOptions {
|
||||
materialize?: (image: { data: string; mimeType: string | null }) => MaterializedProviderImage;
|
||||
materialize?: (image: {
|
||||
data: string;
|
||||
mimeType: string | null;
|
||||
}) => Promise<MaterializedProviderImage>;
|
||||
}
|
||||
|
||||
function nonEmptyString(value: string | null | undefined): string | null {
|
||||
@@ -169,10 +227,10 @@ function escapeMarkdownImageSource(value: string): string {
|
||||
return markdownImageSource(value).replace(/\\/g, "\\\\").replace(/\)/g, "\\)");
|
||||
}
|
||||
|
||||
export function renderProviderImageOutputAsAssistantMarkdown(
|
||||
export async function renderProviderImageOutputAsAssistantMarkdown(
|
||||
image: ProviderImageOutput,
|
||||
options: RenderProviderImageOutputOptions = {},
|
||||
): AgentTimelineItem | null {
|
||||
): Promise<AgentTimelineItem | null> {
|
||||
const source = nonEmptyString(image.path) ?? nonEmptyString(image.url);
|
||||
if (source && !isDataImageSource(source)) {
|
||||
const altText = escapeMarkdownImageAlt(nonEmptyString(image.altText) ?? "Image");
|
||||
@@ -191,7 +249,7 @@ export function renderProviderImageOutputAsAssistantMarkdown(
|
||||
let materialized: MaterializedProviderImage | null = null;
|
||||
try {
|
||||
materialized = options.materialize
|
||||
? options.materialize({
|
||||
? await options.materialize({
|
||||
data,
|
||||
mimeType: nonEmptyString(image.mimeType),
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user