fix(pi): show tool-result images live and after reload (#8280)

* pi: render tool-result images as inline image rows (paseo #1717 parity)

* fix(pi): preserve turn ownership while rendering images

* fix(pi): preserve metadata when stripping nested image payloads

---------

Co-authored-by: be-my-bot <bot@be-my-code.local>
Co-authored-by: paseo-bot[bot] <266920839+paseo-bot[bot]@users.noreply.github.com>
Co-authored-by: duyiliu <duyiliu@users.noreply.github.com>
This commit is contained in:
paseo-bot[bot]
2026-10-30 10:10:39 +00:00
committed by GitHub
co-authored by be-my-bot duyiliu
parent e299b21e94
commit 531370c4ce
10 changed files with 429 additions and 26 deletions
@@ -3643,3 +3643,114 @@ test("closing Pi during image conversion rejects its pending start", async () =>
await session.close();
await rejected;
});
describe("Pi tool-result images", () => {
test.each([false, true])(
"tool-result images precede the reply and completion (isError=%s)",
async (isError) => {
const { pi, session, events } = await createSession();
onTestFinished(() => session.close());
const fakeSession = pi.latestSession();
await session.startTurn("run");
fakeSession.emit({
type: "tool_execution_start",
toolCallId: "shot-1",
toolName: "screenshot",
args: {},
});
fakeSession.emit({
type: "tool_execution_end",
toolCallId: "shot-1",
toolName: "screenshot",
result: {
content: [
{ type: "text", text: "captured" },
{ type: "image", data: Buffer.from("hello").toString("base64"), mimeType: "image/png" },
],
structuredContent: {
type: "image",
data: Buffer.from("hello").toString("base64"),
mimeType: "image/png",
note: "captured",
},
},
isError,
});
fakeSession.emit({
type: "message_update",
assistantMessageEvent: { type: "text_delta", delta: "Image read complete." },
});
fakeSession.finishTurn();
await events.nextTurnCompletion();
const items = events.timelineItems();
const toolIndex = items.findIndex(
(item) =>
item.type === "tool_call" &&
item.callId === "shot-1" &&
item.status === (isError ? "failed" : "completed"),
);
expect(toolIndex).toBeGreaterThanOrEqual(0);
// Base64 must never reach the tool output stringifier.
expect(JSON.stringify(items[toolIndex])).not.toContain(
Buffer.from("hello").toString("base64"),
);
expect(items.slice(toolIndex + 1)).toEqual([
expect.objectContaining({
type: "assistant_message",
text: expect.stringContaining("![Image]("),
}),
expect.objectContaining({ type: "assistant_message", text: "Image read complete." }),
]);
expect(events.timelineAndCompletionEvents().at(-1)).toEqual({ type: "turn_completed" });
},
);
});
test("Pi image rendering preserves turn ownership across an interrupt and follow-up", async () => {
const { pi, session, events } = await createSession();
onTestFinished(() => session.close());
const emitted: AgentStreamEvent[] = [];
session.subscribe((event) => emitted.push(event));
const native = pi.latestSession();
const first = await session.startTurn("read image");
native.emit({
type: "tool_execution_end",
toolCallId: "large-image",
toolName: "screenshot",
result: {
content: [
{
type: "image",
data: Buffer.alloc(8 * 1024 * 1024, 73).toString("base64"),
mimeType: "image/png",
},
],
},
isError: false,
});
native.finishTurn();
await session.interrupt();
const followup = await session.startTurn("follow-up after image");
await waitForImmediate();
await waitForImmediate();
await expect
.poll(() =>
events
.timelineItems()
.some((item) => item.type === "assistant_message" && item.text.includes("![Image](")),
)
.toBe(true);
expect(events.turnCompletedEvents().some((event) => event.turnId === followup.turnId)).toBe(
false,
);
const imageEvent = emitted.find(
(event) =>
event.type === "timeline" &&
event.item.type === "assistant_message" &&
event.item.text.includes("![Image]("),
);
expect(imageEvent).toMatchObject({ turnId: first.turnId });
native.finishTurn();
await events.nextTurnCompletion();
});
@@ -64,7 +64,11 @@ import {
streamPiHistory,
type PiCapturedUserMessageEntry,
} from "./history-mapper.js";
import { materializeProviderImage } from "../provider-image-output.js";
import {
materializeProviderImage,
renderProviderImageOutputAsAssistantMarkdown,
type ProviderImageOutput,
} from "../provider-image-output.js";
import { PiCliRuntime } from "./cli-runtime.js";
import {
createPiExtensionHost,
@@ -89,6 +93,7 @@ import {
mapToolDetail,
parseToolArgs,
parseToolResult,
splitPiToolResultImages,
type PiToolResult,
type PiTrackedToolCall,
} from "./tool-call-mapper.js";
@@ -1201,6 +1206,7 @@ export class PiRpcAgentSession implements AgentSession {
private closed = false;
private readonly closeController = new AbortController();
private readonly pendingExtensionHydrations = new Set<Promise<void>>();
private pendingOutput: Promise<void> | null = null;
// Pi publishes the terminal before acknowledging abort. Autonomous runs have no
// turn ID; retain their errors too until the cancellation request settles.
private interruptingTurn: { turnId: string | undefined; error: string | null } | null = null;
@@ -1602,6 +1608,7 @@ export class PiRpcAgentSession implements AgentSession {
this.pendingExtensionUiRequests.size ||
this.pendingExtensionResults.size ||
this.pendingExtensionHydrations.size ||
this.pendingOutput ||
this.pendingSteerSubmissions.length ||
this.pendingNoTurnOutputs.length ||
this.interruptingTurn
@@ -1622,7 +1629,7 @@ export class PiRpcAgentSession implements AgentSession {
try {
await this.runtimeSession.close();
} finally {
await Promise.all(this.pendingExtensionHydrations);
await Promise.all([this.pendingOutput, ...this.pendingExtensionHydrations]);
this.rejectAllExtensionResults(new Error("Pi session closed"));
this.cleanup?.();
}
@@ -1686,9 +1693,31 @@ export class PiRpcAgentSession implements AgentSession {
}
private emit(event: AgentStreamEvent): void {
for (const subscriber of this.subscribers) {
subscriber(event);
if (this.pendingOutput) {
this.emitAsync(() => Promise.resolve([event]));
return;
}
this.publish(event);
}
private publish(event: AgentStreamEvent): void {
for (const subscriber of this.subscribers) subscriber(event);
}
// Keep timeline order while images are materialized, without delaying provider state changes.
private emitAsync(render: () => Promise<AgentStreamEvent[]>): void {
const pending = (this.pendingOutput ?? Promise.resolve())
.then(render)
.then((events) => {
if (!this.closed) for (const event of events) this.publish(event);
return undefined;
})
.catch((error) => this.logger.error({ err: error }, "Pi output rendering failed"));
this.pendingOutput = pending;
void pending.then(() => {
if (this.pendingOutput === pending) this.pendingOutput = null;
return undefined;
});
}
private emitExtensionOutput(output: PiExtensionEventOutput | undefined, turnId?: string): void {
@@ -2315,8 +2344,11 @@ export class PiRpcAgentSession implements AgentSession {
this.activeToolCalls.get(event.toolCallId) ?? parseToolArgs(event.toolName, null);
this.activeToolCalls.delete(event.toolCallId);
const result = parseToolResult(event.result);
const error = event.isError ? event.result : null;
// Pull image blocks out of the result so base64 never reaches the tool output, then render
// each one as an assistant_message markdown image after the tool_call (mirroring the Claude
// provider, upstream #1717).
const { images, result } = splitPiToolResultImages(parseToolResult(event.result));
const error = event.isError ? result : null;
const status = event.isError ? "failed" : "completed";
this.extensionHost.onToolEnd({
callId: event.toolCallId,
@@ -2326,6 +2358,30 @@ export class PiRpcAgentSession implements AgentSession {
result,
});
this.emitToolCallEvent(event.toolCallId, toolCall, status, result, error);
if (images.length === 0) return;
const turnId = this.currentTurnIdForEvent();
this.emitAsync(() => this.renderToolResultImages(images, turnId));
}
private async renderToolResultImages(
images: ProviderImageOutput[],
turnId: string | undefined,
): Promise<AgentStreamEvent[]> {
const events: AgentStreamEvent[] = [];
for (const image of images) {
const imageItem = await renderProviderImageOutputAsAssistantMarkdown(image, {
materialize: materializeProviderImage,
});
if (imageItem && !this.closed) {
events.push({
type: "timeline",
provider: this.provider,
turnId,
item: imageItem,
});
}
}
return events;
}
private emitCompactionTimeline(input: {
@@ -21,11 +21,11 @@ export function mapPiChildSession(
return new PiChildSessionFollower(id, file, maxBytes).readNew();
}
function mapChildLine(
async function mapChildLine(
mapper: PiHistoryMapper,
id: string,
line: string,
): ProviderSubagentInputEvent[] {
): Promise<ProviderSubagentInputEvent[]> {
let entry: { message?: PiAgentMessage; timestamp?: string };
try {
entry = JSON.parse(line);
@@ -33,7 +33,7 @@ function mapChildLine(
return [];
}
if (!entry.message || typeof entry.message !== "object" || !("role" in entry.message)) return [];
return mapper.mapMessages([entry.message]).flatMap((mapped) =>
return (await mapper.mapMessages([entry.message])).flatMap((mapped) =>
mapped.type === "timeline"
? [
{
@@ -83,7 +83,10 @@ export class PiChildSessionFollower {
const events: ProviderSubagentInputEvent[] = [];
for (const line of data.subarray(0, lastNewline).toString("utf8").split("\n")) {
if (this.items >= MAX_ITEMS) break;
const mapped = mapChildLine(this.mapper, this.id, line).slice(0, MAX_ITEMS - this.items);
const mapped = (await mapChildLine(this.mapper, this.id, line)).slice(
0,
MAX_ITEMS - this.items,
);
this.items += mapped.length;
events.push(...mapped);
}
@@ -83,7 +83,7 @@ describe("Pi extension host", () => {
test("declines a throwing adapter and preserves generic history", async () => {
const host = createPiExtensionHost(undefined, [throwingAdapter]);
const mapper = new PiHistoryMapper("pi", [], {}, host);
const events = mapper.mapMessages([
const events = await mapper.mapMessages([
{
role: "assistant",
content: [{ type: "toolCall", id: "call-1", name: "other", arguments: {} }],
@@ -193,7 +193,7 @@ describe("Pi extension host", () => {
await output?.hydration;
expect(reads).toHaveLength(8);
});
test("claims a tool, declines an unrelated one, and maps history like live events", () => {
test("claims a tool, declines an unrelated one, and maps history like live events", async () => {
const host = createPiExtensionHost();
const args = { agent: "scout", task: "Inspect files" };
const result = parseToolResult({ content: [{ type: "text", text: "Found two files" }] });
@@ -225,7 +225,7 @@ describe("Pi extension host", () => {
).toBeUndefined();
const mapper = new PiHistoryMapper("pi");
const events = mapper.mapMessages([
const events = await mapper.mapMessages([
{
role: "assistant",
content: [{ type: "toolCall", id: "call-1", name: "subagent", arguments: args }],
@@ -79,7 +79,7 @@ test("cancellation declines each remaining real RPC dialog", () => {
}
});
test("captured result renders in live and history tool rows, while foreign shapes decline", () => {
test("captured result renders in live and history tool rows, while foreign shapes decline", async () => {
const call = {
callId: end.toolCallId,
toolName: end.toolName,
@@ -93,7 +93,7 @@ test("captured result renders in live and history tool rows, while foreign shape
label: "Questions",
text: "Which snack?: Cookie\nWhich colors?: Red, Blue\nWhat project name?: Nebula",
});
const history = new PiHistoryMapper("pi").mapMessages([
const history = await new PiHistoryMapper("pi").mapMessages([
{
role: "assistant",
content: [
@@ -89,7 +89,7 @@ export async function verifyTaskFixture(path: URL, expectedStatuses: string[]):
for (const event of fixture.events) pi.latestSession().emit(event);
pi.latestSession().finishTurn();
expect(todos(live)).toEqual(mapped);
const replay = new PiHistoryMapper("pi").mapMessages(fixture.messages);
const replay = await new PiHistoryMapper("pi").mapMessages(fixture.messages);
expect(todos(replay)).toEqual(mapped);
expect(
replay.filter(
@@ -221,3 +221,75 @@ describe("Pi history mapper", () => {
]);
});
});
describe("Pi tool-result image replay", () => {
test("recreates an image from native tool-result history without its source file", async () => {
const events = await collectHistory([
{
role: "assistant",
content: [
{
type: "toolCall",
id: "image-read",
name: "read",
arguments: { path: "missing-preview.png" },
},
],
},
{
role: "toolResult",
toolCallId: "image-read",
toolName: "read",
content: [
{ type: "text", text: "Read image file [image/png]" },
{ type: "image", data: "aGVsbG8=", mimeType: "image/png" },
],
},
]);
expect(events.map((event) => event.item?.type)).toEqual([
"tool_call",
"tool_call",
"assistant_message",
]);
const image = events.find(
(event) => event.type === "timeline" && event.item.type === "assistant_message",
);
expect(image).toMatchObject({
type: "timeline",
item: { type: "assistant_message", text: expect.stringContaining("![Image](file://") },
});
if (image?.type !== "timeline" || image.item.type !== "assistant_message")
throw new Error("Missing image");
const url = image.item.text.match(/\((file:[^)]+)\)/)![1];
const { readFile, unlink } = await import("node:fs/promises");
expect(await readFile(new URL(url))).toEqual(Buffer.from("hello"));
await unlink(new URL(url));
const replay = await collectHistory([
{
role: "toolResult",
toolCallId: "image-read",
toolName: "read",
content: [{ type: "image", data: "aGVsbG8=", mimeType: "image/png" }],
},
]);
expect(replay).toContainEqual(image);
expect(await readFile(new URL(url))).toEqual(Buffer.from("hello"));
});
});
test("replayed Pi images keep their position before the following reply", async () => {
const events = await collectHistory([
{
role: "toolResult",
toolCallId: "image-position",
toolName: "read",
content: [{ type: "image", data: "aGVsbG8=", mimeType: "image/png" }],
},
{ role: "assistant", content: [{ type: "text", text: "Image read complete." }] },
]);
expect(
events.map((event) =>
event.item?.type === "assistant_message" ? event.item.text : event.item?.type,
),
).toEqual(["tool_call", expect.stringContaining("![Image]("), "Image read complete."]);
});
@@ -1,5 +1,9 @@
import { mapCustomMessageToToolCall } from "../custom-message.js";
import type { AgentStreamEvent, AgentTimelineItem, ToolCallDetail } from "../../agent-sdk-types.js";
import {
materializeProviderImage,
renderProviderImageOutputAsAssistantMarkdown,
} from "../provider-image-output.js";
import {
createPiExtensionHost,
type PiExtensionEventOutput,
@@ -11,6 +15,7 @@ import {
mapToolDetail,
parseToolArgs,
parseToolResult,
splitPiToolResultImages,
type PiToolResult,
type PiTrackedToolCall,
} from "./tool-call-mapper.js";
@@ -71,7 +76,7 @@ export class PiHistoryMapper {
private readonly extensionHost: PiExtensionHost = createPiExtensionHost(),
) {}
mapMessages(messages: readonly PiAgentMessage[]): AgentStreamEvent[] {
async mapMessages(messages: readonly PiAgentMessage[]): Promise<AgentStreamEvent[]> {
const events: AgentStreamEvent[] = [];
for (const message of messages) {
@@ -86,7 +91,7 @@ export class PiHistoryMapper {
events.push(...this.mapAssistantMessage(message));
break;
case "toolResult": {
events.push(...this.mapToolResultMessage(message));
events.push(...(await this.mapToolResultMessage(message)));
break;
}
case "bashExecution":
@@ -207,24 +212,35 @@ export class PiHistoryMapper {
return events;
}
private mapToolResultMessage(
private async mapToolResultMessage(
message: Extract<PiAgentMessage, { role: "toolResult" }>,
): AgentStreamEvent[] {
): Promise<AgentStreamEvent[]> {
const tracked =
this.pendingToolCalls.get(message.toolCallId) ?? parseToolArgs(message.toolName, null);
this.pendingToolCalls.delete(message.toolCallId);
const result = parseToolResult({ content: message.content, details: message.details });
// Pull image blocks out of the result so base64 never reaches the tool output, then render
// each one as an assistant_message markdown image after the tool_call (mirroring the live
// path and the Claude provider, upstream #1717).
const { images, result: splitResult } = splitPiToolResultImages(result);
const mapping = this.extensionHost.mapToolCall({
callId: message.toolCallId,
toolName: tracked.toolName,
args: tracked.args,
status: message.isError ? "failed" : "completed",
result,
result: splitResult,
});
const detail = this.mapToolDetail(message.toolCallId, tracked, result, mapping?.detail);
const detail = this.mapToolDetail(message.toolCallId, tracked, splitResult, mapping?.detail);
if (!detail) {
return [];
}
const imageEvents: AgentStreamEvent[] = [];
for (const image of images) {
const item = await renderProviderImageOutputAsAssistantMarkdown(image, {
materialize: materializeProviderImage,
});
if (item) imageEvents.push({ type: "timeline", provider: this.provider, item });
}
return [
{
type: "timeline",
@@ -234,9 +250,10 @@ export class PiHistoryMapper {
name: mapping?.name ?? tracked.toolName,
isError: Boolean(message.isError),
detail,
errorText: extractTextFromToolResult(result) ?? "Tool call failed",
errorText: extractTextFromToolResult(splitResult) ?? "Tool call failed",
}),
},
...imageEvents,
...this.extensionEvents(mapping),
];
}
@@ -297,9 +314,10 @@ export async function* streamPiHistory(
signal?: AbortSignal,
): AsyncGenerator<AgentStreamEvent> {
const mapper = new PiHistoryMapper(provider, userEntries, hooks, extensionHost);
for (const event of mapper.mapMessages(messages)) {
for (const message of messages) {
if (signal?.aborted) return;
if (event) {
for (const event of await mapper.mapMessages([message])) {
if (signal?.aborted) return;
yield event;
}
}
@@ -1,6 +1,12 @@
import { describe, expect, test } from "vitest";
import { mapToolDetail, parseToolArgs, parseToolResult } from "./tool-call-mapper.js";
import {
extractTextFromToolResult,
mapToolDetail,
parseToolArgs,
parseToolResult,
splitPiToolResultImages,
} from "./tool-call-mapper.js";
describe("Pi tool call mapper", () => {
test("maps bash args and result to shell detail", () => {
@@ -56,3 +62,77 @@ describe("Pi tool call mapper", () => {
});
});
});
describe("splitPiToolResultImages", () => {
test("extracts image blocks and replaces them with [image] placeholders", () => {
const parsed = parseToolResult({
content: [
{ type: "text", text: "shot taken" },
{ type: "image", data: "aGVsbG8=", mimeType: "image/png" },
],
});
const { images, result } = splitPiToolResultImages(parsed);
expect(images).toEqual([{ data: "aGVsbG8=", mimeType: "image/png" }]);
expect(result).toEqual({
content: [
{ type: "text", text: "shot taken" },
{ type: "text", text: "[image]" },
],
});
});
test("replaces image-only results with a non-empty placeholder", () => {
const parsed = parseToolResult({
content: [{ type: "image", data: "aGVsbG8=", mimeType: "image/jpeg" }],
});
const { images, result } = splitPiToolResultImages(parsed);
expect(images).toHaveLength(1);
expect(extractTextFromToolResult(result)).toBe("[image]");
});
test("passes through string results and results without images", () => {
expect(splitPiToolResultImages("plain output")).toEqual({
images: [],
result: "plain output",
});
const textOnly = parseToolResult({ content: [{ type: "text", text: "no images" }] });
expect(splitPiToolResultImages(textOnly)).toEqual({ images: [], result: textOnly });
expect(splitPiToolResultImages(null)).toEqual({ images: [], result: null });
});
test("keeps other fields on the result object", () => {
const parsed = parseToolResult({
exitCode: 0,
content: [{ type: "image", data: "aGVsbG8=" }],
});
const { result } = splitPiToolResultImages(parsed);
expect(result).toMatchObject({ exitCode: 0 });
});
});
test("tool-result images remove nested structured image data while preserving metadata", () => {
const data = Buffer.from("nested screenshot bytes").toString("base64");
const result = parseToolResult({
content: [{ type: "image", data, mimeType: "image/png" }],
structuredContent: {
content: [{ type: "image", data, mimeType: "image/png", width: 64 }],
captureId: "capture-1",
read: { type: "image", data, mimeType: "image/png", note: "Read image file" },
extra: { data: "keep ordinary data" },
},
});
const split = splitPiToolResultImages(result);
const detail = mapToolDetail(parseToolArgs("screenshot", {}), split.result);
expect(JSON.stringify(detail)).not.toContain(data);
expect(split.images).toEqual([{ data, mimeType: "image/png" }]);
expect(split.result).toMatchObject({
structuredContent: {
content: [{ type: "image", data: "[image]", mimeType: "image/png", width: 64 }],
captureId: "capture-1",
read: { type: "image", data: "[image]", mimeType: "image/png", note: "Read image file" },
extra: { data: "keep ordinary data" },
},
});
});
@@ -1,6 +1,7 @@
import { z } from "zod";
import type { ToolCallDetail } from "../../agent-sdk-types.js";
import type { ProviderImageOutput } from "../provider-image-output.js";
interface BashToolInput {
command: string;
@@ -52,6 +53,7 @@ interface PiToolResultObject {
stdout?: string;
text?: string;
content?: PiToolResultContent[];
structuredContent?: unknown;
exitCode?: number;
code?: number;
details?: PiToolResultDetails;
@@ -67,6 +69,12 @@ interface PiToolResultTextContent {
text: string;
}
const PiToolResultImageContentSchema = z.object({
type: z.literal("image"),
data: z.string().min(1),
mimeType: z.string().optional(),
});
interface PiToolResultUnknownContent {
type: string;
}
@@ -236,6 +244,61 @@ export function parseToolResult(rawResult: unknown): PiToolResult {
return null;
}
// Pi returns images inside tool-result content as base64 blocks. Left in place they reach the
// tool-output stringifier, which dumps base64 into the tool output. We pull those blocks out to
// render them as image markdown and leave a "[image]" placeholder so image-only results still
// produce non-empty output (mirroring splitClaudeToolResultImages).
export function splitPiToolResultImages(result: PiToolResult): {
images: ProviderImageOutput[];
result: PiToolResult;
} {
if (!result || typeof result === "string" || !result.content) {
return { images: [], result };
}
const images: ProviderImageOutput[] = [];
let replaced = false;
const content = result.content.map((block) => {
const image = toPiImageOutput(block);
if (image) {
images.push(image);
replaced = true;
return { type: "text", text: "[image]" } satisfies PiToolResultTextContent;
}
return block;
});
if (!replaced) {
return { images: [], result };
}
return {
images,
result: {
...result,
content,
...(result.structuredContent === undefined
? {}
: { structuredContent: withoutImageData(result.structuredContent) }),
},
};
}
function withoutImageData(value: unknown): unknown {
if (Array.isArray(value)) return value.map(withoutImageData);
if (value === null || typeof value !== "object") return value;
const image = toPiImageOutput(value);
return Object.fromEntries(
Object.entries(value).map(([key, child]) => [
key,
image && key === "data" ? "[image]" : withoutImageData(child),
]),
);
}
function toPiImageOutput(block: unknown): ProviderImageOutput | null {
const parsed = PiToolResultImageContentSchema.safeParse(block);
if (!parsed.success) return null;
return { data: parsed.data.data, mimeType: parsed.data.mimeType ?? null };
}
export function extractTextFromToolResult(result: PiToolResult): string | undefined {
if (typeof result === "string") {
return result;