test(app): cover real assistant image files

This commit is contained in:
Mohamed Boudra
2026-07-28 19:25:53 +02:00
parent 29acf52a9a
commit 5de9021c78
4 changed files with 451 additions and 144 deletions
@@ -1,163 +1,85 @@
import type { Page } from "@playwright/test";
import { expect, test as base } from "./fixtures";
import { test as base } from "./fixtures";
import {
clickSessionRow,
openSessions,
openWorkspaceWithAgents,
type ArchiveTabAgent,
} from "./helpers/archive-tab";
import { seedWorkspace, type SeedDaemonClient, type SeededWorkspace } from "./helpers/seed-client";
const IMAGE_ALT = "Synthetic image";
const IMAGE_PREVIEW_ERROR = "Unable to load image preview.";
const IMAGE_PROMPT = "Emit 4096 byte image agent stream update";
appendSettledTimelineTurns,
createNearTenMegabyteAssistantPng,
createSettledMockAgent,
createSmallAssistantPng,
emitSettledAssistantImage,
expectAssistantImageNotMounted,
expectAssistantImageRendered,
openAssistantImageTimeline,
openExistingImageAgentTabs,
remountAndRecoverAssistantImageFromHistory,
sendFollowUpAndExpectVisibleResponse,
switchAwayAndBackWithoutImageInstability,
userPagesUntilAssistantImageRenders,
} from "./helpers/assistant-images";
import { seedWorkspace, type SeededWorkspace } from "./helpers/seed-client";
const test = base.extend<{ imageWorkspace: SeededWorkspace }>({
imageWorkspace: async ({ page: _page }, provide) => {
const workspace = await seedWorkspace({ repoPrefix: "agent-tab-image-stability-" });
await provide(workspace);
await workspace.cleanup();
try {
await provide(workspace);
} finally {
await workspace.cleanup();
}
},
});
async function createMockAgent(
client: SeedDaemonClient,
input: { cwd: string; workspaceId: string; title: string },
): Promise<ArchiveTabAgent> {
const agent = await client.createAgent({
provider: "mock",
model: "ten-second-stream",
modeId: "load-test",
cwd: input.cwd,
workspaceId: input.workspaceId,
title: input.title,
});
await client.waitForAgentUpsert(agent.id, (snapshot) => snapshot.status === "idle", 30_000);
return { id: agent.id, ...input };
}
async function observeImagePreviewInstability(page: Page, imageAgentId: string): Promise<void> {
await page.evaluate(
({ alt, errorText, tabTestId }) => {
const root = document.documentElement;
root.dataset.agentTabImageErrorFlash = "false";
root.dataset.agentTabImageMissingFlash = "false";
const isVisible = (element: Element) => {
const rect = element.getBoundingClientRect();
return rect.width > 0 && rect.height > 0;
};
const inspect = () => {
const errorVisible = Array.from(document.querySelectorAll("*")).some((element) => {
if (element.children.length > 0 || element.textContent?.trim() !== errorText) {
return false;
}
return isVisible(element);
});
if (errorVisible) {
root.dataset.agentTabImageErrorFlash = "true";
}
const imageTab = document.querySelector(`[data-testid="${tabTestId}"]`);
if (imageTab?.getAttribute("aria-selected") !== "true") {
return;
}
const imageVisible = Array.from(document.querySelectorAll('[role="img"]')).some(
(element) => element.getAttribute("aria-label") === alt && isVisible(element),
);
if (!imageVisible) {
root.dataset.agentTabImageMissingFlash = "true";
}
};
const observer = new MutationObserver(inspect);
observer.observe(document.body, {
attributes: true,
childList: true,
subtree: true,
characterData: true,
});
window.setTimeout(() => observer.disconnect(), 5_000);
},
{
alt: IMAGE_ALT,
errorText: IMAGE_PREVIEW_ERROR,
tabTestId: `workspace-tab-agent_${imageAgentId}`,
},
);
}
async function expectNoObservedImagePreviewInstability(page: Page): Promise<void> {
await page.waitForTimeout(250);
expect(await page.locator("html").getAttribute("data-agent-tab-image-error-flash")).toBe("false");
expect(await page.locator("html").getAttribute("data-agent-tab-image-missing-flash")).toBe(
"false",
);
}
async function switchToSettledAgentTab(page: Page, agent: ArchiveTabAgent): Promise<void> {
const tab = page.getByRole("button", { name: agent.title, exact: true });
await tab.click();
await expect(page).toHaveTitle(agent.title);
await expect(page.locator('[data-testid="agent-chat-scroll"]:visible').first()).toBeVisible({
timeout: 30_000,
});
await expect(tab).toHaveAttribute("aria-selected", "true");
}
test("switching between settled agent tabs keeps the image timeline valid", async ({
test("switching between settled agent tabs keeps a real assistant PNG rendered", async ({
imageWorkspace: workspace,
page,
}) => {
test.setTimeout(120_000);
const imageAgent = await createMockAgent(workspace.client, {
cwd: workspace.repoPath,
workspaceId: workspace.workspaceId,
title: `Image timeline ${Date.now()}`,
const image = await createSmallAssistantPng(workspace, {
alt: "Real file image",
fileName: "assistant-preview.png",
});
const otherAgent = await createMockAgent(workspace.client, {
cwd: workspace.repoPath,
workspaceId: workspace.workspaceId,
title: `Other timeline ${Date.now()}`,
});
await workspace.client.sendAgentMessage(imageAgent.id, IMAGE_PROMPT);
await workspace.client.waitForFinish(imageAgent.id, 15_000);
await openWorkspaceWithAgents(page, [otherAgent, imageAgent]);
await expect(page.getByRole("img", { name: IMAGE_ALT })).toBeVisible({ timeout: 30_000 });
const imageAgent = await createSettledMockAgent(workspace, "Image timeline");
const otherAgent = await createSettledMockAgent(workspace, "Other timeline");
await emitSettledAssistantImage(workspace.client, imageAgent, image);
await switchToSettledAgentTab(page, otherAgent);
await observeImagePreviewInstability(page, imageAgent.id);
await switchToSettledAgentTab(page, imageAgent);
await expect(page.getByRole("img", { name: IMAGE_ALT })).toBeVisible({ timeout: 30_000 });
await expectNoObservedImagePreviewInstability(page);
await openExistingImageAgentTabs(page, { imageAgent, otherAgent });
await expectAssistantImageRendered(page, image);
await switchAwayAndBackWithoutImageInstability(page, { image, imageAgent, otherAgent });
});
test("reopening an image timeline after editing another draft keeps its image available", async ({
test("a real assistant PNG remains reachable through pagination and remount", async ({
imageWorkspace: workspace,
page,
}) => {
test.setTimeout(120_000);
const imageAgent = await createMockAgent(workspace.client, {
cwd: workspace.repoPath,
workspaceId: workspace.workspaceId,
title: `Image cleanup ${Date.now()}`,
const image = await createSmallAssistantPng(workspace, {
alt: "Paginated real file image",
fileName: "paginated-assistant-preview.png",
});
const otherAgent = await createMockAgent(workspace.client, {
cwd: workspace.repoPath,
workspaceId: workspace.workspaceId,
title: `Draft cleanup ${Date.now()}`,
});
await workspace.client.sendAgentMessage(imageAgent.id, IMAGE_PROMPT);
await workspace.client.waitForFinish(imageAgent.id, 15_000);
await openWorkspaceWithAgents(page, [otherAgent, imageAgent]);
await expect(page.getByRole("img", { name: IMAGE_ALT })).toBeVisible({ timeout: 30_000 });
const imageAgent = await createSettledMockAgent(workspace, "Paginated image timeline");
await emitSettledAssistantImage(workspace.client, imageAgent, image);
await appendSettledTimelineTurns(workspace.client, imageAgent, 40);
await page.getByRole("button", { name: otherAgent.title, exact: true }).click();
await page
.getByRole("textbox", { name: "Message agent..." })
.fill("Keep this draft while I check history");
await openSessions(page);
await observeImagePreviewInstability(page, imageAgent.id);
await clickSessionRow(page, imageAgent.title);
await openAssistantImageTimeline(page, imageAgent);
await expectAssistantImageNotMounted(page, image);
await userPagesUntilAssistantImageRenders(page, image);
await remountAndRecoverAssistantImageFromHistory(page, image);
});
await expect(page.getByRole("img", { name: IMAGE_ALT })).toBeVisible({ timeout: 30_000 });
await expectNoObservedImagePreviewInstability(page);
test("a near-10 MiB real assistant PNG renders and the app remains responsive", async ({
imageWorkspace: workspace,
page,
}) => {
test.setTimeout(180_000);
const image = await createNearTenMegabyteAssistantPng(workspace, {
alt: "Large real file image",
fileName: "large-assistant-preview.png",
});
const imageAgent = await createSettledMockAgent(workspace, "Large image timeline");
await emitSettledAssistantImage(workspace.client, imageAgent, image);
await openAssistantImageTimeline(page, imageAgent);
await expectAssistantImageRendered(page, image);
await sendFollowUpAndExpectVisibleResponse(page, {
prompt: "confirm responsiveness: emit 1 coalesced agent stream updates",
response: "stress-update-0",
});
});
@@ -0,0 +1,343 @@
import { writeFile } from "node:fs/promises";
import path from "node:path";
import { deflateSync } from "node:zlib";
import { expect, type Page } from "@playwright/test";
import type { ArchiveTabAgent } from "./archive-tab";
import { openWorkspaceWithAgents } from "./archive-tab";
import { submitMessage } from "./composer";
import type { SeedDaemonClient, SeededWorkspace } from "./seed-client";
import { openAgentRoute } from "./mock-agent";
import { rememberTimelineViewport, userScrollsTimelineToHistoryStart } from "./timeline-pagination";
const IMAGE_PREVIEW_ERROR = "Unable to load image preview.";
const SMALL_PNG = Buffer.from(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==",
"base64",
);
const TEN_MEBIBYTES = 10 * 1024 * 1024;
export interface AssistantImageFixture {
alt: string;
height: number;
relativePath: string;
size: number;
width: number;
}
export async function createSmallAssistantPng(
workspace: SeededWorkspace,
input: { alt: string; fileName: string },
): Promise<AssistantImageFixture> {
await writeFile(path.join(workspace.repoPath, input.fileName), SMALL_PNG);
return {
alt: input.alt,
height: 1,
relativePath: input.fileName,
size: SMALL_PNG.byteLength,
width: 1,
};
}
export async function createNearTenMegabyteAssistantPng(
workspace: SeededWorkspace,
input: { alt: string; fileName: string },
): Promise<AssistantImageFixture> {
const width = 2_048;
const height = 1_280;
const bytes = encodeDeterministicPng(width, height);
if (bytes.byteLength < TEN_MEBIBYTES * 0.95 || bytes.byteLength > TEN_MEBIBYTES * 1.05) {
throw new Error(`Expected a PNG near 10 MiB, encoded ${bytes.byteLength} bytes`);
}
await writeFile(path.join(workspace.repoPath, input.fileName), bytes);
return {
alt: input.alt,
height,
relativePath: input.fileName,
size: bytes.byteLength,
width,
};
}
export async function createSettledMockAgent(
workspace: SeededWorkspace,
title: string,
): Promise<ArchiveTabAgent> {
const agent = await workspace.client.createAgent({
provider: "mock",
model: "ten-second-stream",
modeId: "load-test",
cwd: workspace.repoPath,
workspaceId: workspace.workspaceId,
title,
});
await workspace.client.waitForAgentUpsert(
agent.id,
(snapshot) => snapshot.status === "idle",
30_000,
);
return {
id: agent.id,
title,
cwd: workspace.repoPath,
workspaceId: workspace.workspaceId,
};
}
export async function emitSettledAssistantImage(
client: SeedDaemonClient,
agent: ArchiveTabAgent,
image: AssistantImageFixture,
): Promise<void> {
await client.sendAgentMessage(
agent.id,
`Emit settled assistant image Markdown: ![${image.alt}](${image.relativePath})`,
);
const result = await client.waitForFinish(agent.id, 30_000);
if (result.status !== "idle" || result.final?.lastError) {
throw new Error(
`Assistant image agent did not settle: ${result.final?.lastError ?? result.status}`,
);
}
}
export async function appendSettledTimelineTurns(
client: SeedDaemonClient,
agent: ArchiveTabAgent,
count: number,
): Promise<void> {
for (let index = 0; index < count; index += 1) {
await client.sendAgentMessage(
agent.id,
`image-history-turn-${index}: emit 1 coalesced agent stream updates`,
);
const result = await client.waitForFinish(agent.id, 30_000);
if (result.status !== "idle" || result.final?.lastError) {
throw new Error(
`Assistant image history turn did not settle: ${result.final?.lastError ?? result.status}`,
);
}
}
}
export async function expectAssistantImageRendered(
page: Page,
image: AssistantImageFixture,
): Promise<void> {
const rendered = page.getByRole("img", { name: image.alt }).first();
await expect(rendered).toBeVisible({ timeout: 30_000 });
await expect
.poll(async () =>
rendered.evaluate((element) => {
const imageElement =
element instanceof HTMLImageElement ? element : element.querySelector("img");
return imageElement?.complete
? { height: imageElement.naturalHeight, width: imageElement.naturalWidth }
: null;
}),
)
.toEqual({ height: image.height, width: image.width });
}
export async function switchAwayAndBackWithoutImageInstability(
page: Page,
input: {
image: AssistantImageFixture;
imageAgent: ArchiveTabAgent;
otherAgent: ArchiveTabAgent;
},
): Promise<void> {
await beginVisibleImageStabilityObservation(page, input.image.alt, input.imageAgent.id);
await selectSettledAgentTab(page, input.otherAgent);
await selectSettledAgentTab(page, input.imageAgent);
await expectAssistantImageRendered(page, input.image);
await expectNoVisibleImageInstability(page);
}
export async function openExistingImageAgentTabs(
page: Page,
input: { imageAgent: ArchiveTabAgent; otherAgent: ArchiveTabAgent },
): Promise<void> {
await openWorkspaceWithAgents(page, [input.otherAgent, input.imageAgent]);
}
export async function openAssistantImageTimeline(
page: Page,
agent: ArchiveTabAgent,
): Promise<void> {
await openAgentRoute(page, { workspaceId: agent.workspaceId, agentId: agent.id });
}
export async function expectAssistantImageNotMounted(
page: Page,
image: AssistantImageFixture,
): Promise<void> {
await expect(page.getByRole("img", { name: image.alt })).toHaveCount(0);
}
export async function userPagesUntilAssistantImageRenders(
page: Page,
image: AssistantImageFixture,
): Promise<void> {
const rendered = page.getByRole("img", { name: image.alt });
for (let attempt = 0; attempt < 10; attempt += 1) {
if ((await rendered.count()) > 0) {
await expectAssistantImageRendered(page, image);
return;
}
const previous = await rememberTimelineViewport(page);
await userScrollsTimelineToHistoryStart(page);
await expect
.poll(async () => (await rememberTimelineViewport(page)).scrollHeight)
.toBeGreaterThan(previous.scrollHeight);
}
await expectAssistantImageRendered(page, image);
}
export async function remountAndRecoverAssistantImageFromHistory(
page: Page,
image: AssistantImageFixture,
): Promise<void> {
await page.reload();
await userPagesUntilAssistantImageRenders(page, image);
}
export async function sendFollowUpAndExpectVisibleResponse(
page: Page,
input: { prompt: string; response: string },
): Promise<void> {
await submitMessage(page, input.prompt);
await expect(page.getByText(input.prompt, { exact: true })).toBeVisible({ timeout: 30_000 });
await expect(page.getByText(input.response, { exact: true })).toBeVisible({ timeout: 30_000 });
}
async function selectSettledAgentTab(page: Page, agent: ArchiveTabAgent): Promise<void> {
const tab = page.getByRole("button", { name: agent.title, exact: true });
await tab.click();
await expect(page).toHaveTitle(agent.title);
await expect(tab).toHaveAttribute("aria-selected", "true");
await expect(page.locator('[data-testid="agent-chat-scroll"]:visible').first()).toBeVisible({
timeout: 30_000,
});
}
async function beginVisibleImageStabilityObservation(
page: Page,
alt: string,
imageAgentId: string,
): Promise<void> {
await page.evaluate(
({ accessibleName, errorText, tabTestId }) => {
const record = {
errorSeen: false,
missingSeen: false,
inspect: () => undefined,
observer: null as MutationObserver | null,
};
const isVisible = (element: Element) => {
const rect = element.getBoundingClientRect();
return rect.width > 0 && rect.height > 0;
};
record.inspect = () => {
record.errorSeen ||= Array.from(document.querySelectorAll("*")).some(
(element) =>
element.children.length === 0 &&
element.textContent?.trim() === errorText &&
isVisible(element),
);
const imageTab = document.querySelector(`[data-testid="${tabTestId}"]`);
if (imageTab?.getAttribute("aria-selected") !== "true") return;
const imageVisible = Array.from(document.querySelectorAll('[role="img"]')).some(
(element) => element.getAttribute("aria-label") === accessibleName && isVisible(element),
);
record.missingSeen ||= !imageVisible;
};
record.observer = new MutationObserver(record.inspect);
record.observer.observe(document.body, {
attributes: true,
childList: true,
subtree: true,
characterData: true,
});
record.inspect();
(
window as unknown as {
__paseoAssistantImageObservation?: typeof record;
}
).__paseoAssistantImageObservation = record;
},
{
accessibleName: alt,
errorText: IMAGE_PREVIEW_ERROR,
tabTestId: `workspace-tab-agent_${imageAgentId}`,
},
);
}
async function expectNoVisibleImageInstability(page: Page): Promise<void> {
const observation = await page.evaluate(() => {
const owner = window as unknown as {
__paseoAssistantImageObservation?: {
errorSeen: boolean;
missingSeen: boolean;
inspect(): void;
observer: MutationObserver | null;
};
};
const record = owner.__paseoAssistantImageObservation;
if (!record) throw new Error("Assistant image stability observation was not started");
record.inspect();
record.observer?.disconnect();
delete owner.__paseoAssistantImageObservation;
return { errorSeen: record.errorSeen, missingSeen: record.missingSeen };
});
expect(observation).toEqual({ errorSeen: false, missingSeen: false });
}
function encodeDeterministicPng(width: number, height: number): Buffer {
const stride = width * 4 + 1;
const scanlines = Buffer.allocUnsafe(stride * height);
let state = 0x9e3779b9;
for (let row = 0; row < height; row += 1) {
const rowOffset = row * stride;
scanlines[rowOffset] = 0;
for (let offset = 1; offset < stride; offset += 1) {
state ^= state << 13;
state ^= state >>> 17;
state ^= state << 5;
scanlines[rowOffset + offset] = state & 0xff;
}
}
const header = Buffer.alloc(13);
header.writeUInt32BE(width, 0);
header.writeUInt32BE(height, 4);
header[8] = 8;
header[9] = 6;
return Buffer.concat([
Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]),
pngChunk("IHDR", header),
pngChunk("IDAT", deflateSync(scanlines, { level: 0 })),
pngChunk("IEND", Buffer.alloc(0)),
]);
}
function pngChunk(type: string, data: Buffer): Buffer {
const typeBytes = Buffer.from(type, "ascii");
const chunk = Buffer.allocUnsafe(12 + data.byteLength);
chunk.writeUInt32BE(data.byteLength, 0);
typeBytes.copy(chunk, 4);
data.copy(chunk, 8);
chunk.writeUInt32BE(crc32(Buffer.concat([typeBytes, data])), 8 + data.byteLength);
return chunk;
}
function crc32(bytes: Buffer): number {
let crc = 0xffffffff;
for (const byte of bytes) {
crc ^= byte;
for (let bit = 0; bit < 8; bit += 1) {
crc = (crc >>> 1) ^ (crc & 1 ? 0xedb88320 : 0);
}
}
return (crc ^ 0xffffffff) >>> 0;
}
@@ -284,6 +284,39 @@ describe("MockLoadTestAgentClient", () => {
unsubscribe();
});
test("emits a settled assistant Markdown image path selected by prompt", async () => {
vi.useFakeTimers();
const client = new MockLoadTestAgentClient();
const session = await client.createSession({
provider: "mock",
cwd: process.cwd(),
model: "ten-second-stream",
});
const events: AgentStreamEvent[] = [];
const unsubscribe = session.subscribe((event) => events.push(event));
const markdown = "![Fixture image](screenshots/fixture.png)";
const resultPromise = session.run(`Emit settled assistant image Markdown: ${markdown}`);
await vi.advanceTimersByTimeAsync(0);
expect(
events.flatMap((event): AgentTimelineItem[] =>
event.type === "timeline" && event.item.type === "assistant_message" ? [event.item] : [],
),
).toEqual([
expect.objectContaining({
type: "assistant_message",
text: markdown,
}),
]);
await expect(resultPromise).resolves.toMatchObject({
sessionId: session.id,
finalText: markdown,
canceled: false,
});
unsubscribe();
});
test("agent manager coalesces adjacent assistant tokens into fewer messages", async () => {
vi.useFakeTimers();
const workdir = mkdtempSync(join(tmpdir(), "paseo-mock-load-test-"));
@@ -158,6 +158,13 @@ function shouldEmitTurnFailure(prompt: AgentPromptInput): boolean {
return /emit\s+(?:a\s+)?synthetic\s+turn\s+failure/i.test(promptToText(prompt));
}
function parseSettledAssistantImageMarkdown(prompt: AgentPromptInput): string | null {
const match = /^emit settled assistant image markdown:\s*(!\[[^\]\r\n]*\]\(.+\))\s*$/i.exec(
promptToText(prompt),
);
return match?.[1] ?? null;
}
function parseMockQuestionPrompt(prompt: AgentPromptInput): MockQuestionPromptRequest | null {
const text = promptToText(prompt);
if (!/emit\s+(?:a\s+)?synthetic\s+questions?/i.test(text)) {
@@ -671,10 +678,13 @@ export class MockLoadTestAgentSession implements AgentSession {
const stress = parseAgentStreamStressPrompt(prompt);
const questionPrompt = parseMockQuestionPrompt(prompt);
const structuredBranchName = parseStructuredBranchNamePrompt(prompt);
const settledAssistantImageMarkdown = parseSettledAssistantImageMarkdown(prompt);
if (shouldEmitTurnFailure(prompt)) {
this.scheduleFailedTurn(turn);
} else if (structuredBranchName) {
this.scheduleStructuredJsonTurn(turn, structuredBranchName);
this.scheduleSettledAssistantTurn(turn, JSON.stringify(structuredBranchName));
} else if (settledAssistantImageMarkdown) {
this.scheduleSettledAssistantTurn(turn, settledAssistantImageMarkdown);
} else if (shouldEmitPlanApprovalPrompt(prompt)) {
this.schedulePlanApprovalTurn(turn);
} else if (questionPrompt) {
@@ -889,14 +899,14 @@ export class MockLoadTestAgentSession implements AgentSession {
turn.timer.unref?.();
}
private scheduleStructuredJsonTurn(turn: ActiveTurn, result: Record<string, string>): void {
private scheduleSettledAssistantTurn(turn: ActiveTurn, finalText: string): void {
turn.timer = setTimeout(() => {
this.emitStructuredJsonTurn(turn, result);
this.emitSettledAssistantTurn(turn, finalText);
}, 0);
turn.timer.unref?.();
}
private emitStructuredJsonTurn(turn: ActiveTurn, result: Record<string, string>): void {
private emitSettledAssistantTurn(turn: ActiveTurn, finalText: string): void {
if (this.activeTurn !== turn) {
return;
}
@@ -908,7 +918,6 @@ export class MockLoadTestAgentSession implements AgentSession {
turnId: turn.turnId,
});
const finalText = JSON.stringify(result);
this.emitTimeline(turn.turnId, {
type: "assistant_message",
text: finalText,