import { test, expect } from "../support/fixtures"; import { awaitAssistantMessage } from "../support/helpers/agent-stream"; import { startRunningMockAgent, submitMessage } from "../support/helpers/composer"; import { seedLongMockAgentTimeline, openAgentTimeline, scrollTimelineUntilOlderHistoryIsReachable, scrollTimelineToNewestLoadedEdge, expectTimelinePromptVisible, } from "../support/helpers/timeline-pagination"; import { sampleStreamFrames, summarizeStreamSmoothness, } from "../support/helpers/stream-smoothness"; /** * Streaming smoothness, measured on the path the user actually sees. * * The mock provider's "bursty-stream" model emits tokens in uneven runs separated * by idle gaps, which is how real models arrive. Sampling and the statistics live * in ../support/helpers/stream-smoothness.ts, which also documents what the two * numbers mean. * * Budgets are deliberately loose. This spec exists to catch a regression in the * shape of the stream, not to pin exact numbers on shared CI hardware. For * reference, on a local dev daemon the paced reveal measures a CV around 1.3–1.5 * against 2.7 when painting deltas as they arrive. */ const SAMPLE_WINDOW_MS = 6_000; const CHARS_PER_FRAME_CV_BUDGET = 2; const UPDATE_GAP_P95_BUDGET_MS = 250; const REACT_COMMIT_P95_BUDGET_MS = 12; const REACT_COMMIT_COUNT_BUDGET = 600; const RUN_AGENT_STREAM_PERF = process.env.PASEO_AGENT_STREAM_PERF_E2E === "1"; const agentStreamPerfDescribe = RUN_AGENT_STREAM_PERF ? test.describe : test.describe.skip; interface ReactCommit { id: string; phase: "mount" | "update" | "nested-update"; actualDuration: number; baseDuration: number; startTime: number; commitTime: number; } agentStreamPerfDescribe("Agent stream smoothness", () => { test("reveals bursty model output at a steady rate", async ({ page }, testInfo) => { test.setTimeout(120_000); await page.addInitScript(() => { Reflect.set(globalThis, "__PASEO_RENDER_PROFILE_ENABLED__", true); }); const agent = await startRunningMockAgent(page, { prefix: "stream-smoothness-", model: "bursty-stream", prompt: "Stream bursty output for smoothness measurement.", }); try { await awaitAssistantMessage(page); await page.evaluate(() => { const reset = Reflect.get(globalThis, "__PASEO_RESET_RENDER_PROFILE__"); if (typeof reset !== "function") { throw new Error("Render profiler did not initialize"); } reset(); }); const samples = await sampleStreamFrames(page, { sampleWindowMs: SAMPLE_WINDOW_MS }); const report = summarizeStreamSmoothness(samples, SAMPLE_WINDOW_MS); const reactCommits = await readStreamReactCommits(page); const reactReport = summarizeReactCommits(reactCommits); await testInfo.attach("agent-stream-smoothness-report", { body: JSON.stringify(report, null, 2), contentType: "application/json", }); await testInfo.attach("agent-stream-react-profile", { body: JSON.stringify(reactReport, null, 2), contentType: "application/json", }); console.log(`[perf] Agent stream React: ${JSON.stringify(reactReport)}`); console.log(`[perf] Agent stream smoothness: ${JSON.stringify(report)}`); expect( report.growthFrames, "the stream must actually produce text during the sample window", ).toBeGreaterThan(20); expect( report.charsPerFrameCv, `characters painted per frame should be steady (cv < ${CHARS_PER_FRAME_CV_BUDGET})`, ).toBeLessThan(CHARS_PER_FRAME_CV_BUDGET); expect( report.updateGapP95Ms, `the text should not stall (p95 gap < ${UPDATE_GAP_P95_BUDGET_MS}ms)`, ).toBeLessThan(UPDATE_GAP_P95_BUDGET_MS); expect(reactReport.count, "streaming should produce measured React commits").toBeGreaterThan( 20, ); expect( reactReport.count, `the reveal should cap commits below ${REACT_COMMIT_COUNT_BUDGET} per sample window`, ).toBeLessThan(REACT_COMMIT_COUNT_BUDGET); expect( reactReport.actualDurationP95Ms, `React commits should stay below ${REACT_COMMIT_P95_BUDGET_MS}ms at p95`, ).toBeLessThan(REACT_COMMIT_P95_BUDGET_MS); } finally { await agent.cleanup(); } }); test("keeps streaming work bounded after revealing older history", async ({ page }, testInfo) => { test.setTimeout(120_000); await page.addInitScript(() => { Reflect.set(globalThis, "__PASEO_RENDER_PROFILE_ENABLED__", true); }); const agent = await seedLongMockAgentTimeline({ turns: 80, liveTurns: "thirty-minute-stream" }); try { await openAgentTimeline(page, agent); await expect(page.getByRole("textbox", { name: "Message agent..." }).first()).toBeVisible(); await scrollTimelineUntilOlderHistoryIsReachable(page, agent.oldestPrompt); await expectTimelinePromptVisible(page, agent.oldestPrompt); await scrollTimelineToNewestLoadedEdge(page); await submitMessage(page, "Stream a long answer after reading older messages."); await awaitAssistantMessage(page); for (let sample = 0; sample < 3; sample += 1) { await page.evaluate(() => globalThis.__PASEO_RESET_RENDER_PROFILE__?.()); const frames = await sampleStreamFrames(page, { sampleWindowMs: SAMPLE_WINDOW_MS }); const smoothness = summarizeStreamSmoothness(frames, SAMPLE_WINDOW_MS); const react = summarizeReactCommits(await readStreamReactCommits(page)); await testInfo.attach(`revealed-history-stream-${sample}`, { body: JSON.stringify({ smoothness, react }, null, 2), contentType: "application/json", }); console.log(`[perf] Revealed history stream: ${JSON.stringify({ smoothness, react })}`); expect(smoothness.growthFrames).toBeGreaterThan(20); expect(react.count).toBeGreaterThan(20); expect(react.actualDurationP95Ms).toBeLessThan(REACT_COMMIT_P95_BUDGET_MS); } } finally { await agent.cleanup(); } }); }); async function readStreamReactCommits( page: Parameters[0], ): Promise { return await page.evaluate(() => { const samples = Reflect.get(globalThis, "__PASEO_RENDER_PROFILE__"); if (!Array.isArray(samples)) { return []; } return samples.filter( (sample): sample is ReactCommit => typeof sample === "object" && sample !== null && typeof Reflect.get(sample, "id") === "string" && Reflect.get(sample, "id").startsWith("AgentStreamSection:"), ); }); } function summarizeReactCommits(commits: readonly ReactCommit[]) { const durations = commits .map((commit) => commit.actualDuration) .sort((left, right) => left - right); const total = durations.reduce((sum, duration) => sum + duration, 0); return { count: commits.length, actualDurationTotalMs: round2(total), actualDurationMeanMs: round2(commits.length > 0 ? total / commits.length : 0), actualDurationP50Ms: percentile(durations, 50), actualDurationP95Ms: percentile(durations, 95), actualDurationMaxMs: durations.at(-1) ?? 0, }; } function round2(value: number): number { return Math.round(value * 100) / 100; } function percentile(sorted: readonly number[], percentileValue: number): number { if (sorted.length === 0) { return 0; } return sorted[ Math.min(sorted.length - 1, Math.floor((sorted.length - 1) * (percentileValue / 100))) ]; }