diff --git a/packages/server/src/server/agent/change-request-watches/index.ts b/packages/server/src/server/agent/change-request-watches/index.ts index 8ebe907c19..490b5f4157 100644 --- a/packages/server/src/server/agent/change-request-watches/index.ts +++ b/packages/server/src/server/agent/change-request-watches/index.ts @@ -353,6 +353,12 @@ export function createChangeRequestWatches(deps: Dependencies) { return Promise.all(activeDeliveries.values()).then(() => undefined); } + function stop(): void { + closed = true; + clearTimeout(timer); + for (const url of observations.keys()) stopObserving(url); + } + return { async start() { let restored: WatchGroup[]; @@ -407,11 +413,11 @@ export function createChangeRequestWatches(deps: Dependencies) { retries.delete(agentId); void flush(); }, + stop, async close() { - closed = true; + stop(); + // Providers have stopped. Any start still without an acknowledgment stays queued. shutdown.abort(); - clearTimeout(timer); - for (const url of observations.keys()) stopObserving(url); await Promise.all(activeDeliveries.values()); await mutations; }, diff --git a/packages/server/src/server/agent/change-request-watches/watches.test.ts b/packages/server/src/server/agent/change-request-watches/watches.test.ts index 0468b82381..8e0ed8f9a8 100644 --- a/packages/server/src/server/agent/change-request-watches/watches.test.ts +++ b/packages/server/src/server/agent/change-request-watches/watches.test.ts @@ -3,6 +3,7 @@ import { join } from "node:path"; import pino from "pino"; import { afterEach, expect, test, vi } from "vitest"; import { createChangeRequestWatches } from "./index.js"; +import { waitForAgentRunStart } from "../agent-prompt.js"; import type { ChangeRequestSnapshot, ChangeRequestSource, @@ -188,3 +189,56 @@ test("retains the current observation as the last subscriber leaves and a new on await rm(home, { recursive: true, force: true }); } }); + +test("records an admitted update acknowledged during provider shutdown", async () => { + const home = await createTestHome(); + vi.useFakeTimers(); + let observer: Parameters[1] | undefined; + const starting = Promise.withResolvers(); + const accepted = Promise.withResolvers(); + const deliveries: string[] = []; + const create = () => + createChangeRequestWatches({ + source: { + resolve: (input) => input, + observe(_address, callbacks) { + observer = callbacks; + queueMicrotask(() => void callbacks.snapshot(initial)); + return () => { + observer = undefined; + }; + }, + }, + paseoHome: home, + isActive: async () => true, + async deliver(_agent, digest) { + starting.resolve(); + await waitForAgentRunStart(accepted.promise, digest.signal); + deliveries.push(digest.id); + return true; + }, + logger: pino({ enabled: false }), + }); + let watches = create(); + try { + await watches.start(); + await watches.subscribe("first", url); + await observer!.snapshot({ ...initial, state: "merged" }); + await vi.advanceTimersByTimeAsync(30_000); + await starting.promise; + + // The daemon freezes observation and new admission, then tears down providers. + watches.stop(); + accepted.resolve(); + await watches.close(); + + watches = create(); + await watches.start(); + await watches.flush(); + expect(deliveries).toHaveLength(1); + } finally { + accepted.resolve(); + await watches.close(); + await rm(home, { recursive: true, force: true }); + } +}); diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 201a83ce4c..2ef5d656f5 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -2056,7 +2056,7 @@ export async function createPaseoDaemon( // Freeze both ingress and registration before taking the agent closure snapshot. wsServer?.prepareForShutdown(); unsubscribeChangeRequestDelivery(); - await changeRequestWatches.close(); + changeRequestWatches.stop(); await finishSubscriptions.close(); await wsServer?.agentMessages.close(); agentManager.prepareForShutdown(); @@ -2071,6 +2071,7 @@ export async function createPaseoDaemon( detachAgentStoragePersistence(); await agentStorage.flush().catch(() => undefined); await agentProviderRuntime.shutdown(); + await changeRequestWatches.close(); await pluginRuntime.stopAllPlugins(); try { if (keepTerminals) { diff --git a/packages/server/src/services/github-change-requests.ts b/packages/server/src/services/github-change-requests.ts index c0a52c53b1..239b63557a 100644 --- a/packages/server/src/services/github-change-requests.ts +++ b/packages/server/src/services/github-change-requests.ts @@ -126,7 +126,19 @@ interface PageRead { accept(value: unknown): void; } -type ReadBatch = (query: string) => Promise>; +interface ChangeRequestPageSelection { + url: string; + graphql: string; +} + +interface ReadOptions { + read(selections: ChangeRequestPageSelection[]): Promise>; + batchSize: number; + firstBatchSize: number; + onChecks(url: string, headSha: string, contexts: unknown[]): void; + knownActivity(url: string): ReadonlySet | undefined; + isObserved(url: string): boolean; +} const pullRequestResult = z .object({ pullRequest: z.unknown() }) @@ -138,10 +150,7 @@ const threadFields = `id ${connection("comments", inlineFields, undefined, 10)}` export async function readGitHubChangeRequests( urls: string[], - read: ReadBatch, - maxBatchSize: number, - onChecks: (url: string, headSha: string, contexts: unknown[]) => void, - knownActivity: (url: string) => ReadonlySet | undefined, + { read, batchSize, firstBatchSize, onChecks, knownActivity, isObserved }: ReadOptions, ): Promise> { const snapshots = new Map(); const failed = new Map(); @@ -294,17 +303,21 @@ export async function readGitHubChangeRequests( }); } + let pageSize = firstBatchSize; while (pending.length) { - const batch = pending.splice(0, maxBatchSize).filter(({ url }) => !failed.has(url)); + const batch = pending + .splice(0, pageSize) + .filter(({ url }) => !failed.has(url) && isObserved(url)); + pageSize = batchSize; if (!batch.length) continue; try { - const aliases = batch.map(({ selection }, index) => `request${index}: ${selection}`); - const data = await read(`query PaseoWatchedChangeRequests { - rateLimit { limit remaining resetAt } - ${aliases.join("\n")} - }`); + const selections = batch.map(({ url, selection }, index) => ({ + url, + graphql: `request${index}: ${selection}`, + })); + const data = await read(selections); for (const [index, page] of batch.entries()) { - if (failed.has(page.url)) continue; + if (failed.has(page.url) || !isObserved(page.url)) continue; try { page.accept(data[`request${index}`]); } catch (cause) { @@ -323,5 +336,7 @@ export async function readGitHubChangeRequests( } } - return new Map(urls.map((url) => [url, failed.get(url) ?? snapshots.get(url)!])); + return new Map( + urls.filter(isObserved).map((url) => [url, failed.get(url) ?? snapshots.get(url)!]), + ); } diff --git a/packages/server/src/services/github-service.test.ts b/packages/server/src/services/github-service.test.ts index 5d83987f19..8254ea5ab3 100644 --- a/packages/server/src/services/github-service.test.ts +++ b/packages/server/src/services/github-service.test.ts @@ -5182,7 +5182,7 @@ describe("exact GitHub pull request observations", () => { }); }); -it("reuses an exact watch's check rollup for a co-due workspace on the same head", async () => { +it("batches co-due workspace discovery and exact PR reads with one check rollup", async () => { vi.useFakeTimers(); let now = 0; const pageInfo = { hasNextPage: false, endCursor: null }; @@ -5202,8 +5202,6 @@ it("reuses an exact watch's check rollup for a co-due workspace on the same head state: "OPEN", }), ]), - }), - batchPollStatusJson({ rateLimit: { limit: 5000, remaining: 4900, @@ -5250,7 +5248,11 @@ it("reuses an exact watch's check rollup for a co-due workspace on the same head now = 25; await vi.advanceTimersByTimeAsync(25); expect(errors).toEqual([]); - expect(runner.calls).toHaveLength(2); + expect(runner.calls).toHaveLength(1); + expect(runner.calls[0]?.args.at(-1)).toMatch(/query PaseoMixedPullRequestPoll/); + expect(runner.calls[0]?.args.at(-1)).toContain('pullRequests(headRefName: "feature"'); + expect(runner.calls[0]?.args.at(-1)).toContain("pullRequest(number: 41)"); + expect(runner.calls[0]?.args.at(-1)?.match(/statusCheckRollup/g)).toHaveLength(1); expect(snapshots).toEqual([expect.objectContaining({ headSha: "abc", checksState: "passed" })]); expect(statuses).toEqual([expect.objectContaining({ number: 41, checksStatus: "success" })]); } finally { @@ -5258,3 +5260,52 @@ it("reuses an exact watch's check rollup for a co-due workspace on the same head vi.useRealTimers(); } }); + +it("skips pagination after the last exact observer releases its PR", async () => { + vi.useFakeTimers(); + const runner = createDeferredRunner(); + const service = createGitHubService({ runner: runner.runner, resolveGhPath: async () => "gh" }); + const snapshots: unknown[] = []; + const errors: unknown[] = []; + try { + const release = service.changeRequests.observe("https://github.com/acme/widgets/pull/41", { + async snapshot(value) { + snapshots.push(value); + }, + error(error) { + errors.push(error); + }, + }); + await vi.advanceTimersByTimeAsync(25); + expect(runner.calls).toHaveLength(1); + const empty = { nodes: [], pageInfo: { hasNextPage: false, endCursor: null } }; + release(); + runner.resolveNext( + batchPollStatusJson({ + rateLimit: { + limit: 5000, + remaining: 4900, + resetAt: new Date(Date.now() + 100_000).toISOString(), + }, + request0: { + pullRequest: { + state: "OPEN", + headRefOid: "abc", + statusCheckRollup: null, + reviewDecision: null, + comments: { nodes: [], pageInfo: { hasNextPage: true, endCursor: "more-comments" } }, + reviews: empty, + reviewThreads: empty, + }, + }, + }), + ); + await vi.advanceTimersByTimeAsync(60_000); + expect(runner.calls).toHaveLength(1); + expect(snapshots).toEqual([]); + expect(errors).toEqual([]); + } finally { + service.dispose?.(); + vi.useRealTimers(); + } +}); diff --git a/packages/server/src/services/github-service.ts b/packages/server/src/services/github-service.ts index 0510c4fe5e..ecaa47f076 100644 --- a/packages/server/src/services/github-service.ts +++ b/packages/server/src/services/github-service.ts @@ -792,7 +792,7 @@ function graphqlString(value: string): string { return JSON.stringify(value); } -function buildBatchPullRequestStatusQuery(entries: GitHubBatchPollEntry[]): string { +function buildBatchPullRequestStatusSelections(entries: GitHubBatchPollEntry[]): string[] { const aliases = entries.map( (entry, index) => ` ${batchAlias(index)}: repository(owner: ${graphqlString( entry.owner, @@ -837,14 +837,32 @@ function buildBatchPullRequestStatusQuery(entries: GitHubBatchPollEntry[]): stri }`, ], ); - return `query PaseoBatchPullRequestStatus { - rateLimit { - limit - remaining - resetAt - } -${[...aliases, ...originAliases].join("\n")} -}\n${BATCH_PR_STATUS_FRAGMENT}`; + return [...aliases, ...originAliases]; +} + +interface GitHubPollQuery { + name: string; + cwd: string; + targetCount: number; + selections(): string[]; + fragments?: string; +} + +interface GitHubPollResponse { + aliases: Record; + stdout: string; + rateLimit: z.infer | null; + paused: boolean; +} + +type GitHubPollReader = (query: GitHubPollQuery) => Promise; + +function renderGitHubPollQuery(name: string, selections: string[], fragments = ""): string { + return `query ${name} { + rateLimit { limit remaining resetAt } +${selections.join("\n")} +} +${fragments}`; } function buildBatchPullRequestChecksQuery( @@ -1565,44 +1583,139 @@ export function createGitHubService(options: CreateGitHubServiceOptions = {}): G batchGroups.set(key, group); } + const hosts = new Set([...batchGroups.keys(), ...exactGroups.keys()]); await Promise.all([ ...legacyTargets.map((target) => runGitHubPoll(target)), - ...[...batchGroups.values()].map((group) => runGitHubPollGroup(group, now)), - ...[...exactGroups].map(([host, urls]) => runWatchedChangeRequests(host, urls, now)), + ...[...hosts].map((host) => + runGitHubPollWave(host, batchGroups.get(host) ?? [], exactGroups.get(host) ?? [], now), + ), ]); } + async function runGitHubPollWave( + host: string, + entries: GitHubBatchPollEntry[], + urls: string[], + startedAt: number, + ): Promise { + const read = createCoDueGitHubPollReader(host); + if (urls.length && host !== "github.com" && !authenticatedWatchHosts.has(host)) { + const authenticating = urls.map((url) => [url, watchedRequests.get(url)] as const); + try { + await run( + ["auth", "status", "--hostname", host], + { cwd: process.cwd(), envOverlay: watchAuthProbeEnv }, + host, + ); + authenticatedWatchHosts.add(host); + urls = authenticating.flatMap(([url, watch]) => + watch && watchedRequests.get(url) === watch ? [url] : [], + ); + } catch (error) { + for (const [url, watch] of authenticating) { + if (!watch || watchedRequests.get(url) !== watch) continue; + watch.callbacks.error(error); + watch.nextDueAt = alignGitHubPollTime(deps.now() + watchPollMs); + } + armGitHubPollTimer(); + urls = []; + } + } + const workspaceSlots = Math.min( + entries.length, + GITHUB_POLL_BATCH_MAX - Math.min(urls.length, Math.floor(GITHUB_POLL_BATCH_MAX / 2)), + ); + const exactSlots = GITHUB_POLL_BATCH_MAX - workspaceSlots; + await Promise.all([ + runGitHubPollGroup(entries, startedAt, read, workspaceSlots), + runWatchedChangeRequests(host, urls, startedAt, read, exactSlots), + ]); + } + + function createCoDueGitHubPollReader(host: string): GitHubPollReader { + interface PendingQuery { + query: GitHubPollQuery; + resolve(response: GitHubPollResponse): void; + reject(error: unknown): void; + } + let pending: PendingQuery[] = []; + + function issue(batch: PendingQuery[]): void { + const cwd = batch.length === 1 ? batch[0]!.query.cwd : process.cwd(); + const response = runPollGraphql( + host, + () => { + const current = batch.flatMap(({ query }) => { + const selections = query.selections(); + return selections.length ? [{ query, selections }] : []; + }); + if (!current.length) throw new Error("GitHub poll targets were released."); + const name = current.length === 1 ? current[0]!.query.name : "PaseoMixedPullRequestPoll"; + const selections = current.flatMap((item) => item.selections); + const fragments = current.flatMap(({ query }) => query.fragments ?? []).join("\n"); + const graphql = renderGitHubPollQuery(name, selections, fragments); + return run(["api", "graphql", "-f", `query=${graphql}`], { cwd }, host); + }, + () => run(["api", "rate_limit"], { cwd }, host), + ); + void response.then( + (result) => { + for (const task of batch) task.resolve(result); + }, + (error) => { + for (const task of batch) task.reject(error); + }, + ); + } + + return (query) => + new Promise((resolve, reject) => { + if (!pending.length) { + queueMicrotask(() => { + const due = pending; + pending = []; + const targetCount = due.reduce((count, task) => count + task.query.targetCount, 0); + if (targetCount <= GITHUB_POLL_BATCH_MAX) issue(due); + else for (const task of due) issue([task]); + }); + } + pending.push({ query, resolve, reject }); + }); + } + async function runWatchedChangeRequests( host: string, urls: string[], startedAt: number, + read: GitHubPollReader, + firstBatchSize: number, ): Promise { + if (!urls.length) return; const checks = new Map void>(); - const watching = urls.map((url) => { - const watch = watchedRequests.get(url)!; - const result = new Promise((resolve) => checks.set(url, resolve)); - watch.checks = { startedAt, result }; - return { url, watch }; - }); + const watching = new Map( + urls.flatMap((url) => { + const watch = watchedRequests.get(url); + if (!watch) return []; + const result = new Promise((resolve) => + checks.set(url, resolve), + ); + watch.checks = { startedAt, result }; + return [[url, watch] as const]; + }), + ); + const isObserved = (url: string) => + watching.has(url) && watchedRequests.get(url) === watching.get(url); const cwd = process.cwd(); try { - if (host !== "github.com" && !authenticatedWatchHosts.has(host)) { - await run( - ["auth", "status", "--hostname", host], - { cwd, envOverlay: watchAuthProbeEnv }, - host, - ); - authenticatedWatchHosts.add(host); - } - const results = await readGitHubChangeRequests( - urls, - async (query) => { - const args = ["api", "graphql", "--hostname", host, "-f", `query=${query}`]; - const response = await runPollGraphql( - host, - () => run(args, { cwd }, host), - () => run(["api", "rate_limit", "--hostname", host], { cwd }, host), - ); + const results = await readGitHubChangeRequests(urls, { + async read(selections) { + const response = await read({ + name: "PaseoWatchedChangeRequests", + cwd, + targetCount: selections.length, + selections: () => + selections.filter(({ url }) => isObserved(url)).map(({ graphql }) => graphql), + }); if (!response.rateLimit) throw new Error("GitHub returned no rate limit for the polling request."); const data = { ...response.aliases }; @@ -1618,14 +1731,16 @@ export function createGitHubService(options: CreateGitHubServiceOptions = {}): G } return data; }, - GITHUB_POLL_BATCH_MAX, - (url, headSha, contexts) => checks.get(url)!({ headSha, contexts }), - (url) => watchedRequests.get(url)?.callbacks.knownActivity?.(), - ); + batchSize: GITHUB_POLL_BATCH_MAX, + onChecks: (url, headSha, contexts) => checks.get(url)!({ headSha, contexts }), + knownActivity: (url) => watching.get(url)?.callbacks.knownActivity?.(), + firstBatchSize, + isObserved, + }); for (const complete of checks.values()) complete(); await Promise.all( - watching.map(async ({ url, watch }) => { - if (watchedRequests.get(url) !== watch) return; + [...watching].map(async ([url, watch]) => { + if (!isObserved(url)) return; const result = results.get(url)!; if (result instanceof Error) watch.callbacks.error(result); else await watch.callbacks.snapshot(result); @@ -1633,12 +1748,12 @@ export function createGitHubService(options: CreateGitHubServiceOptions = {}): G ); } catch (error) { if (error instanceof GitHubAuthenticationError) authenticatedWatchHosts.delete(host); - for (const { url, watch } of watching) { - if (watchedRequests.get(url) === watch) watch.callbacks.error(error); + for (const [url, watch] of watching) { + if (isObserved(url)) watch.callbacks.error(error); } } finally { for (const complete of checks.values()) complete(); - for (const { watch } of watching) + for (const watch of watching.values()) watch.nextDueAt = alignGitHubPollTime(deps.now() + watchPollMs); armGitHubPollTimer(); } @@ -1647,13 +1762,19 @@ export function createGitHubService(options: CreateGitHubServiceOptions = {}): G async function runGitHubPollGroup( entries: GitHubBatchPollEntry[], startedAt: number, + read: GitHubPollReader, + firstBatchSize: number, ): Promise { - for (let start = 0; start < entries.length; start += GITHUB_POLL_BATCH_MAX) { - const chunk = entries.slice(start, start + GITHUB_POLL_BATCH_MAX); + let start = 0; + let size = firstBatchSize; + while (start < entries.length) { + const chunk = entries.slice(start, start + size); + start += size; + size = GITHUB_POLL_BATCH_MAX; if ((pollPausedUntilByHost.get(chunk[0]?.host ?? "") ?? 0) > deps.now()) { return; } - await runGitHubPollBatch(chunk, startedAt); + await runGitHubPollBatch(chunk, startedAt, read); } } @@ -1774,15 +1895,28 @@ export function createGitHubService(options: CreateGitHubServiceOptions = {}): G async function runGitHubPollBatch( entries: GitHubBatchPollEntry[], startedAt: number, + read: GitHubPollReader, ): Promise { const addressedEntries = entries.map(addressGitHubBatchEntry); const firstEntry = addressedEntries[0]; if (!firstEntry) { return; } - const query = buildBatchPullRequestStatusQuery(addressedEntries); - const args = ["api", "graphql", "-f", `query=${query}`]; - const response = await runWorkspacePollGraphql(firstEntry, args).catch((error) => { + const selections = buildBatchPullRequestStatusSelections(addressedEntries); + const document = { + name: "PaseoBatchPullRequestStatus", + cwd: firstEntry.target.cwd, + targetCount: addressedEntries.length, + selections: () => selections, + fragments: BATCH_PR_STATUS_FRAGMENT, + }; + const args = [ + "api", + "graphql", + "-f", + `query=${renderGitHubPollQuery(document.name, selections, document.fragments)}`, + ]; + const response = await read(document).catch((error) => { reportGitHubBatchError(addressedEntries, error); return undefined; }); @@ -1806,7 +1940,7 @@ export function createGitHubService(options: CreateGitHubServiceOptions = {}): G } await Promise.all(distribution.legacyFallback.map((target) => runGitHubPoll(target))); if (distribution.redirected.length > 0) { - await runGitHubPollBatch(distribution.redirected, startedAt); + await runGitHubPollBatch(distribution.redirected, startedAt, read); } }