diff --git a/docs/data-model.md b/docs/data-model.md index 1568d13d5e..8f6b80e9d9 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -187,6 +187,7 @@ Single file, validated with `PersistedConfigSchema`. hostnames: true | string[], // legacy alias `allowedHosts` is migrated on load trustedProxies: true | string[], // defaults to ["loopback"]; Express proxy names/CIDRs mcp: { enabled: boolean, injectIntoAgents: boolean }, + git: { maxProcessesPerSecond: number, maxProcessConcurrency: number }, appendSystemPrompt: string, // appended to supported provider system/developer prompts cors: { allowedOrigins: string[] }, relay: { enabled: boolean, endpoint: string, publicEndpoint: string, useTls: boolean, publicUseTls: boolean }, // new homes materialize enabled: false @@ -234,6 +235,40 @@ Single file, validated with `PersistedConfigSchema`. All fields are optional with sensible defaults. +### Git process limits + +Git process limits are global to one daemon. The start-rate limit defaults to `64` processes per +second, and the concurrency limit defaults to `8`: + +```json +{ + "daemon": { + "git": { + "maxProcessesPerSecond": 64, + "maxProcessConcurrency": 8 + } + } +} +``` + +`maxProcessesPerSecond` limits Git process starts in any one-second interval. The allowance can +start as a burst; it does not wait for earlier processes to exit. `maxProcessConcurrency` limits +the number of Git processes that have started but not exited. Every Git command uses both limits, +including initial workspace reads, filesystem-triggered refreshes, background checks, and explicit +requests. + +Environment variables override `config.json`: + +| Environment variable | Setting | +| ------------------------------------ | ------------------------ | +| `PASEO_GIT_MAX_PROCESSES_PER_SECOND` | `maxProcessesPerSecond` | +| `PASEO_GIT_MAX_PROCESS_CONCURRENCY` | `maxProcessConcurrency` | +| `PASEO_GIT_CONCURRENCY` | Legacy concurrency alias | + +`PASEO_GIT_MAX_PROCESS_CONCURRENCY` wins when it and the legacy alias are both set. Restart the +daemon after changing the file or environment. Run `paseo daemon restart` for a standalone daemon. +For a desktop-managed daemon, fully quit and reopen Paseo Desktop. + `agents.metadataGeneration.providers` controls the preferred structured-generation fallback order for daemon-side metadata tasks such as commit messages, PR text, branch names, and generated agent titles. Entries are tried first in the configured order, then Paseo falls through to dynamically discovered defaults and finally the current selection when available. Local speech model ids are intentionally narrow: STT uses `parakeet-tdt-0.6b-v2-int8`, TTS uses `kokoro-en-v0_19`, and turn detection uses the bundled Silero VAD model. diff --git a/docs/development.md b/docs/development.md index 90984441d4..f9361cc293 100644 --- a/docs/development.md +++ b/docs/development.md @@ -244,6 +244,27 @@ The supervisor rotates `daemon.log`. Persisted `log.file.rotate` settings in `PASEO_LOG_ROTATE_SIZE` and `PASEO_LOG_ROTATE_COUNT` env vars override the defaults. The default rotation is `10m` x `3` files everywhere. +### Git process pressure + +If Git refreshes consume too much CPU, disk, or antivirus capacity, especially on Windows, reduce +the daemon-global Git process limits in `$PASEO_HOME/config.json`: + +```json +{ + "daemon": { + "git": { + "maxProcessesPerSecond": 5, + "maxProcessConcurrency": 4 + } + } +} +``` + +Restart the daemon with `paseo daemon restart`. If Paseo Desktop manages the daemon, fully quit and +reopen the desktop app. Lower values reduce machine pressure but make Git-backed workspace state and +Git RPCs wait longer. See [Git process limits](data-model.md#git-process-limits) for defaults, +semantics, and environment-variable overrides. + ### Agent Tool Catalog Measurement Measure the MCP `tools/list` payload that Paseo injects into agents with: diff --git a/nix/npm-deps.hash b/nix/npm-deps.hash index d2a8ed0ec4..d23e9ece26 100644 --- a/nix/npm-deps.hash +++ b/nix/npm-deps.hash @@ -1 +1 @@ -sha256-UunIQIS/Y2LhJWqhZMcU1CM9aVOwQUoKKR5z/kj+JDE= +sha256-Mg6cn08RoEFmAYKDjowYd7DZSeeJezix0rzQeDadRAs= diff --git a/package-lock.json b/package-lock.json index b4ecfc8ca0..ad0390d58c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28414,6 +28414,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-throttle": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/p-throttle/-/p-throttle-8.1.0.tgz", + "integrity": "sha512-c1wmXavsHZIC4g1OLhOsafK6jZSAeMo0Ap3yivj59PUcCkpacy5YgWdgIp/dB4vp1JZrfBSsPCR0YuADB+ENLQ==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", @@ -38210,6 +38222,7 @@ "openai": "^6.44.0", "p-limit": "^7.3.0", "p-memoize": "^8.0.0", + "p-throttle": "^8.1.0", "pino": "^10.2.0", "pino-pretty": "^13.1.3", "qrcode": "^1.5.4", diff --git a/packages/server/package.json b/packages/server/package.json index 964acd7a31..51c43a1049 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -90,6 +90,7 @@ "openai": "^6.44.0", "p-limit": "^7.3.0", "p-memoize": "^8.0.0", + "p-throttle": "^8.1.0", "pino": "^10.2.0", "pino-pretty": "^13.1.3", "qrcode": "^1.5.4", diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 734f619326..19b9e44494 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -198,6 +198,8 @@ import { createWebUiMiddleware } from "./web-ui.js"; import { WorkspaceAutoName } from "./workspace-auto-name.js"; import { createGitMutationService } from "./session/git-mutation/git-mutation-service.js"; import { workspaceIdsOnCheckout } from "./workspace-directory.js"; +import { configureGitProcessPolicy } from "../utils/run-git-command.js"; +import { resolveGitProcessPolicy } from "../utils/git-process-scheduler.js"; import { resolveFirstAgentPromptTitle } from "./agent/create-agent-title.js"; import { createAgentCommand, @@ -388,6 +390,10 @@ export interface PaseoDaemonConfig { mcpEnabled?: boolean; mcpInjectIntoAgents?: boolean; browserToolsEnabled?: boolean; + git?: { + maxProcessesPerSecond: number; + maxProcessConcurrency: number; + }; autoArchiveAfterMerge?: boolean; enableTerminalAgentHooks?: boolean; appendSystemPrompt?: string; @@ -539,6 +545,7 @@ export async function createPaseoDaemon( rootLogger: Logger, dependencies: PaseoDaemonDependencies = {}, ): Promise { + configureGitProcessPolicy(config.git ?? resolveGitProcessPolicy({ env: process.env })); const logger = rootLogger.child({ module: "bootstrap" }); const bootstrapStart = performance.now(); const elapsed = () => `${(performance.now() - bootstrapStart).toFixed(0)}ms`; diff --git a/packages/server/src/server/config-git.test.ts b/packages/server/src/server/config-git.test.ts new file mode 100644 index 0000000000..b8cdc3fcae --- /dev/null +++ b/packages/server/src/server/config-git.test.ts @@ -0,0 +1,89 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, test } from "vitest"; + +import { loadConfig } from "./config.js"; + +const roots: string[] = []; + +async function createHome(config: object = {}): Promise { + const home = await mkdtemp(path.join(os.tmpdir(), "paseo-config-git-")); + roots.push(home); + await writeFile(path.join(home, "config.json"), JSON.stringify(config)); + return home; +} + +describe("daemon Git process config", () => { + afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); + }); + + test("defaults the global process limits", async () => { + const home = await createHome(); + + expect(loadConfig(home, { env: {} }).git).toEqual({ + maxProcessesPerSecond: 64, + maxProcessConcurrency: 8, + }); + }); + + test("loads both limits from daemon.git", async () => { + const home = await createHome({ + daemon: { + git: { + maxProcessesPerSecond: 5, + maxProcessConcurrency: 4, + }, + }, + }); + + expect(loadConfig(home, { env: {} }).git).toEqual({ + maxProcessesPerSecond: 5, + maxProcessConcurrency: 4, + }); + }); + + test("new environment variables override config.json", async () => { + const home = await createHome({ + daemon: { + git: { + maxProcessesPerSecond: 5, + maxProcessConcurrency: 4, + }, + }, + }); + + expect( + loadConfig(home, { + env: { + PASEO_GIT_MAX_PROCESSES_PER_SECOND: "12", + PASEO_GIT_MAX_PROCESS_CONCURRENCY: "6", + }, + }).git, + ).toEqual({ + maxProcessesPerSecond: 12, + maxProcessConcurrency: 6, + }); + }); + + test("accepts legacy PASEO_GIT_CONCURRENCY below the renamed variable", async () => { + const home = await createHome(); + + expect( + loadConfig(home, { + env: { + PASEO_GIT_CONCURRENCY: "3", + }, + }).git?.maxProcessConcurrency, + ).toBe(3); + expect( + loadConfig(home, { + env: { + PASEO_GIT_CONCURRENCY: "3", + PASEO_GIT_MAX_PROCESS_CONCURRENCY: "7", + }, + }).git?.maxProcessConcurrency, + ).toBe(7); + }); +}); diff --git a/packages/server/src/server/config.ts b/packages/server/src/server/config.ts index e5979b529e..93ab9409a1 100644 --- a/packages/server/src/server/config.ts +++ b/packages/server/src/server/config.ts @@ -22,6 +22,7 @@ import { AgentProviderSchema } from "@getpaseo/protocol/provider-manifest"; import { hashDaemonPassword } from "./auth.js"; import { resolveSpeechConfig } from "./speech/speech-config-resolver.js"; import { mergeHostnames, parseHostnamesEnv, type HostnamesConfig } from "./hostnames.js"; +import { resolveGitProcessPolicy } from "../utils/git-process-scheduler.js"; const DEFAULT_PORT = 6767; const DEFAULT_RELAY_ENDPOINT = "relay.paseo.sh:443"; @@ -87,6 +88,16 @@ function normalizeLogEnv(value: string | undefined): string | undefined { return value.trim().toLowerCase(); } +function resolveGitProcessConfig( + env: NodeJS.ProcessEnv, + persisted: ReturnType, +): NonNullable { + return resolveGitProcessPolicy({ + env, + persisted: persisted.daemon?.git, + }); +} + export type CliConfigOverrides = Partial<{ listen: string; relayEnabled: boolean; @@ -504,6 +515,7 @@ export function loadConfig( mcpEnabled, mcpInjectIntoAgents, browserToolsEnabled, + git: resolveGitProcessConfig(env, persisted), autoArchiveAfterMerge, enableTerminalAgentHooks: persisted.daemon?.enableTerminalAgentHooks ?? false, appendSystemPrompt, diff --git a/packages/server/src/server/persisted-config.ts b/packages/server/src/server/persisted-config.ts index 3d08b28ef2..e864450682 100644 --- a/packages/server/src/server/persisted-config.ts +++ b/packages/server/src/server/persisted-config.ts @@ -248,6 +248,13 @@ export const PersistedConfigSchema = z }) .passthrough() .optional(), + git: z + .object({ + maxProcessesPerSecond: z.number().int().positive().optional(), + maxProcessConcurrency: z.number().int().positive().optional(), + }) + .strict() + .optional(), autoArchiveAfterMerge: z.boolean().optional(), enableTerminalAgentHooks: z.boolean().optional(), appendSystemPrompt: z.string().optional(), diff --git a/packages/server/src/server/session.test.ts b/packages/server/src/server/session.test.ts index dbcefba8e6..0b0568686f 100644 --- a/packages/server/src/server/session.test.ts +++ b/packages/server/src/server/session.test.ts @@ -201,10 +201,13 @@ const agentResponseMocks = vi.hoisted(() => ({ })); const spawnMocks = vi.hoisted(() => ({ - execCommand: vi.fn(), spawnWorkspaceScript: vi.fn(), })); +const gitCommandMocks = vi.hoisted(() => ({ + runGitCommand: vi.fn(), +})); + const paseoWorktreeServiceMocks = vi.hoisted(() => ({ createPaseoWorktree: vi.fn(), })); @@ -253,11 +256,11 @@ vi.mock("./paseo-worktree-service.js", async (importOriginal) => { }; }); -vi.mock("../utils/spawn.js", async (importOriginal) => { - const actual = await importOriginal(); +vi.mock("../utils/run-git-command.js", async (importOriginal) => { + const actual = await importOriginal(); return { ...actual, - execCommand: spawnMocks.execCommand, + runGitCommand: gitCommandMocks.runGitCommand, }; }); @@ -4045,7 +4048,7 @@ describe("session stash mutation handling", () => { const messages: unknown[] = []; const workspaceGitService = { getSnapshot: vi.fn().mockResolvedValue({}) }; const session = createSessionForTest({ workspaceGitService, messages }); - spawnMocks.execCommand.mockResolvedValue({ + gitCommandMocks.runGitCommand.mockResolvedValue({ stdout: "", stderr: "", exitCode: 0, @@ -4060,6 +4063,10 @@ describe("session stash mutation handling", () => { requestId: "request-stash-push", }); + expect(gitCommandMocks.runGitCommand).toHaveBeenCalledWith( + ["stash", "push", "--include-untracked", "-m", "paseo-auto-stash: feature"], + { cwd: "/tmp/repo", timeout: 120_000 }, + ); expect(workspaceGitService.getSnapshot).toHaveBeenCalledWith("/tmp/repo", { force: true, reason: "stash-push", @@ -4079,7 +4086,7 @@ describe("session stash mutation handling", () => { const messages: unknown[] = []; const workspaceGitService = { getSnapshot: vi.fn().mockResolvedValue({}) }; const session = createSessionForTest({ workspaceGitService, messages }); - spawnMocks.execCommand.mockResolvedValue({ + gitCommandMocks.runGitCommand.mockResolvedValue({ stdout: "", stderr: "", exitCode: 0, @@ -4094,6 +4101,10 @@ describe("session stash mutation handling", () => { requestId: "request-stash-pop", }); + expect(gitCommandMocks.runGitCommand).toHaveBeenCalledWith(["stash", "pop", "stash@{0}"], { + cwd: "/tmp/repo", + timeout: 120_000, + }); expect(workspaceGitService.getSnapshot).toHaveBeenCalledWith("/tmp/repo", { force: true, reason: "stash-pop", diff --git a/packages/server/src/server/session/checkout/checkout-session.ts b/packages/server/src/server/session/checkout/checkout-session.ts index e9d58f7629..091ee620f7 100644 --- a/packages/server/src/server/session/checkout/checkout-session.ts +++ b/packages/server/src/server/session/checkout/checkout-session.ts @@ -51,7 +51,7 @@ import { listCheckoutCommits, getCommitFileDiff, } from "../../../utils/checkout-git.js"; -import { execCommand } from "../../../utils/spawn.js"; +import { runGitCommand } from "../../../utils/run-git-command.js"; import { expandTilde } from "../../../utils/path.js"; import type { GitMetadataGenerator } from "./git-metadata-generator.js"; @@ -614,8 +614,9 @@ export class CheckoutSession { const message = branchLabel ? `${CheckoutSession.PASEO_STASH_PREFIX} ${branchLabel}` : `${CheckoutSession.PASEO_STASH_PREFIX} unnamed`; - await execCommand("git", ["stash", "push", "--include-untracked", "-m", message], { + await runGitCommand(["stash", "push", "--include-untracked", "-m", message], { cwd, + timeout: 120_000, }); await this.gitMutation.notifyGitMutation(cwd, "stash-push"); this.scheduleDiffRefresh(cwd); @@ -636,8 +637,9 @@ export class CheckoutSession { ): Promise { const { cwd, stashIndex, requestId } = msg; try { - await execCommand("git", ["stash", "pop", `stash@{${stashIndex}}`], { + await runGitCommand(["stash", "pop", `stash@{${stashIndex}}`], { cwd, + timeout: 120_000, }); await this.gitMutation.notifyGitMutation(cwd, "stash-pop"); this.scheduleDiffRefresh(cwd); diff --git a/packages/server/src/server/session/git-mutation/git-mutation-service.ts b/packages/server/src/server/session/git-mutation/git-mutation-service.ts index 314767ef3e..7611ba89bc 100644 --- a/packages/server/src/server/session/git-mutation/git-mutation-service.ts +++ b/packages/server/src/server/session/git-mutation/git-mutation-service.ts @@ -5,7 +5,7 @@ import { type CheckoutExistingBranchResult, type GitMutationRefreshReason, } from "../../../utils/checkout-git.js"; -import { execCommand } from "../../../utils/spawn.js"; +import { runGitCommand } from "../../../utils/run-git-command.js"; import type { WorkspaceGitService } from "../../workspace-git-service.js"; import { assertSafeGitRef as assertWorktreeSafeGitRef } from "../../worktree-session.js"; @@ -118,7 +118,10 @@ export function createGitMutationService(deps: { } await ensureCleanWorkingTree(cwd); - await execCommand("git", ["checkout", "-b", newBranchName, baseBranch], { cwd }); + await runGitCommand(["checkout", "-b", newBranchName, baseBranch], { + cwd, + timeout: 120_000, + }); await notifyGitMutation(cwd, "create-branch"); }, diff --git a/packages/server/src/server/workspace-git-service.primitive.test.ts b/packages/server/src/server/workspace-git-service.primitive.test.ts index a5a91f54f0..77380d72bf 100644 --- a/packages/server/src/server/workspace-git-service.primitive.test.ts +++ b/packages/server/src/server/workspace-git-service.primitive.test.ts @@ -1153,12 +1153,13 @@ describe("WorkspaceGitServiceImpl primitive refresh entrypoint", () => { test("settled GitHub self-heal reads stay on the slow poll window without refreshing git", async () => { let nowMs = 0; const githubReadCalls: Array<{ reason: string | undefined; tickMs: number }> = []; + const runner = vi.fn(async () => ({ + stdout: currentPullRequestJson(), + stderr: "", + })); const github = createGitHubService({ ttlMs: 0, - runner: vi.fn(async () => ({ - stdout: currentPullRequestJson(), - stderr: "", - })), + runner, resolveGhPath: async () => "/usr/bin/gh", now: () => nowMs, }); @@ -1180,6 +1181,9 @@ describe("WorkspaceGitServiceImpl primitive refresh entrypoint", () => { const subscription = service.registerWorkspace({ cwd: REPO_CWD }, vi.fn()); await flushPromises(); await vi.advanceTimersByTimeAsync(0); + await vi.waitFor(() => { + expect(runner).toHaveBeenCalledTimes(1); + }); await flushPromises(); const gitReadsAfterInitialSnapshot = getCheckoutStatus.mock.calls.length; diff --git a/packages/server/src/services/github-service.test.ts b/packages/server/src/services/github-service.test.ts index d82defec0c..c6370e84e4 100644 --- a/packages/server/src/services/github-service.test.ts +++ b/packages/server/src/services/github-service.test.ts @@ -820,6 +820,7 @@ describe("ForgeService", () => { reason: "self-heal-github", }), ]); + await vi.advanceTimersByTimeAsync(0); expect(currentPullRequestStatusCalls(runner.calls)).toHaveLength(1); subscription?.unsubscribe(); @@ -856,6 +857,7 @@ describe("ForgeService", () => { now = EXPECTED_GITHUB_SLOW_POLL_MS; await vi.advanceTimersByTimeAsync(EXPECTED_GITHUB_SLOW_POLL_MS - EXPECTED_GITHUB_FAST_POLL_MS); + await vi.advanceTimersByTimeAsync(0); expect(currentPullRequestStatusCalls(runner.calls)).toHaveLength(2); expect(reads.map((read) => read.reason)).toEqual([undefined, "self-heal-github"]); diff --git a/packages/server/src/utils/git-command-runtime-metrics.test.ts b/packages/server/src/utils/git-command-runtime-metrics.test.ts index e5bcd75d29..7f882ceb77 100644 --- a/packages/server/src/utils/git-command-runtime-metrics.test.ts +++ b/packages/server/src/utils/git-command-runtime-metrics.test.ts @@ -1,10 +1,13 @@ import { describe, expect, test } from "vitest"; import { GitCommandRuntimeMetricsWindow } from "./git-command-runtime-metrics.js"; -function createMetricsWindow(concurrencyLimit = 2) { +function createMetricsWindow(concurrencyLimit = 2, maxProcessesPerSecond = 8) { let now = 1_000; return { - metrics: new GitCommandRuntimeMetricsWindow(concurrencyLimit, () => now), + metrics: new GitCommandRuntimeMetricsWindow( + { concurrencyLimit, maxProcessesPerSecond }, + () => now, + ), advance(ms: number) { now += ms; }, @@ -42,6 +45,7 @@ describe("GitCommandRuntimeMetricsWindow", () => { expect(metrics.snapshotAndReset()).toMatchObject({ concurrencyLimit: 1, + maxProcessesPerSecond: 8, active: 1, pending: 1, peakActive: 1, diff --git a/packages/server/src/utils/git-command-runtime-metrics.ts b/packages/server/src/utils/git-command-runtime-metrics.ts index 69ff54e35c..debced34bb 100644 --- a/packages/server/src/utils/git-command-runtime-metrics.ts +++ b/packages/server/src/utils/git-command-runtime-metrics.ts @@ -7,6 +7,7 @@ export interface GitCommandDurationStats { export interface GitCommandRuntimeMetricsSnapshot { concurrencyLimit: number; + maxProcessesPerSecond: number; active: number; pending: number; peakActive: number; @@ -44,7 +45,10 @@ export class GitCommandRuntimeMetricsWindow { private readonly operationCounts = new Map(); constructor( - private readonly concurrencyLimit: number, + private readonly limits: { + concurrencyLimit: number; + maxProcessesPerSecond: number; + }, private readonly clock: Clock = Date.now, ) {} @@ -98,7 +102,8 @@ export class GitCommandRuntimeMetricsWindow { ...Array.from(this.pendingCommands, (metric) => metric.queuedAtMs), ); const snapshot: GitCommandRuntimeMetricsSnapshot = { - concurrencyLimit: this.concurrencyLimit, + concurrencyLimit: this.limits.concurrencyLimit, + maxProcessesPerSecond: this.limits.maxProcessesPerSecond, active: limiter.active, pending: limiter.pending, peakActive: this.peakActive, diff --git a/packages/server/src/utils/git-process-scheduler.test.ts b/packages/server/src/utils/git-process-scheduler.test.ts new file mode 100644 index 0000000000..0440ce5181 --- /dev/null +++ b/packages/server/src/utils/git-process-scheduler.test.ts @@ -0,0 +1,90 @@ +import { afterEach, describe, expect, test, vi } from "vitest"; + +import { GitProcessScheduler, resolveGitProcessPolicy } from "./git-process-scheduler.js"; + +function createConcurrencyTask( + index: number, + started: number[], + releases: Array<() => void>, +): () => { result: Promise; exited: Promise } { + return () => { + started.push(index); + if (index >= 2) { + const completed = Promise.resolve(); + return { result: completed, exited: completed }; + } + const completed = new Promise((resolve) => { + releases.push(resolve); + }); + return { result: completed, exited: completed }; + }; +} + +describe("GitProcessScheduler", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + test("limits unresolved processes independently of their start rate", async () => { + const scheduler = new GitProcessScheduler({ + maxProcessesPerSecond: 100, + maxProcessConcurrency: 2, + }); + const releases: Array<() => void> = []; + const started: number[] = []; + const tasks = Array.from({ length: 4 }, (_, index) => + scheduler.run(createConcurrencyTask(index, started, releases)), + ); + + await vi.waitFor(() => expect(started).toEqual([0, 1])); + releases.shift()?.(); + await vi.waitFor(() => expect(started).toContain(2)); + releases.shift()?.(); + await Promise.all(tasks); + + expect(scheduler.activeCount).toBe(0); + expect(scheduler.pendingCount).toBe(0); + }); + + test("limits process starts in each strict one-second interval", async () => { + vi.useFakeTimers(); + const scheduler = new GitProcessScheduler({ + maxProcessesPerSecond: 2, + maxProcessConcurrency: 4, + }); + const startedAt: number[] = []; + const recordStart = () => { + startedAt.push(Date.now()); + const completed = Promise.resolve(); + return { result: completed, exited: completed }; + }; + const tasks = Array.from({ length: 4 }, () => scheduler.run(recordStart)); + + await vi.advanceTimersByTimeAsync(0); + expect(startedAt).toHaveLength(2); + await vi.advanceTimersByTimeAsync(999); + expect(startedAt).toHaveLength(2); + await vi.advanceTimersByTimeAsync(1); + expect(startedAt).toHaveLength(3); + await vi.advanceTimersByTimeAsync(500); + await Promise.all(tasks); + + expect(startedAt).toHaveLength(4); + expect(startedAt[2]! - startedAt[0]!).toBeGreaterThanOrEqual(1_000); + }); +}); + +describe("resolveGitProcessPolicy", () => { + test("prefers renamed environment variables over legacy and persisted values", () => { + expect( + resolveGitProcessPolicy({ + env: { + PASEO_GIT_MAX_PROCESSES_PER_SECOND: "12", + PASEO_GIT_MAX_PROCESS_CONCURRENCY: "7", + PASEO_GIT_CONCURRENCY: "3", + }, + persisted: { maxProcessesPerSecond: 5, maxProcessConcurrency: 4 }, + }), + ).toEqual({ maxProcessesPerSecond: 12, maxProcessConcurrency: 7 }); + }); +}); diff --git a/packages/server/src/utils/git-process-scheduler.ts b/packages/server/src/utils/git-process-scheduler.ts new file mode 100644 index 0000000000..81ce87c4b2 --- /dev/null +++ b/packages/server/src/utils/git-process-scheduler.ts @@ -0,0 +1,88 @@ +import pLimit from "p-limit"; +import pThrottle from "p-throttle"; + +export interface GitProcessPolicy { + maxProcessesPerSecond: number; + maxProcessConcurrency: number; +} + +export const DEFAULT_GIT_PROCESS_POLICY: GitProcessPolicy = { + maxProcessesPerSecond: 64, + maxProcessConcurrency: 8, +}; + +export interface ScheduledGitProcess { + result: Promise; + exited: Promise; +} + +function parsePositiveInteger(value: string | undefined): number | undefined { + if (value === undefined) { + return undefined; + } + const parsed = Number(value); + return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined; +} + +export function resolveGitProcessPolicy(input: { + env: NodeJS.ProcessEnv; + persisted?: Partial; +}): GitProcessPolicy { + return { + maxProcessesPerSecond: + parsePositiveInteger(input.env.PASEO_GIT_MAX_PROCESSES_PER_SECOND) ?? + input.persisted?.maxProcessesPerSecond ?? + DEFAULT_GIT_PROCESS_POLICY.maxProcessesPerSecond, + maxProcessConcurrency: + parsePositiveInteger(input.env.PASEO_GIT_MAX_PROCESS_CONCURRENCY) ?? + // COMPAT(gitConcurrencyEnv): renamed in v0.2.6; remove after 2027-02-02. + parsePositiveInteger(input.env.PASEO_GIT_CONCURRENCY) ?? + input.persisted?.maxProcessConcurrency ?? + DEFAULT_GIT_PROCESS_POLICY.maxProcessConcurrency, + }; +} + +export class GitProcessScheduler { + private readonly limit; + private readonly throttle; + private running = 0; + private waiting = 0; + + constructor(readonly policy: GitProcessPolicy) { + this.limit = pLimit(policy.maxProcessConcurrency); + this.throttle = pThrottle({ + limit: policy.maxProcessesPerSecond, + interval: 1_000, + strict: true, + }); + } + + get activeCount(): number { + return this.running; + } + + get pendingCount(): number { + return this.waiting; + } + + run(start: () => ScheduledGitProcess): Promise { + this.waiting += 1; + return new Promise((resolve, reject) => { + void this.limit( + this.throttle(async () => { + this.waiting = Math.max(0, this.waiting - 1); + this.running += 1; + try { + const process = start(); + void process.result.then(resolve, reject); + await process.exited; + } catch (error) { + reject(error); + } finally { + this.running = Math.max(0, this.running - 1); + } + }), + ).catch(reject); + }); + } +} diff --git a/packages/server/src/utils/run-git-command.test.ts b/packages/server/src/utils/run-git-command.test.ts index 371b0d3355..7bbbe3e2b8 100644 --- a/packages/server/src/utils/run-git-command.test.ts +++ b/packages/server/src/utils/run-git-command.test.ts @@ -2,9 +2,14 @@ import { EventEmitter } from "node:events"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; interface FakeSpawnBehavior { + closeDelayMs?: number; delayMs?: number; emitError?: Error; + exitDelayMs?: number; exitCode?: number | null; + killCloseDelayMs?: number; + killExitDelayMs?: number; + spawnError?: Error; stderrData?: Buffer | string; stdoutData?: Buffer | string; } @@ -43,10 +48,11 @@ class FakeChildProcess extends EventEmitter { public readonly stdout = new EventEmitter(); public killed = false; public killSignals: NodeJS.Signals[] = []; + public closed = false; + public exited = false; private readonly behavior: FakeSpawnBehavior; private readonly timers: NodeJS.Timeout[] = []; - private closed = false; public constructor(behavior: FakeSpawnBehavior) { super(); @@ -69,17 +75,31 @@ class FakeChildProcess extends EventEmitter { this.killed = true; this.killSignals.push(signal); this.clearTimers(); + const exitDelayMs = this.behavior.killExitDelayMs ?? 0; this.schedule(() => { - this.finishClose({ + this.finishExit({ exitCode: null, signal, }); - }, 0); + }, exitDelayMs); + this.schedule( + () => { + this.finishClose({ + exitCode: null, + signal, + }); + }, + Math.max(exitDelayMs, this.behavior.killCloseDelayMs ?? exitDelayMs), + ); return true; } public dispose(): void { this.clearTimers(); + if (!this.exited) { + fakeSpawnController.activeCount -= 1; + this.exited = true; + } this.closed = true; } @@ -108,12 +128,22 @@ class FakeChildProcess extends EventEmitter { return; } + const exitDelayMs = this.behavior.exitDelayMs ?? this.behavior.delayMs ?? 0; this.schedule(() => { - this.finishClose({ + this.finishExit({ exitCode: this.behavior.exitCode ?? 0, signal: null, }); - }, this.behavior.delayMs ?? 0); + }, exitDelayMs); + this.schedule( + () => { + this.finishClose({ + exitCode: this.behavior.exitCode ?? 0, + signal: null, + }); + }, + Math.max(exitDelayMs, this.behavior.closeDelayMs ?? this.behavior.delayMs ?? exitDelayMs), + ); } private finishClose({ @@ -125,19 +155,32 @@ class FakeChildProcess extends EventEmitter { }): void { if (this.closed) return; + this.finishExit({ exitCode, signal }); this.closed = true; this.clearTimers(); - fakeSpawnController.activeCount -= 1; this.emit("close", exitCode, signal); } + private finishExit({ + exitCode, + signal, + }: { + exitCode: number | null; + signal: NodeJS.Signals | null; + }): void { + if (this.exited) return; + + this.exited = true; + fakeSpawnController.activeCount -= 1; + this.emit("exit", exitCode, signal); + } + private finishError(error: Error): void { if (this.closed) return; - this.closed = true; this.clearTimers(); - fakeSpawnController.activeCount -= 1; this.emit("error", error); + this.finishClose({ exitCode: null, signal: null }); } private schedule(callback: () => void, delayMs: number): void { @@ -160,6 +203,9 @@ vi.mock("node:child_process", async () => { ...actual, spawn: vi.fn(() => { const behavior = fakeSpawnController.queue.shift() ?? {}; + if (behavior.spawnError) { + throw behavior.spawnError; + } const child = new FakeChildProcess(behavior); fakeSpawnController.processes.push(child); return child as unknown as ReturnType; @@ -230,6 +276,72 @@ describe("runGitCommand", () => { }); }); + it("holds the limiter slot until a timed out process exits without waiting for close", async () => { + const { runGitCommand } = await loadRunGitCommand(1); + + enqueueSpawnBehaviors( + { delayMs: 5_000, killExitDelayMs: 100, killCloseDelayMs: 500 }, + { delayMs: 0, stdoutData: "next" }, + ); + + const timedOut = runGitCommand(["status"], { + cwd: process.cwd(), + timeout: 20, + }).then( + () => null, + (error: unknown) => error, + ); + const next = runGitCommand(["rev-parse", "--show-toplevel"], { + cwd: process.cwd(), + }); + + await vi.waitFor(() => { + expect(fakeSpawnController.processes[0]?.killed).toBe(true); + }); + expect(fakeSpawnController.processes).toHaveLength(1); + expect(fakeSpawnController.activeCount).toBe(1); + + await expect(timedOut).resolves.toEqual( + expect.objectContaining({ + message: "Git command timed out after 20ms: git status", + }), + ); + await vi.waitFor(() => { + expect(fakeSpawnController.processes).toHaveLength(2); + }); + expect(fakeSpawnController.processes[0]?.exited).toBe(true); + expect(fakeSpawnController.processes[0]?.closed).toBe(false); + await expect(next).resolves.toMatchObject({ + exitCode: 0, + stdout: "next", + }); + expect(fakeSpawnController.peakActiveCount).toBe(1); + }); + + it("finishes process metrics at exit without waiting for close", async () => { + const { runGitCommand, startGitCommandMetrics, stopGitCommandMetrics } = + await loadRunGitCommand(1); + + enqueueSpawnBehaviors( + { exitDelayMs: 0, closeDelayMs: 500 }, + { delayMs: 0, stdoutData: "next" }, + ); + startGitCommandMetrics(); + + void runGitCommand(["status"], { cwd: process.cwd() }); + await expect( + runGitCommand(["rev-parse", "--show-toplevel"], { cwd: process.cwd() }), + ).resolves.toMatchObject({ stdout: "next" }); + + expect(fakeSpawnController.processes[0]?.exited).toBe(true); + expect(fakeSpawnController.processes[0]?.closed).toBe(false); + expect(stopGitCommandMetrics()).toMatchObject({ + total: 2, + failed: 0, + maxConcurrent: 1, + }); + }); + it("resolves truncated stdout, caps output, and kills the child process", async () => { const { runGitCommand } = await loadRunGitCommand(1); @@ -275,6 +387,21 @@ describe("runGitCommand", () => { }); }); + it("releases the limiter after a synchronous spawn failure", async () => { + const { runGitCommand } = await loadRunGitCommand(1); + + enqueueSpawnBehaviors( + { spawnError: new Error("spawn threw") }, + { delayMs: 0, stdoutData: "ok" }, + ); + + await expect(runGitCommand(["status"], { cwd: process.cwd() })).rejects.toThrow("spawn threw"); + await expect(runGitCommand(["status"], { cwd: process.cwd() })).resolves.toMatchObject({ + exitCode: 0, + stdout: "ok", + }); + }, 1_000); + it("traces git command spawn and close metadata when a logger is provided", async () => { const { runGitCommand } = await loadRunGitCommand(1); const trace = vi.fn(); diff --git a/packages/server/src/utils/run-git-command.ts b/packages/server/src/utils/run-git-command.ts index 835f5d32e8..89c6406486 100644 --- a/packages/server/src/utils/run-git-command.ts +++ b/packages/server/src/utils/run-git-command.ts @@ -1,5 +1,4 @@ import { existsSync } from "node:fs"; -import pLimit from "p-limit"; import type { Logger } from "pino"; import type { ProcessEnvRecord } from "../server/paseo-env.js"; import { @@ -13,14 +12,30 @@ import { submitGitCommandTrace, } from "./git-command-trace.js"; import { spawnProcess } from "./spawn.js"; +import { + GitProcessScheduler, + resolveGitProcessPolicy, + type GitProcessPolicy, +} from "./git-process-scheduler.js"; const DEFAULT_TIMEOUT_MS = 30_000; const DEFAULT_MAX_OUTPUT_BYTES = 20 * 1024 * 1024; // 20MB const DEFAULT_STDERR_LIMIT = 2048; -const gitConcurrency = parseInt(process.env.PASEO_GIT_CONCURRENCY ?? "8", 10) || 8; -const gitLimit = pLimit(gitConcurrency); -const gitRuntimeMetrics = new GitCommandRuntimeMetricsWindow(gitConcurrency); +let gitProcessScheduler = new GitProcessScheduler(resolveGitProcessPolicy({ env: process.env })); +let gitRuntimeMetrics = createGitCommandRuntimeMetricsWindow(gitProcessScheduler.policy); + +function createGitCommandRuntimeMetricsWindow(policy: GitProcessPolicy) { + return new GitCommandRuntimeMetricsWindow({ + concurrencyLimit: policy.maxProcessConcurrency, + maxProcessesPerSecond: policy.maxProcessesPerSecond, + }); +} + +export function configureGitProcessPolicy(policy: GitProcessPolicy): void { + gitProcessScheduler = new GitProcessScheduler(policy); + gitRuntimeMetrics = createGitCommandRuntimeMetricsWindow(policy); +} export interface GitCommandOptions { cwd: string; @@ -94,8 +109,8 @@ export function stopGitCommandMetrics(): GitCommandMetricsSnapshot { export function snapshotGitCommandRuntimeMetrics(): GitCommandRuntimeMetricsSnapshot { return gitRuntimeMetrics.snapshotAndReset({ - active: gitLimit.activeCount, - pending: gitLimit.pendingCount, + active: gitProcessScheduler.activeCount, + pending: gitProcessScheduler.pendingCount, }); } @@ -142,232 +157,271 @@ export function runGitCommand( options: GitCommandOptions, ): Promise { const commandTrace = submitGitCommandTrace(args, options.cwd, { - active: gitLimit.activeCount, - pending: gitLimit.pendingCount, + active: gitProcessScheduler.activeCount, + pending: gitProcessScheduler.pendingCount, }); const runtimeMetric = gitRuntimeMetrics.submit(getGitOperation(args)); - const promise = gitLimit( - () => - new Promise((resolve, reject) => { - startGitCommandTrace(commandTrace, { - active: gitLimit.activeCount, - pending: gitLimit.pendingCount, + const promise = gitProcessScheduler.run(() => { + let releaseProcessSlot!: () => void; + const exited = new Promise((resolve) => { + releaseProcessSlot = resolve; + }); + const resultPromise = new Promise((resolve, reject) => { + startGitCommandTrace(commandTrace, { + active: gitProcessScheduler.activeCount, + pending: gitProcessScheduler.pendingCount, + }); + gitRuntimeMetrics.start(runtimeMetric); + const timeout = options.timeout ?? DEFAULT_TIMEOUT_MS; + const maxOutputBytes = options.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES; + const acceptExitCodes = options.acceptExitCodes ?? [0]; + const command = formatGitCommand(args); + const envOverlay = mergeEnvOverlays(options.env, options.envOverlay); + const startedAt = Date.now(); + const metricsState = beginGitCommandMetric(); + const logger = typeof options.logger?.trace === "function" ? options.logger : undefined; + const traceContext = logger + ? { + command: "git", + args, + cwd: options.cwd, + cwdExists: existsSync(options.cwd), + timeout, + maxOutputBytes, + acceptExitCodes, + envOverlayKeys: getEnvOverlayKeys(envOverlay), + } + : null; + + if (logger && traceContext) { + logger.trace(traceContext, "Spawning git command"); + } + + let settled = false; + let metricFinished = false; + let processError: Error | null = null; + let processExit: { exitCode: number | null; signal: NodeJS.Signals | null } | null = null; + let timeoutError: Error | null = null; + let truncated = false; + let stdoutBytes = 0; + let stderrBytes = 0; + const stdoutChunks: Buffer[] = []; + const stderrChunks: Buffer[] = []; + let timer: NodeJS.Timeout | undefined; + + const settle = (callback: () => void) => { + if (settled) return; + settled = true; + if (timer) clearTimeout(timer); + callback(); + }; + + const finishMetricOnce = (metric: GitCommandMetric, timedOut = false) => { + if (metricFinished) return; + metricFinished = true; + finishGitCommandMetric(metricsState, metric); + gitRuntimeMetrics.finish(runtimeMetric, { success: metric.success, timedOut }); + }; + + const settleTimeoutTrace = (exitCode: number | null, signal: NodeJS.Signals | null) => { + settleGitCommandTrace(commandTrace, { + outcome: "timed_out", + exitCode, + signal, }); - gitRuntimeMetrics.start(runtimeMetric); - const timeout = options.timeout ?? DEFAULT_TIMEOUT_MS; - const maxOutputBytes = options.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES; - const acceptExitCodes = options.acceptExitCodes ?? [0]; - const command = formatGitCommand(args); - const envOverlay = mergeEnvOverlays(options.env, options.envOverlay); - const startedAt = Date.now(); - const metricsState = beginGitCommandMetric(); - const logger = typeof options.logger?.trace === "function" ? options.logger : undefined; - const traceContext = logger - ? { - command: "git", - args, - cwd: options.cwd, - cwdExists: existsSync(options.cwd), - timeout, - maxOutputBytes, - acceptExitCodes, - envOverlayKeys: getEnvOverlayKeys(envOverlay), - } - : null; + }; - if (logger && traceContext) { - logger.trace(traceContext, "Spawning git command"); + const markProcessExited = (exitCode: number | null, signal: NodeJS.Signals | null) => { + if (processExit) return; + processExit = { exitCode, signal }; + const timedOut = timeoutError !== null; + finishMetricOnce( + { + args, + cwd: options.cwd, + startedAtMs: startedAt, + durationMs: Date.now() - startedAt, + exitCode, + signal, + success: + !timedOut && !processError && (truncated || acceptExitCodes.includes(exitCode ?? -1)), + }, + timedOut, + ); + releaseProcessSlot(); + if (timedOut) { + settleTimeoutTrace(exitCode, signal); } + }; + const rejectSpawnFailure = (error: unknown) => { + processError = error instanceof Error ? error : new Error(String(error)); + markProcessExited(null, null); + settleGitCommandTrace(commandTrace, { + outcome: "spawn_error", + exitCode: null, + signal: null, + }); + settle(() => reject(processError)); + }; + + let child: ReturnType; + try { // `core.quotepath=false` makes git emit raw UTF-8 paths instead of // octal-escaping non-ASCII bytes (e.g. `测试文件.txt` vs `"\346\265\213..."`). - const child = spawnProcess("git", ["-c", "core.quotepath=false", ...args], { + child = spawnProcess("git", ["-c", "core.quotepath=false", ...args], { cwd: options.cwd, envOverlay, shell: false, stdio: ["ignore", "pipe", "pipe"], }); spawnGitCommandTrace(commandTrace, child.pid); + } catch (error) { + rejectSpawnFailure(error); + return; + } - let settled = false; - let metricFinished = false; - let truncated = false; - let stdoutBytes = 0; - let stderrBytes = 0; - const stdoutChunks: Buffer[] = []; - const stderrChunks: Buffer[] = []; + const stdout = child.stdout; + const stderr = child.stderr; + if (!stdout || !stderr) { + child.kill("SIGKILL"); + rejectSpawnFailure(new Error("Git process did not expose piped stdout and stderr")); + return; + } - const settle = (callback: () => void) => { - if (settled) return; - settled = true; - clearTimeout(timer); - callback(); - }; + timer = setTimeout(() => { + timeoutError = new Error(`Git command timed out after ${timeout}ms: ${command}`); + child.kill("SIGKILL"); + settle(() => reject(timeoutError)); + if (processExit) { + settleTimeoutTrace(processExit.exitCode, processExit.signal); + } + }, timeout); - const finishMetricOnce = (metric: GitCommandMetric, timedOut = false) => { - if (metricFinished) return; - metricFinished = true; - finishGitCommandMetric(metricsState, metric); - gitRuntimeMetrics.finish(runtimeMetric, { success: metric.success, timedOut }); - }; + stdout.on("data", (chunk: Buffer | string) => { + if (settled || truncated) return; - const timer = setTimeout(() => { - const error = new Error(`Git command timed out after ${timeout}ms: ${command}`); + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + const remainingBytes = maxOutputBytes - stdoutBytes; + + if (remainingBytes <= 0) { + truncated = true; child.kill("SIGKILL"); - settleGitCommandTrace(commandTrace, { - outcome: "timed_out", - exitCode: null, - signal: "SIGKILL", - }); - finishMetricOnce( + return; + } + + if (buffer.length > remainingBytes) { + stdoutChunks.push(buffer.subarray(0, remainingBytes)); + stdoutBytes += remainingBytes; + truncated = true; + child.kill("SIGKILL"); + return; + } + + stdoutChunks.push(buffer); + stdoutBytes += buffer.length; + }); + + stderr.on("data", (chunk: Buffer | string) => { + if (settled || stderrBytes >= DEFAULT_STDERR_LIMIT) return; + + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + const remainingBytes = DEFAULT_STDERR_LIMIT - stderrBytes; + + if (buffer.length > remainingBytes) { + stderrChunks.push(buffer.subarray(0, remainingBytes)); + stderrBytes += remainingBytes; + return; + } + + stderrChunks.push(buffer); + stderrBytes += buffer.length; + }); + + child.on("error", (error) => { + processError = error; + if (logger && traceContext) { + logger.trace( { - args, - cwd: options.cwd, - startedAtMs: startedAt, + ...traceContext, + err: error, durationMs: Date.now() - startedAt, - exitCode: null, - signal: "SIGKILL", - success: false, }, - true, + "Git command process error", ); - settle(() => reject(error)); - }, timeout); + } + }); - child.stdout!.on("data", (chunk: Buffer | string) => { - if (settled || truncated) return; + child.on("exit", markProcessExited); - const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); - const remainingBytes = maxOutputBytes - stdoutBytes; - - if (remainingBytes <= 0) { - truncated = true; - child.kill("SIGKILL"); - return; - } - - if (buffer.length > remainingBytes) { - stdoutChunks.push(buffer.subarray(0, remainingBytes)); - stdoutBytes += remainingBytes; - truncated = true; - child.kill("SIGKILL"); - return; - } - - stdoutChunks.push(buffer); - stdoutBytes += buffer.length; - }); - - child.stderr!.on("data", (chunk: Buffer | string) => { - if (settled || stderrBytes >= DEFAULT_STDERR_LIMIT) return; - - const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); - const remainingBytes = DEFAULT_STDERR_LIMIT - stderrBytes; - - if (buffer.length > remainingBytes) { - stderrChunks.push(buffer.subarray(0, remainingBytes)); - stderrBytes += remainingBytes; - return; - } - - stderrChunks.push(buffer); - stderrBytes += buffer.length; - }); - - child.on("error", (error) => { - settleGitCommandTrace(commandTrace, { - outcome: "spawn_error", - exitCode: null, - signal: null, - }); - finishMetricOnce({ - args, - cwd: options.cwd, - startedAtMs: startedAt, - durationMs: Date.now() - startedAt, - exitCode: null, - signal: null, - success: false, - }); - if (logger && traceContext) { - logger.trace( - { - ...traceContext, - err: error, - durationMs: Date.now() - startedAt, - }, - "Git command process error", - ); - } - settle(() => reject(error)); - }); - - child.on("close", (exitCode, signal) => { - settleGitCommandTrace(commandTrace, { - outcome: "closed", - exitCode, - signal, - truncated, - }); - const result: GitCommandResult = { - stdout: Buffer.concat(stdoutChunks).toString("utf8"), - stderr: Buffer.concat(stderrChunks).toString("utf8"), - truncated, - exitCode, - signal, - }; - if (logger && traceContext) { - logger.trace( - { - ...traceContext, - durationMs: Date.now() - startedAt, - exitCode, - signal, - truncated, - stdoutBytes, - stderrBytes, - }, - "Git command closed", - ); - } - - if (!truncated && !acceptExitCodes.includes(exitCode ?? -1)) { - finishMetricOnce({ - args, - cwd: options.cwd, - startedAtMs: startedAt, + child.on("close", (exitCode, signal) => { + markProcessExited(exitCode, signal); + const result: GitCommandResult = { + stdout: Buffer.concat(stdoutChunks).toString("utf8"), + stderr: Buffer.concat(stderrChunks).toString("utf8"), + truncated, + exitCode, + signal, + }; + if (logger && traceContext) { + logger.trace( + { + ...traceContext, durationMs: Date.now() - startedAt, exitCode, signal, - success: false, - }); - const stderrPreview = result.stderr.trim() || "(no stderr)"; - const truncationNote = result.truncated ? " (stdout truncated)" : ""; + truncated, + stdoutBytes, + stderrBytes, + }, + "Git command closed", + ); + } - settle(() => - reject( - new Error( - `Git command failed: ${command}${truncationNote} (exit code: ${String(exitCode)}, signal: ${signal ?? "none"})\n${stderrPreview}`, - ), - ), - ); - return; - } + if (timeoutError) { + return; + } - finishMetricOnce({ - args, - cwd: options.cwd, - startedAtMs: startedAt, - durationMs: Date.now() - startedAt, + if (processError) { + settleGitCommandTrace(commandTrace, { + outcome: "spawn_error", exitCode, signal, - success: true, }); - settle(() => resolve(result)); + settle(() => reject(processError)); + return; + } + + settleGitCommandTrace(commandTrace, { + outcome: "closed", + exitCode, + signal, + truncated, }); - }), + + if (!truncated && !acceptExitCodes.includes(exitCode ?? -1)) { + const stderrPreview = result.stderr.trim() || "(no stderr)"; + const truncationNote = result.truncated ? " (stdout truncated)" : ""; + + settle(() => + reject( + new Error( + `Git command failed: ${command}${truncationNote} (exit code: ${String(exitCode)}, signal: ${signal ?? "none"})\n${stderrPreview}`, + ), + ), + ); + return; + } + + settle(() => resolve(result)); + }); + }); + return { result: resultPromise, exited }; + }); + gitRuntimeMetrics.observeLimiter( + gitProcessScheduler.activeCount, + gitProcessScheduler.pendingCount, ); - gitRuntimeMetrics.observeLimiter(gitLimit.activeCount, gitLimit.pendingCount); return promise; } diff --git a/packages/server/vitest.config.ts b/packages/server/vitest.config.ts index c6af449a33..9f3717a9bc 100644 --- a/packages/server/vitest.config.ts +++ b/packages/server/vitest.config.ts @@ -8,6 +8,10 @@ export default defineConfig({ }, }, test: { + // Fake-timer suites freeze p-throttle's clock, so a real per-second cap deadlocks them. + env: { + PASEO_GIT_MAX_PROCESSES_PER_SECOND: "10000", + }, testTimeout: 30000, hookTimeout: 60000, globals: true, diff --git a/packages/website/public/schemas/paseo.config.v1.json b/packages/website/public/schemas/paseo.config.v1.json index 41e2aceb6a..033b496130 100644 --- a/packages/website/public/schemas/paseo.config.v1.json +++ b/packages/website/public/schemas/paseo.config.v1.json @@ -72,6 +72,20 @@ }, "additionalProperties": true }, + "git": { + "type": "object", + "properties": { + "maxProcessesPerSecond": { + "type": "integer", + "exclusiveMinimum": 0 + }, + "maxProcessConcurrency": { + "type": "integer", + "exclusiveMinimum": 0 + } + }, + "additionalProperties": false + }, "autoArchiveAfterMerge": { "type": "boolean" }, diff --git a/vitest.config.ts b/vitest.config.ts index 326780e1af..77e75f6d6c 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -57,6 +57,10 @@ export default defineConfig({ ], }, test: { + // Fake-timer suites freeze p-throttle's clock, so a real per-second cap deadlocks them. + env: { + PASEO_GIT_MAX_PROCESSES_PER_SECOND: "10000", + }, exclude: [...configDefaults.exclude, "**/.claude/**", "**/.dev/**"], }, });