Files
paseo/packages/server/src/utils/tree-kill.ts
T
01a1d3b2b6 Recover daemon workers stalled by workspace Git pressure (#3033)
* fix(server): bound workspace git pipelines

* fix(server): bound workspace git watcher setup

* fix(server): recover unresponsive daemon workers

* fix(server): tolerate transient daemon heartbeat delays

A five-second watchdog threshold can replace a healthy worker during a short event-loop delay. Keep recovery bounded while allowing a 15-second heartbeat gap, and preserve process-level coverage for workers that ignore graceful termination.

* test(server): normalize workspace fixture paths

Workspace Git entry points resolve paths before invoking dependencies. Keep admission-control fixtures keyed by the same canonical paths so their assertions and deferred gates work on Windows.

---------

Co-authored-by: Timi Ajiboye <trypio00@gmail.com>
Co-authored-by: Mohamed Boudra <boudra.moha@gmail.com>
2026-08-09 03:01:34 +02:00

123 lines
3.1 KiB
TypeScript

import treeKill from "tree-kill";
export interface TreeKillTarget {
pid?: number;
exitCode?: number | null;
signalCode?: NodeJS.Signals | null;
kill(signal?: NodeJS.Signals | number): boolean;
once?(event: "exit", listener: () => void): unknown;
}
export interface TerminateWithTreeKillOptions {
gracefulSignal?: NodeJS.Signals;
forceSignal?: NodeJS.Signals;
gracefulTimeoutMs: number;
forceTimeoutMs?: number;
onForceSignal?: () => void;
}
export type TerminateWithTreeKillResult =
| "already-exited"
| "terminated"
| "killed"
| "kill-timeout";
// Injection seam: production wires terminateWithTreeKill; tests wire a fake that
// records which children were terminated as observable state.
export type ProcessTerminator = (
child: TreeKillTarget,
options: TerminateWithTreeKillOptions,
) => Promise<TerminateWithTreeKillResult>;
export async function terminateWithTreeKill(
child: TreeKillTarget,
options: TerminateWithTreeKillOptions,
): Promise<TerminateWithTreeKillResult> {
if (isProcessExited(child)) {
return "already-exited";
}
const exitPromise = waitForProcessExit(child);
await signalProcessTree(child, options.gracefulSignal ?? "SIGTERM");
if (await waitForExitOrTimeout(exitPromise, options.gracefulTimeoutMs)) {
return "terminated";
}
options.onForceSignal?.();
await signalProcessTree(child, options.forceSignal ?? "SIGKILL");
if (options.forceTimeoutMs === undefined) {
return "killed";
}
return (await waitForExitOrTimeout(exitPromise, options.forceTimeoutMs))
? "killed"
: "kill-timeout";
}
export function signalProcessTree(child: TreeKillTarget, signal: NodeJS.Signals): Promise<void> {
if (isProcessExited(child)) {
return Promise.resolve();
}
const pid = child.pid;
if (typeof pid !== "number" || pid <= 0) {
signalDirectChild(child, signal);
return Promise.resolve();
}
return new Promise((resolve) => {
treeKill(pid, signal, (error) => {
if (error) {
signalDirectChild(child, signal);
}
resolve();
});
});
}
function signalDirectChild(child: TreeKillTarget, signal: NodeJS.Signals): void {
try {
child.kill(signal);
} catch {
// Ignore cleanup races.
}
}
function isProcessExited(child: TreeKillTarget): boolean {
return (
(child.exitCode !== null && child.exitCode !== undefined) ||
(child.signalCode !== null && child.signalCode !== undefined)
);
}
function waitForProcessExit(child: TreeKillTarget): Promise<void> {
if (isProcessExited(child)) {
return Promise.resolve();
}
if (!child.once) {
return new Promise(() => undefined);
}
return new Promise((resolve) => {
child.once?.("exit", resolve);
});
}
async function waitForExitOrTimeout(
exitPromise: Promise<void>,
timeoutMs: number,
): Promise<boolean> {
let timer: NodeJS.Timeout | null = null;
try {
return await Promise.race([
exitPromise.then(() => true),
new Promise<boolean>((resolve) => {
timer = setTimeout(() => resolve(false), timeoutMs);
}),
]);
} finally {
if (timer) {
clearTimeout(timer);
}
}
}