fix(workspaces): unblock desktop runtime lifecycle

Electron-hosted Local helpers bypassed the sanitized self-Node launch policy, so they lacked ELECTRON_RUN_AS_NODE and provider probes hung.

Command-runtime adapters lacked exact detached POSIX process-group ownership, so forced lifecycle cleanup could leave adapter descendants alive.
This commit is contained in:
Mohamed Boudra
2026-08-13 00:46:16 +02:00
parent 33afa7283b
commit cbb8295ceb
11 changed files with 295 additions and 35 deletions
+79 -14
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env node
import { spawn } from "node:child_process";
import { spawn, spawnSync } from "node:child_process";
import fs from "node:fs";
import { createServer } from "node:http";
import net from "node:net";
@@ -10,6 +10,7 @@ import process from "node:process";
import { setTimeout as delay } from "node:timers/promises";
import { fileURLToPath } from "node:url";
import { experimental_createMCPClient } from "ai";
import { DaemonClient } from "@getpaseo/client/internal/daemon-client";
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
import { chromium } from "playwright";
import { runAppearanceFontSizeRegression } from "./appearance-font-size.electron.mjs";
@@ -180,7 +181,13 @@ async function waitForDesktopStatus(page) {
if (typeof window.paseoDesktop?.invoke !== "function") return null;
return await window.paseoDesktop.invoke("desktop_daemon_status");
});
if (typeof status?.serverId === "string") return status;
if (
status?.status === "running" &&
status.desktopManaged === true &&
typeof status.serverId === "string"
) {
return status;
}
} catch (error) {
// Metro may replace the renderer execution context during its initial load.
lastError = error;
@@ -192,6 +199,68 @@ async function waitForDesktopStatus(page) {
);
}
function processDescendants(parentPid) {
const rows = spawnSync("ps", ["-axo", "pid=,ppid=,command="], { encoding: "utf8" })
.stdout.trim()
.split("\n")
.map((line) => {
const match = line.trim().match(/^(\d+)\s+(\d+)\s+(.+)$/);
return match
? { pid: Number(match[1]), parentPid: Number(match[2]), command: match[3] }
: null;
})
.filter(Boolean);
const descendants = [];
const pending = [parentPid];
while (pending.length > 0) {
const current = pending.pop();
for (const row of rows) {
if (row.parentPid !== current) continue;
descendants.push(row);
pending.push(row.pid);
}
}
return descendants;
}
async function proveManagedLocalProbe(status, daemonPort) {
assert(status.desktopManaged === true, "Electron did not start its managed daemon");
assert(Number.isInteger(status.pid), "Desktop daemon status did not expose its PID");
const daemonClient = new DaemonClient({
url: `ws://127.0.0.1:${daemonPort}/ws`,
clientId: "desktop-managed-local-probe-e2e",
appVersion: "0.3.1",
reconnect: { enabled: false },
});
try {
await daemonClient.connect();
const ensured = await Promise.race([
daemonClient.ensureWorkspaceRuntimeProbe({
projectId: `project-${workspaceIds[0]}`,
runtimeId: "local",
}),
delay(10_000).then(() => {
throw new Error("Electron-managed explicit Local provider probe exceeded 10 seconds");
}),
]);
assert(
ensured.status === "ready" && typeof ensured.workspaceId === "string",
`Explicit Local provider probe failed: ${ensured.error ?? ensured.status}`,
);
await delay(250);
const helpers = processDescendants(status.pid).filter((processInfo) =>
processInfo.command.includes("workspace-helper/executable.mjs"),
);
assert(
helpers.length === 0,
`Explicit Local provider probe left Electron helper children: ${helpers.map(({ pid }) => pid).join(", ")}`,
);
return ensured.workspaceId;
} finally {
await daemonClient.close().catch(() => undefined);
}
}
async function startTargetPage() {
const server = createServer((_request, response) => {
response.writeHead(200, { "content-type": "text/html; charset=utf-8" });
@@ -723,6 +792,7 @@ async function main() {
const children = [];
let browser = null;
let client = null;
let page = null;
try {
const commonEnv = {
@@ -737,16 +807,6 @@ async function main() {
FORCE_COLOR: "0",
NO_COLOR: "1",
};
const daemon = spawnLogged(
"daemon",
process.execPath,
["--import", "tsx", path.join(rootDir, "packages/server/scripts/dev-runner.ts")],
{ cwd: rootDir, env: { ...commonEnv, PASEO_NODE_ENV: "development" } },
artifactDir,
);
children.push(daemon.child);
await waitForPort(daemonPort, "daemon", daemon);
const desktopArgs = [
process.execPath,
devRunner,
@@ -778,8 +838,10 @@ async function main() {
await waitForPort(cdpPort, "Electron CDP", desktop);
browser = await chromium.connectOverCDP(`http://127.0.0.1:${cdpPort}`);
const page = await waitForAppPage(browser, expoPort);
page = await waitForAppPage(browser, expoPort);
const status = await waitForDesktopStatus(page);
await waitForPort(daemonPort, "Desktop-managed daemon");
const localProbeWorkspaceId = await proveManagedLocalProbe(status, daemonPort);
await runAppearanceFontSizeRegression(page);
@@ -797,7 +859,7 @@ async function main() {
targetUrl: target.url,
callerAgentId,
});
writeJson(path.join(artifactDir, "result.json"), report);
writeJson(path.join(artifactDir, "result.json"), { ...report, localProbeWorkspaceId });
console.log(
`Browser desktop browser E2E passed: WebContents ${report.originalWebContentsId} remained ${report.finalWebContentsId}; viewport, inactive capture, focus continuity, list, snapshot, click passed.`,
);
@@ -807,6 +869,9 @@ async function main() {
throw error;
} finally {
await client?.close().catch(() => undefined);
await page
?.evaluate(() => window.paseoDesktop?.invoke("stop_desktop_daemon", { reason: "quit" }))
.catch(() => undefined);
await browser?.close().catch(() => undefined);
for (const child of children.toReversed()) stopProcess(child);
await closeServer(target.server);
@@ -448,7 +448,19 @@ async function signal(id, execId, signalName) {
await waitForProcessExit(pid);
await rm(execFile(id, execId, options), { force: true });
if (options.signalHelperFailure === "error") throw new Error("fixture signal helper failed");
if (options.signalHelperFailure === "hang") await new Promise(() => {});
if (options.signalHelperFailure === "hang") {
if (options.signalHelperDescendantPidFileName) {
const state = await readState(id, options);
const descendant = spawn(process.execPath, ["-e", "setInterval(()=>{},1000)"], {
stdio: "ignore",
});
await writeFile(
path.join(state.root, options.signalHelperDescendantPidFileName),
String(descendant.pid),
);
}
await new Promise(() => setInterval(() => {}, 1_000));
}
}
async function waitForProcessExit(pid) {
@@ -24,7 +24,7 @@ import type {
WorkspacePtyProcess,
WorkspaceRuntimeDriver,
} from "../../drivers/index.js";
const COMMAND_RUNTIME_CLEANUP_TIMEOUT_MS = 1_000;
const COMMAND_RUNTIME_CLEANUP_TIMEOUT_MS = 750;
export interface CommandRuntimeConfig {
command: readonly [string, ...string[]];
@@ -76,7 +76,10 @@ export function createCommandRuntime(
id: runtimeId,
requiresGitProject: true,
reconciliationDomainId: JSON.stringify({ command: config.command, options: config.options }),
workspaceHelperCommand: config.helperCommand ?? ["paseo-workspace-helper"],
workspaceHelper: {
command: config.helperCommand ?? ["paseo-workspace-helper"],
env: {},
},
scriptTerminal: { kind: "direct-command", command: "/bin/sh", argsPrefix: ["-lc"] },
async create(input) {
const response = await lifecycle("create", input.workspaceId, input);
@@ -204,6 +207,7 @@ function spawnCommandPty(
[...command.slice(1), "exec", "--workspace-id", input.workspaceId],
{
env: commandEnvironment(),
detached: process.platform !== "win32",
shell: false,
stdio: ["pipe", "pipe", "pipe", "pipe", "pipe"],
},
@@ -421,7 +425,7 @@ async function forceKillCommandRuntime(
workspaceId: string,
execId: string,
options: Readonly<Record<string, unknown>>,
wrapper: { kill(signal?: NodeJS.Signals): boolean },
wrapper: { pid?: number; kill(signal?: NodeJS.Signals): boolean },
wrapperClosed: Promise<void>,
): Promise<void> {
const signalCommand = spawn(
@@ -436,7 +440,12 @@ async function forceKillCommandRuntime(
"--signal",
"SIGKILL",
],
{ env: commandEnvironment(), shell: false, stdio: ["pipe", "ignore", "pipe"] },
{
env: commandEnvironment(),
detached: process.platform !== "win32",
shell: false,
stdio: ["pipe", "ignore", "pipe"],
},
);
let signalStderr = "";
signalCommand.stderr.on("data", (chunk: Buffer | string) => {
@@ -471,12 +480,12 @@ async function forceKillCommandRuntime(
let cleanupError: Error | null = null;
if (helper.timedOut) {
cleanupError = new Error("signal helper timed out");
signalCommand.kill("SIGKILL");
killOwnedProcessGroup(signalCommand, "SIGKILL");
} else {
cleanupError = helper.value;
if (cleanupError) signalCommand.kill("SIGKILL");
if (cleanupError) killOwnedProcessGroup(signalCommand, "SIGKILL");
}
wrapper.kill("SIGKILL");
killOwnedProcessGroup(wrapper, "SIGKILL");
const wrapperResult = await waitBounded(wrapperClosed, COMMAND_RUNTIME_CLEANUP_TIMEOUT_MS);
if (wrapperResult.timedOut) {
const wrapperError = new Error("wrapper remained alive after SIGKILL");
@@ -538,6 +547,7 @@ function spawnCommandProcess(
[...command.slice(1), "exec", "--workspace-id", input.workspaceId],
{
env: commandEnvironment(),
detached: process.platform !== "win32",
shell: false,
stdio: ["pipe", "pipe", "pipe", "pipe", "pipe"],
},
@@ -702,6 +712,18 @@ function spawnCommandProcess(
}
}
function killOwnedProcessGroup(
child: { pid?: number; kill(signal?: NodeJS.Signals): boolean },
signal: NodeJS.Signals,
): void {
try {
if (process.platform === "win32" || !child.pid) child.kill(signal);
else process.kill(-child.pid, signal);
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;
}
}
async function collect(stream: Readable): Promise<string> {
const chunks: Buffer[] = [];
for await (const chunk of stream)
@@ -99,8 +99,11 @@ export interface WorkspaceRuntimeDriver {
readonly id: WorkspaceRuntimeId;
readonly requiresGitProject: boolean;
readonly reconciliationDomainId?: string;
/** Runtime-local command for Paseo's compatible workspace helper. */
readonly workspaceHelperCommand: readonly [string, ...string[]];
/** Runtime-local launch specification for Paseo's compatible workspace helper. */
readonly workspaceHelper: {
command: readonly [string, ...string[]];
env: Readonly<Record<string, string>>;
};
readonly scriptTerminal: import("../index.js").WorkspaceScriptTerminal;
create(input: WorkspaceDriverCreateInput): Promise<WorkspaceDriverCreation>;
inspect(workspaceId: WorkspaceId): Promise<WorkspaceDriverInspection>;
@@ -0,0 +1,12 @@
import { describe, expect, test } from "vitest";
import { hostWorkspaceHelper } from "./host-helper.js";
describe("hostWorkspaceHelper", () => {
test("uses the external host environment plus Electron Node mode", () => {
expect(hostWorkspaceHelper.env.PATH).toBe(process.env.PATH);
expect(hostWorkspaceHelper.env.ELECTRON_RUN_AS_NODE).toBe("1");
expect(hostWorkspaceHelper.env.PASEO_DESKTOP_MANAGED).toBeUndefined();
expect(hostWorkspaceHelper.env.PASEO_SUPERVISED).toBeUndefined();
});
});
@@ -1,6 +1,11 @@
import { fileURLToPath } from "node:url";
export const hostWorkspaceHelperCommand = [
process.execPath,
fileURLToPath(new URL("../../workspace-helper/executable.mjs", import.meta.url)),
] as const;
import { buildSelfNodeCommand } from "../../paseo-env.js";
const executable = fileURLToPath(new URL("../../workspace-helper/executable.mjs", import.meta.url));
const launch = buildSelfNodeCommand([executable]);
export const hostWorkspaceHelper = {
command: [launch.command, ...launch.args] as readonly [string, ...string[]],
env: launch.env,
};
@@ -9,7 +9,7 @@ import type {
WorkspaceRuntimeDriver,
} from "../drivers/index.js";
import { resolveRuntimeCwd, spawnHostProcess, spawnHostPty } from "./host-process.js";
import { hostWorkspaceHelperCommand } from "./host-helper.js";
import { hostWorkspaceHelper } from "./host-helper.js";
import type { HostGitObservationOwner } from "./host-git-observation.js";
import { createRuntimeStateStore } from "./runtime-state.js";
@@ -55,7 +55,7 @@ export function createLocalRuntime(
return {
id: "local",
requiresGitProject: false,
workspaceHelperCommand: hostWorkspaceHelperCommand,
workspaceHelper: hostWorkspaceHelper,
scriptTerminal: { kind: "persistent-shell" },
async create(input: WorkspaceDriverCreateInput) {
const existing = await inspect(input.workspaceId);
@@ -183,7 +183,7 @@ export function createService(
ownsNewResource = before.status === "missing";
if (ready.state.lifecycle === "ready") {
const helper = bindWorkspaceHelper({
command: driver.workspaceHelperCommand,
command: driver.workspaceHelper.command,
launch: (argv) => launchHelper(driver, input.workspaceId, argv),
});
try {
@@ -544,7 +544,7 @@ export function createService(
}
if (cached) await cached.client.close();
const client = bindWorkspaceHelper({
command: driver.workspaceHelperCommand,
command: driver.workspaceHelper.command,
launch: async (argv) => {
return launchHelper(driver, workspaceId, argv);
},
@@ -626,7 +626,7 @@ export function createService(
const runtimeProcess = await driver.spawn({
workspaceId,
argv,
env: {},
env: driver.workspaceHelper.env,
purpose: { kind: "workspace-helper" },
stdio: { kind: "pipes" },
});
@@ -15,7 +15,7 @@ import type {
WorkspaceRuntimeDriver,
} from "../drivers/index.js";
import { resolveRuntimeCwd, spawnHostProcess, spawnHostPty } from "./host-process.js";
import { hostWorkspaceHelperCommand } from "./host-helper.js";
import { hostWorkspaceHelper } from "./host-helper.js";
import type { HostGitObservationOwner } from "./host-git-observation.js";
import { createRuntimeStateStore } from "./runtime-state.js";
import { writePaseoWorktreeFirstAgentBranchAutoNameMetadata } from "../../../utils/worktree-metadata.js";
@@ -66,7 +66,7 @@ export function createWorktreeRuntime(options: {
return {
id: "worktree",
requiresGitProject: true,
workspaceHelperCommand: hostWorkspaceHelperCommand,
workspaceHelper: hostWorkspaceHelper,
scriptTerminal: { kind: "persistent-shell" },
async create(input: WorkspaceDriverCreateInput) {
const existing = await inspect(input.workspaceId);
@@ -655,6 +655,60 @@ test.each(["error", "hang"] as const)(
5_000,
);
test("archive and permanent deletion reap runtime-owned processes when forced cleanup hangs", async () => {
const descendantPidFileName = "signal-helper-descendant.pid";
const fixture = await createFixture("lifecycle-hanging-cleanup", false, "pty", {
signalHelperFailure: "hang",
signalHelperDescendantPidFileName: descendantPidFileName,
});
const pidFile = path.join(fixture.source, "lifecycle-workload.pid");
const descendantPidFile = path.join(fixture.source, descendantPidFileName);
await fixture.service.create(fixture.createInput);
const workload = await fixture.service.run({
workspaceId: fixture.workspaceId,
argv: [
processExecPath(),
"-e",
`require('node:fs').writeFileSync(${JSON.stringify(pidFile)},String(process.pid));process.on('SIGTERM',()=>{});setInterval(()=>{},1000)`,
],
env: {},
purpose: { kind: "provider-probe", provider: "codex" },
});
workload.stdin.end();
const workloadPid = await vi.waitFor(async () => {
const pid = Number(await readFile(pidFile, "utf8"));
expect(processExists(pid)).toBe(true);
return pid;
});
let descendantPid: number | undefined;
try {
await fixture.service.archive(fixture.workspaceId);
descendantPid = Number(await readFile(descendantPidFile, "utf8"));
expect(fixture.archivedWorkspaceIds).toContain(fixture.workspaceId);
await expect(workload.exited).rejects.toThrow("forced process cleanup failed");
expect(processExists(workloadPid)).toBe(false);
expect(processExists(descendantPid)).toBe(false);
expect(runtimeAdapterProcesses(fixture.workspaceId)).toEqual([]);
await fixture.service.destroy(fixture.workspaceId);
await fixture.service.destroy(fixture.workspaceId);
expect(fixture.archivedWorkspaceIds).not.toContain(fixture.workspaceId);
expect(await readdir(fixture.stateDirectory)).toEqual([]);
expect(runtimeAdapterProcesses(fixture.workspaceId)).toEqual([]);
} finally {
if (processExists(workloadPid)) {
try {
process.kill(-workloadPid, "SIGKILL");
} catch {
// Failure-only containment for the process ownership assertion above.
}
}
if (descendantPid && processExists(descendantPid)) process.kill(descendantPid, "SIGKILL");
await fixture.service.destroy(fixture.workspaceId).catch(() => undefined);
}
}, 10_000);
test("run admission racing pause cannot leave an unregistered workload running", async () => {
const fixture = await createFixture("race", true);
await fixture.service.create(fixture.createInput);
@@ -714,6 +768,7 @@ async function createFixture(
const barrierDirectory = path.join(root, "barrier");
await Promise.all([mkdir(source), mkdir(stateDirectory), mkdir(barrierDirectory)]);
const runtimeIds = new Map<string, string>();
const archivedWorkspaceIds = new Set<string>();
const workspaceId = `${name}-workspace`;
const service = createWorkspaceRuntimeService({
paseoHome: path.join(root, "paseo-home"),
@@ -722,8 +777,11 @@ async function createFixture(
runtimeIds.set(id, runtimeId);
},
beginWorkspaceDeletion: async () => {},
archiveWorkspaceRecord: async (id) => void archivedWorkspaceIds.add(id),
restoreWorkspaceRecord: async (id) => void archivedWorkspaceIds.delete(id),
removeWorkspaceRecord: async (id) => {
runtimeIds.delete(id);
archivedWorkspaceIds.delete(id);
},
externalRuntimes: {
fixture: {
@@ -751,6 +809,7 @@ async function createFixture(
stateDirectory,
barrierDirectory,
workspaceId,
archivedWorkspaceIds,
service,
createInput: {
workspaceId,
@@ -780,6 +839,12 @@ function processExists(pid: number): boolean {
}
}
function runtimeAdapterProcesses(workspaceId: string): string[] {
return execFileSync("ps", ["-axo", "command="], { encoding: "utf8" })
.split("\n")
.filter((command) => command.includes(fixtureExecutable) && command.includes(workspaceId));
}
function nextFile(directory: string, expectedName: string): Promise<void> {
return new Promise((resolve, reject) => {
const watcher = watch(directory, (_event, filename) => {
@@ -367,6 +367,82 @@ test("the command runtime owns Docker workspace materialization, pipes, and life
}
}, 180_000);
test("production archive and deletion reap Docker runtime processes and owned resources", async () => {
const root = await mkdtemp(path.join(tmpdir(), "paseo-runtime-docker-lifecycle-cleanup-"));
const repo = await createRepository(root);
const paseoHome = path.join(root, "paseo-home");
const workspaceId = `docker-lifecycle-cleanup-${Date.now()}`;
const runtimeIds = new Map<string, string>();
const archivedWorkspaceIds = new Set<string>();
const service = createWorkspaceRuntimeService({
paseoHome,
resolveRuntimeId: async (id) => runtimeIds.get(id) ?? null,
persistRuntimeId: async (id, runtimeId) => void runtimeIds.set(id, runtimeId),
beginWorkspaceDeletion: async () => {},
archiveWorkspaceRecord: async (id) => void archivedWorkspaceIds.add(id),
restoreWorkspaceRecord: async (id) => void archivedWorkspaceIds.delete(id),
removeWorkspaceRecord: async (id) => {
runtimeIds.delete(id);
archivedWorkspaceIds.delete(id);
},
externalRuntimes: { docker: { type: "docker", image } },
});
onTestFinished(async () => {
await service.destroy(workspaceId).catch(() => undefined);
const leakedContainers = dockerResourceCount("ps", workspaceId);
const leakedVolumes = dockerResourceCount("volume", workspaceId);
const leakedProcesses = dockerTestProcessPids(new Set([workspaceId]));
await rm(root, { recursive: true, force: true });
if (leakedContainers || leakedVolumes || leakedProcesses.length) {
throw new Error(
`Production lifecycle leaked containers=${leakedContainers} volumes=${leakedVolumes} processes=${leakedProcesses.join(",")}`,
);
}
}, 30_000);
await service.create({
workspaceId,
runtimeId: "docker",
project: {
id: "docker-lifecycle-cleanup-project",
source: { kind: "host-directory", path: repo },
},
placement: { kind: "existing" },
purpose: "provider-probe",
});
const runtimeProcess = await service.run({
workspaceId,
argv: [
"/usr/local/bin/node",
"-e",
"process.on('SIGTERM',()=>{});console.log('owned-ready');setInterval(()=>{},1000)",
],
env: {},
purpose: { kind: "provider-probe", provider: "codex" },
});
runtimeProcess.stdin.end();
await expect(waitForOutput(runtimeProcess.stdout, "owned-ready")).resolves.toContain(
"owned-ready",
);
await service.archive(workspaceId);
expect(archivedWorkspaceIds).toContain(workspaceId);
await expect(runtimeProcess.exited).resolves.toEqual({ code: null, signal: "SIGKILL" });
expect(dockerRunningContainerCount(workspaceId)).toBe(0);
expect(dockerResourceCount("ps", workspaceId)).toBe(1);
expect(dockerResourceCount("volume", workspaceId)).toBe(1);
expect(dockerTestProcessPids(new Set([workspaceId]))).toEqual([]);
await service.destroy(workspaceId);
await service.destroy(workspaceId);
expect(runtimeIds.has(workspaceId)).toBe(false);
expect(archivedWorkspaceIds.has(workspaceId)).toBe(false);
expect(dockerResourceCount("ps", workspaceId)).toBe(0);
expect(dockerResourceCount("volume", workspaceId)).toBe(0);
expect(dockerTestProcessPids(new Set([workspaceId]))).toEqual([]);
}, 60_000);
test("Docker buffers immediate pipe signals until the workload identity is authoritative", async () => {
const fixture = await dockerTestFixture("paseo-runtime-docker-started-");
const root = fixture.root;