diff --git a/.oxlintrc.json b/.oxlintrc.json index 7a6192215e..c82b517844 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -203,6 +203,31 @@ "readFileSync", "rmSync" ], + // Terminal child process: launcher readiness and PTY record ownership run outside the daemon. + "terminal/terminal-worker-process.ts": [ + "chmodSync", + "closeSync", + "existsSync", + "openSync", + "rmSync", + "writeSync" + ], + // Worker startup only: choose and secure the local listening socket before accepting a controller. + "terminal/terminal-worker-socket-path.ts": ["lstatSync", "mkdirSync"], + // Startup only: build an independent worker installation before launching; Electron ASAR sources require plain fs reads. + "terminal/terminal-worker-runtime.ts": [ + "chmodSync", + "existsSync", + "mkdirSync", + "readdirSync", + "readFileSync", + "renameSync", + "rmSync", + "statSync", + "writeFileSync" + ], + // TODO: CLI shim lookup moved out of terminal.ts; migrate the executable-resolution boundary together. + "terminal/paseo-cli-path.ts": ["existsSync"], // Node loader thread: TypeScript terminal-worker resolution runs in the loader, outside the daemon event loop. "terminal/terminal-ts-loader.mjs": ["existsSync"], // Terminal child process: executable preparation and shell integration run in the worker-backed terminal manager. diff --git a/CLAUDE.md b/CLAUDE.md index 3f1731597c..d72205a01e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -66,6 +66,7 @@ At the start of non-trivial work, list `docs/` and skim anything relevant to the | [docs/android.md](docs/android.md) | App variants, local/cloud builds, EAS workflows, version codes, F-Droid source builds and store metadata | | [docs/docker.md](docs/docker.md) | Running the daemon and bundled web UI in Docker, volumes, agent images, security | | [docs/release.md](docs/release.md) | Release playbook, draft releases, completion checklist | +| [docs/terminal-worker.md](docs/terminal-worker.md) | Terminal worker process model, socket/record/secret, protocol-bump rule and draining, the worker's file copy, Windows desktop | | [docs/terminal-activity.md](docs/terminal-activity.md) | Terminal activity indicators — source-agnostic tracker, agent hook reporting, adding a new hook provider | | [SECURITY.md](SECURITY.md) | Relay threat model, E2E encryption, DNS rebinding, agent auth | | [public-docs/hub/security.md](public-docs/hub/security.md) | Public Hub guide — trust boundaries, untrusted triggers, provider controls, and output authority | diff --git a/docs/architecture.md b/docs/architecture.md index 29f9c7f817..3c5678b0ab 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -66,6 +66,9 @@ WebSocket message schemas. Shared configuration schemas belong in protocol leaf retaining the daemon's runtime and wire-schema allocations. The server export tests guard these dependency trees with tree-shaking disabled to match unbundled production imports. +Terminals do not live in the daemon process: their PTYs run in a detached terminal worker that +survives daemon restarts, crashes and upgrades. See [terminal-worker.md](terminal-worker.md). + Project identity is daemon-global rather than session-owned. After registry bootstrap, the daemon's project Git observer keeps one non-recursive watch on each lexically equivalent active project root and listens only for the root `.git` entry, with a slow rescan as a missed-event fallback. It runs diff --git a/docs/design.md b/docs/design.md index a336b80a64..d115913789 100644 --- a/docs/design.md +++ b/docs/design.md @@ -203,7 +203,7 @@ Inline errors are a single sentence in `palette.red[300]` `xs`, sitting under th Page-level alerts — informational notices, success confirmations, warnings, or recoverable errors that need a small visible block on the page — use `` (`packages/app/src/components/ui/alert.tsx`). Variants: `default`, `info`, `success`, `warning`, `error`. The chrome is quiet by design: a 1px tinted border, transparent background, a small variant-tinted icon, the title in the variant accent, the description in `foregroundMuted`. Actions go in the `children` slot as ` + {terminalsSurviveRestart ? ( + + + + {t("settings.host.daemon.restart.closeTerminalsHint")} + + + + + ) : null} + {restartError ? ( + + + + ) : null} ); } @@ -848,7 +894,7 @@ function UpdateDaemonCard({ host }: { host: HostProfile }) { /> ) : null} {updateState.status === "failed" ? ( - + ({ })); const styles = StyleSheet.create((theme) => ({ - updateFailure: { + cardAlert: { marginHorizontal: theme.spacing[4], marginBottom: theme.spacing[4], }, diff --git a/packages/cli/src/commands/daemon/restart.ts b/packages/cli/src/commands/daemon/restart.ts index ef88e39a23..97e2fbf604 100644 --- a/packages/cli/src/commands/daemon/restart.ts +++ b/packages/cli/src/commands/daemon/restart.ts @@ -3,8 +3,11 @@ import { readDaemonInstance, isSameDaemonInstance, DaemonInstanceError, + retryUnconfirmedTerminalCleanup, + TerminalCleanupNotConfirmedError, } from "@getpaseo/server/daemon-control"; import { setTimeout as delay } from "node:timers/promises"; +import type { DaemonGetStatusResponse } from "@getpaseo/protocol/messages"; import { connectToDaemon } from "../../utils/client.js"; import { withOutput, type CommandOptions } from "../../output/index.js"; import { addJsonAndDaemonHostOptions } from "../../utils/command-options.js"; @@ -20,13 +23,30 @@ export function daemonRestartCommand(): Command { ), ) .option("--timeout ", "Replacement readiness deadline (default: 600)") + .option("--kill-terminals", "Close every terminal instead of keeping them across the restart") .action(withOutput(runRestartCommand)); } +type DaemonStatus = DaemonGetStatusResponse["payload"]; + export async function runRestartCommand(options: CommandOptions, _command: Command) { const target = options.daemonTarget; const deadline = Date.now() + parseTimeoutMs(options.timeout); const remaining = () => Math.max(1, deadline - Date.now()); + const killTerminals = options.killTerminals === true; + // The supervisor reaps before the respawn and the replacement retries what that left; + // anything still named by a closing record survived both. A local caller retries once more + // itself; a remote one has only what the replacement reports. + const confirmTerminalsClosed = async (status: DaemonStatus) => { + if (!killTerminals) return; + if (target.kind === "instance") { + await retryUnconfirmedTerminalCleanup(target.home); + return; + } + // Absent on daemons without terminalsSurviveRestart, which have no terminal cleanup. + const unconfirmed = status.terminalCleanupUnconfirmed ?? []; + if (unconfirmed.length > 0) throw new TerminalCleanupNotConfirmedError(unconfirmed); + }; const instance = target.kind === "instance" ? await readDaemonInstance(target.home) : null; async function checkSupervisor() { if (target.kind !== "instance") return; @@ -54,7 +74,10 @@ export async function runRestartCommand(options: CommandOptions, _command: Comma workerPid = (await client.getDaemonStatus({ timeout: remaining() })).pid; await checkSupervisor(); try { - await client.restartServer("cli_restart", undefined, { timeout: remaining() }); + await client.restartServer("cli_restart", undefined, { + timeout: remaining(), + killTerminals, + }); acknowledged = true; } catch (error) { if (!isReconnectFailure(error)) throw error; @@ -77,6 +100,7 @@ export async function runRestartCommand(options: CommandOptions, _command: Comma const status = await replacement.getDaemonStatus({ timeout: remaining() }); if (status.pid !== workerPid) { await checkSupervisor(); + await confirmTerminalsClosed(status); return { type: "single" as const, data: { diff --git a/packages/cli/src/commands/daemon/status.ts b/packages/cli/src/commands/daemon/status.ts index 14c20534e6..cd499ea6cb 100644 --- a/packages/cli/src/commands/daemon/status.ts +++ b/packages/cli/src/commands/daemon/status.ts @@ -103,6 +103,7 @@ async function probeDaemonStatus( daemonNode: status.nodePath, providers: status.providers, relay: status.relay, + ...(status.terminalsUnavailable ? { terminals: status.terminalsUnavailable } : {}), }; } } finally { diff --git a/packages/cli/src/commands/daemon/stop.test.ts b/packages/cli/src/commands/daemon/stop.test.ts new file mode 100644 index 0000000000..7ee5f930f9 --- /dev/null +++ b/packages/cli/src/commands/daemon/stop.test.ts @@ -0,0 +1,110 @@ +import { spawn, type ChildProcess } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { hostname, tmpdir } from "node:os"; +import path from "node:path"; +import { Command } from "commander"; +import { afterEach, describe, expect, test } from "vitest"; +import { createStopCommandRunner, type StopCommandDeps } from "./stop.js"; + +// The daemon the gate is checked against: a connection whose server_info may or may not +// advertise terminalsSurviveRestart, and that records any shutdown it is asked for. +function createFakeDaemon(features: { terminalsSurviveRestart?: boolean } = {}) { + const shutdownRequests: Array<{ keepTerminals?: boolean }> = []; + const deps: StopCommandDeps = { + connectToDaemon: async () => ({ + getLastServerInfoMessage: () => ({ + status: "server_info", + serverId: "fake-daemon", + version: "0.11.0", + features, + }), + shutdownServer: async (options) => { + shutdownRequests.push({ keepTerminals: options?.keepTerminals }); + return { status: "shutdown_requested", clientId: "fake-client", requestId: "fake-request" }; + }, + close: async () => undefined, + }), + }; + return { deps, shutdownRequests }; +} + +const cleanups: Array<() => void> = []; + +afterEach(() => { + for (const cleanup of cleanups.splice(0)) cleanup(); +}); + +function stop(options: Record, deps: StopCommandDeps) { + return createStopCommandRunner(deps)(options as never, new Command("stop")); +} + +// A home whose lock names a live stand-in supervisor, so the local stop path runs for real. +function homeWithRunningSupervisor(): { home: string; supervisor: ChildProcess } { + const home = mkdtempSync(path.join(tmpdir(), "paseo-stop-keep-")); + const supervisor = spawn(process.execPath, ["-e", "setInterval(() => {}, 1e9)"], { + stdio: "ignore", + }); + writeFileSync( + path.join(home, "paseo.pid"), + JSON.stringify({ + pid: supervisor.pid, + startedAt: new Date().toISOString(), + hostname: hostname(), + uid: process.getuid?.() ?? 0, + listen: "127.0.0.1:1", + }), + ); + cleanups.push(() => { + supervisor.kill("SIGKILL"); + rmSync(home, { recursive: true, force: true }); + }); + return { home, supervisor }; +} + +describe("daemon stop --keep-terminals", () => { + test("refuses an endpoint daemon that does not advertise terminalsSurviveRestart", async () => { + const daemon = createFakeDaemon(); + await expect( + stop( + { daemonTarget: { kind: "endpoint", host: "127.0.0.1:7777" }, keepTerminals: true }, + daemon.deps, + ), + ).rejects.toMatchObject({ code: "TERMINALS_KEEP_UNSUPPORTED" }); + expect(daemon.shutdownRequests).toEqual([]); + }); + + test.skipIf(process.platform === "win32")( + "refuses a local daemon without the capability and never signals its supervisor", + async () => { + const daemon = createFakeDaemon(); + const { home, supervisor } = homeWithRunningSupervisor(); + await expect( + stop({ daemonTarget: { kind: "instance", home }, keepTerminals: true }, daemon.deps), + ).rejects.toMatchObject({ code: "TERMINALS_KEEP_UNSUPPORTED" }); + expect(daemon.shutdownRequests).toEqual([]); + expect(supervisor.exitCode).toBeNull(); + expect(supervisor.signalCode).toBeNull(); + }, + ); + + test("asks a capable daemon to keep its terminals", async () => { + const daemon = createFakeDaemon({ terminalsSurviveRestart: true }); + await stop( + { daemonTarget: { kind: "endpoint", host: "127.0.0.1:7777" }, keepTerminals: true }, + daemon.deps, + ); + expect(daemon.shutdownRequests).toEqual([{ keepTerminals: true }]); + }); + + test("rejects --force with --keep-terminals before contacting the daemon", async () => { + const daemon = createFakeDaemon({ terminalsSurviveRestart: true }); + const { home } = homeWithRunningSupervisor(); + await expect( + stop( + { daemonTarget: { kind: "instance", home }, keepTerminals: true, force: true }, + daemon.deps, + ), + ).rejects.toMatchObject({ code: "INVALID_OPTIONS" }); + expect(daemon.shutdownRequests).toEqual([]); + }); +}); diff --git a/packages/cli/src/commands/daemon/stop.ts b/packages/cli/src/commands/daemon/stop.ts index f2bf8b1255..141d50965e 100644 --- a/packages/cli/src/commands/daemon/stop.ts +++ b/packages/cli/src/commands/daemon/stop.ts @@ -1,6 +1,10 @@ import { Command } from "commander"; +import { + requireTerminalKeepCapability, + type DaemonClient, +} from "@getpaseo/client/internal/daemon-client"; import { stopDaemonInstance, type DaemonInstance } from "@getpaseo/server/daemon-control"; -import { connectToDaemon } from "../../utils/client.js"; +import { connectToDaemon, type ConnectOptions } from "../../utils/client.js"; import { withOutput, type CommandOptions } from "../../output/index.js"; import { addJsonAndDaemonHostOptions } from "../../utils/command-options.js"; import { describeDaemonTarget } from "../../utils/daemon-target.js"; @@ -10,19 +14,43 @@ export function daemonStopCommand(): Command { return addJsonAndDaemonHostOptions(new Command("stop").description("Stop the selected daemon")) .option("--timeout ", "Graceful exit deadline (default: 15)") .option("--force", "Permit forced local process cleanup") + .option( + "--keep-terminals", + "Leave terminals running; the next daemon started for this home takes them back", + ) .option("--kill-timeout ", "Forced exit deadline (default: 3)") .action(withOutput(runStopCommand)); } -export async function runStopCommand(options: CommandOptions, _command: Command) { +// The daemon connection stop needs; stop.test.ts passes an in-memory adapter. +export type StopDaemonConnection = Pick< + DaemonClient, + "getLastServerInfoMessage" | "shutdownServer" | "close" +>; + +export interface StopCommandDeps { + connectToDaemon(options: ConnectOptions): Promise; +} + +export const runStopCommand = createStopCommandRunner({ connectToDaemon }); + +export function createStopCommandRunner(deps: StopCommandDeps) { + return (options: CommandOptions, _command: Command) => runStop(options, deps); +} + +async function runStop(options: CommandOptions, deps: StopCommandDeps) { const target = options.daemonTarget; const timeoutMs = parseTimeoutMs(options.timeout, 15_000); const deadline = Date.now() + timeoutMs; const remaining = () => Math.max(1, deadline - Date.now()); + const keepTerminals = options.keepTerminals === true; const requestShutdown = async (instance?: DaemonInstance) => { - const client = await connectToDaemon({ target, timeout: remaining(), instance }); + const client = await deps.connectToDaemon({ target, timeout: remaining(), instance }); try { - await client.shutdownServer({ timeout: remaining() }); + // Checked on the connection that carries the shutdown: a daemon without the capability + // would close the terminals it was asked to keep. + if (keepTerminals) requireTerminalKeepCapability(client, describeDaemonTarget(target)); + await client.shutdownServer({ timeout: remaining(), keepTerminals }); } finally { await client.close(); } @@ -32,6 +60,11 @@ export async function runStopCommand(options: CommandOptions, _command: Command) code: "INVALID_OPTIONS", message: "--force requires a local --home; an endpoint gives no remote process authority.", }; + if (options.force && keepTerminals) + throw { + code: "INVALID_OPTIONS", + message: "--force kills terminals; it cannot be combined with --keep-terminals.", + }; const result: { action: string; home?: string; @@ -44,6 +77,7 @@ export async function runStopCommand(options: CommandOptions, _command: Command) ? { ...(await stopDaemonInstance(target.home, { force: options.force === true, + keepTerminals, timeoutMs, killTimeoutMs: parseTimeoutMs(options.killTimeout, 3_000), requestShutdown, diff --git a/packages/cli/tests/45-daemon-terminals-survive.test.ts b/packages/cli/tests/45-daemon-terminals-survive.test.ts new file mode 100644 index 0000000000..7804ea6917 --- /dev/null +++ b/packages/cli/tests/45-daemon-terminals-survive.test.ts @@ -0,0 +1,324 @@ +#!/usr/bin/env npx tsx + +/** + * Terminals survive `paseo daemon restart` and `stop --keep-terminals`; `restart + * --kill-terminals`, `stop` and `stop --force` close them. Runs the built CLI and daemon in a + * throwaway home; every process it starts is stopped in `finally`. + */ + +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { existsSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtemp, mkdir, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { readDaemonInstance } from "@getpaseo/server/daemon-control"; +import { runLocalPaseo } from "./helpers/local-cli.ts"; +import { getAvailablePort } from "./helpers/network.ts"; + +if (process.platform === "win32") { + console.log("Skipping terminal survival CLI test on Windows (POSIX shell terminals)."); + process.exit(0); +} + +console.log("=== Daemon terminals survive restart ===\n"); + +const paseoHome = await mkdtemp(join(tmpdir(), "paseo-terminals-survive-")); +const workDir = join(paseoHome, "work"); +await mkdir(workDir, { recursive: true }); +const port = await getAvailablePort(); +const env = { + PASEO_HOME: paseoHome, + PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD: "0", + PASEO_DICTATION_ENABLED: "0", + PASEO_VOICE_MODE_ENABLED: "0", + // The daemon inherits this; terminals then run a plain POSIX shell with no startup files. + SHELL: "/bin/sh", +}; + +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +async function waitFor( + probe: () => Promise | T | null | false, + message: string, + timeoutMs = 30_000, +): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const value = await probe(); + if (value) return value; + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error(message); +} + +async function paseo(args: string[], expectSuccess = true) { + const result = await runLocalPaseo(args, env); + if (expectSuccess) { + assert.equal( + result.exitCode, + 0, + `paseo ${args.join(" ")} failed:\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + } + return result; +} + +async function json(args: string[]): Promise { + return JSON.parse((await paseo([...args, "--json"])).stdout) as T; +} + +async function capture(terminalId: string): Promise { + const { lines } = await json<{ lines: string[] }>(["terminal", "capture", terminalId]); + return lines.join("\n"); +} + +async function waitForScreen(terminalId: string, text: string): Promise { + await waitFor( + async () => (await capture(terminalId)).includes(text), + `terminal ${terminalId} never showed ${text}:\n${await capture(terminalId)}`, + ); +} + +async function terminalIds(): Promise { + return (await json>(["terminal", "ls", "--all"])).map((t) => t.id); +} + +// Worker records and the cleanup records a kill that could not finish leaves beside them. +function workerRecordCount(): number { + const dir = join(paseoHome, "terminal-worker"); + return existsSync(dir) + ? readdirSync(dir).filter((name) => /^(\d+|closing-.+)\.json$/.test(name)).length + : 0; +} + +// A shell terminal that reports its pid and prints a marker. +async function openTerminal(name: string): Promise<{ id: string; pid: number }> { + const terminal = await json<{ id: string }>([ + "terminal", + "create", + "--cwd", + workDir, + "--name", + name, + ]); + const pidFile = join(workDir, `${name}.pid`); + await paseo([ + "terminal", + "send-keys", + terminal.id, + "-l", + `echo $$ > '${pidFile}'; echo MARKER-${name}`, + ]); + await paseo(["terminal", "send-keys", terminal.id, "Enter"]); + const pid = await waitFor( + () => + existsSync(pidFile) && readFileSync(pidFile, "utf8").trim() + ? Number(readFileSync(pidFile, "utf8")) + : null, + `terminal ${name} never wrote its pid`, + ); + await waitForScreen(terminal.id, `MARKER-${name}`); + return { id: terminal.id, pid }; +} + +async function expectKept(terminal: { id: string; pid: number }, name: string, trigger: string) { + assert((await terminalIds()).includes(terminal.id), `${trigger}: terminal is still listed`); + assert(isAlive(terminal.pid), `${trigger}: terminal program is still alive`); + assert((await capture(terminal.id)).includes(`MARKER-${name}`), `${trigger}: marker is kept`); + await paseo(["terminal", "send-keys", terminal.id, "-l", `echo after-${trigger}`]); + await paseo(["terminal", "send-keys", terminal.id, "Enter"]); + await waitForScreen(terminal.id, `\nafter-${trigger}`); +} + +async function expectClosed(pid: number, trigger: string) { + await waitFor(() => !isAlive(pid), `${trigger}: terminal program ${pid} is still alive`, 15_000); +} + +// A zombie: a real process with a birth time that SIGKILL cannot remove while its parent (an +// exec'd `sleep` that never waits) lives. Cleanup can only report it as still alive. +async function startZombie(): Promise<{ zombiePid: number; parentPid: number }> { + const parent = spawn("/bin/sh", ["-c", "sleep 0 & echo $!; exec sleep 600"], { + stdio: ["ignore", "pipe", "ignore"], + }); + const zombiePid = await new Promise((resolve) => { + parent.stdout!.once("data", (chunk: Buffer) => resolve(Number(chunk.toString().trim()))); + }); + parent.unref(); + await waitFor( + async () => (await psField(zombiePid, "stat")).startsWith("Z"), + `process ${zombiePid} never became a zombie`, + ); + return { zombiePid, parentPid: parent.pid! }; +} + +async function psField(pid: number, field: "stat" | "lstart"): Promise { + const child = spawn("ps", ["-o", `${field}=`, "-p", String(pid)], { + env: { ...process.env, LC_ALL: "C" }, + stdio: ["ignore", "pipe", "ignore"], + }); + let out = ""; + child.stdout.on("data", (chunk: Buffer) => (out += chunk.toString())); + await new Promise((resolve) => child.once("close", resolve)); + return out.trim(); +} + +async function goneProcessPid(): Promise { + const child = spawn(process.execPath, ["-e", ""], { stdio: "ignore" }); + await new Promise((resolve) => child.once("exit", resolve)); + return child.pid!; +} + +// A record as a dead terminal worker leaves it, naming one terminal program that is a zombie. +async function plantRecordNaming(programPid: number): Promise { + const workerPid = await goneProcessPid(); + const record = { + recordId: "planted-record", + pid: workerPid, + bootId: null, + createdAt: new Date().toISOString(), + identity: { startedAt: "Thu Jan 1 00:00:00 1970", commandLine: "planted-worker" }, + protocolVersion: 1, + paseoVersion: "0.0.0-test", + socketPath: join(paseoHome, "planted-worker.sock"), + secret: "planted-secret", + terminals: [ + { id: "planted-terminal", pid: programPid, startedAt: await psField(programPid, "lstart") }, + ], + }; + writeFileSync( + join(paseoHome, "terminal-worker", `${workerPid}.json`), + `${JSON.stringify(record, null, 2)}\n`, + { mode: 0o600 }, + ); +} + +const trackedPids: number[] = []; + +try { + for (const [field, value] of [ + ["daemon.listen", `127.0.0.1:${port}`], + ["daemon.relay.enabled", "false"], + ["features.dictation.enabled", "false"], + ["features.voiceMode.enabled", "false"], + ] as const) { + await paseo(["daemon", "config", "set", field, value]); + } + await paseo(["daemon", "start"]); + + console.log("Test 1: daemon restart keeps the terminal"); + const kept = await openTerminal("kept"); + trackedPids.push(kept.pid); + await paseo(["daemon", "restart", "--home", paseoHome]); + await expectKept(kept, "kept", "restart"); + console.log("✓ same id, same program, marker and input after restart\n"); + + console.log("Test 2: stop --keep-terminals, then start, gives the terminal back"); + const forceAndKeep = await paseo( + ["daemon", "stop", "--home", paseoHome, "--force", "--keep-terminals"], + false, + ); + assert.notEqual(forceAndKeep.exitCode, 0, "--force with --keep-terminals is refused"); + await paseo(["daemon", "stop", "--home", paseoHome, "--keep-terminals"]); + assert.equal(await readDaemonInstance(paseoHome), null, "the supervisor stopped"); + assert(isAlive(kept.pid), "the program outlives the stopped daemon"); + await paseo(["daemon", "start"]); + await expectKept(kept, "kept", "keep-stop"); + console.log("✓ adopted with the same id and still usable\n"); + + console.log("Test 3: a plain stop with no daemon running closes kept terminals"); + await paseo(["daemon", "stop", "--home", paseoHome, "--keep-terminals"]); + assert(isAlive(kept.pid), "kept again"); + await paseo(["daemon", "stop", "--home", paseoHome]); + await expectClosed(kept.pid, "stop without a supervisor"); + assert.equal(workerRecordCount(), 0, "no worker record is left"); + console.log("✓ closed\n"); + + console.log("Test 4: restart --kill-terminals closes the terminal"); + await paseo(["daemon", "start"]); + const killedByRestart = await openTerminal("restart-kill"); + trackedPids.push(killedByRestart.pid); + await paseo(["daemon", "restart", "--home", paseoHome, "--kill-terminals"]); + await expectClosed(killedByRestart.pid, "restart --kill-terminals"); + assert.deepEqual(await terminalIds(), [], "no terminal is listed after the kill restart"); + console.log("✓ closed\n"); + + console.log("Test 5: daemon stop closes the terminal and its worker"); + const killedByStop = await openTerminal("stop-kill"); + trackedPids.push(killedByStop.pid); + await paseo(["daemon", "stop", "--home", paseoHome]); + await expectClosed(killedByStop.pid, "stop"); + assert.equal(workerRecordCount(), 0, "no worker record is left"); + console.log("✓ closed\n"); + + console.log("Test 6: daemon stop --force closes the terminal"); + await paseo(["daemon", "start"]); + const killedByForce = await openTerminal("force-kill"); + trackedPids.push(killedByForce.pid); + await paseo(["daemon", "stop", "--home", paseoHome, "--force"]); + await expectClosed(killedByForce.pid, "stop --force"); + assert.equal(workerRecordCount(), 0, "no worker record is left"); + console.log("✓ closed\n"); + + const endpoint = `127.0.0.1:${port}`; + console.log("Test 7: restart --host --kill-terminals closes the terminal"); + await paseo(["daemon", "start"]); + const killedByEndpointRestart = await openTerminal("endpoint-restart-kill"); + trackedPids.push(killedByEndpointRestart.pid); + await paseo(["daemon", "restart", "--host", endpoint, "--kill-terminals"]); + await expectClosed(killedByEndpointRestart.pid, "restart --host --kill-terminals"); + assert.deepEqual(await terminalIds(), [], "no terminal is listed after the kill restart"); + console.log("✓ closed and confirmed by the replacement\n"); + + console.log("Test 8: restart --host --kill-terminals reports a process cleanup could not end"); + const zombie = await startZombie(); + trackedPids.push(zombie.parentPid); + await plantRecordNaming(zombie.zombiePid); + const unconfirmed = await paseo( + ["daemon", "restart", "--host", endpoint, "--kill-terminals"], + false, + ); + assert.notEqual(unconfirmed.exitCode, 0, "the kill restart fails"); + const output = unconfirmed.stdout + unconfirmed.stderr; + assert(output.includes("Terminal cleanup was not confirmed"), `names the failure:\n${output}`); + assert(output.includes(`pid ${zombie.zombiePid}`), `names the surviving pid:\n${output}`); + process.kill(zombie.parentPid, "SIGKILL"); + await expectClosed(zombie.zombiePid, "zombie reaped once its parent exits"); + await paseo(["daemon", "stop", "--home", paseoHome]); + assert.equal(workerRecordCount(), 0, "the next stop retries the cleanup and retires the record"); + console.log("✓ reported, then retried by stop\n"); + + console.log("Test 9: restart --host --kill-terminals reports a worker record it cannot read"); + await paseo(["daemon", "start"]); + // A kill cleanup cannot check what an unreadable record names, and leaves nothing else + // behind, so the replacement daemon must still report it. + const unreadablePath = join(paseoHome, "terminal-worker", `${await goneProcessPid()}.json`); + writeFileSync(unreadablePath, "{ not json", { mode: 0o600 }); + const unreadable = await paseo( + ["daemon", "restart", "--host", endpoint, "--kill-terminals"], + false, + ); + assert.notEqual(unreadable.exitCode, 0, "the kill restart fails"); + const unreadableOutput = unreadable.stdout + unreadable.stderr; + assert(unreadableOutput.includes(unreadablePath), `names the record:\n${unreadableOutput}`); + assert(existsSync(unreadablePath), "the unreadable record is kept"); + rmSync(unreadablePath); + await paseo(["daemon", "stop", "--home", paseoHome]); + assert.equal(workerRecordCount(), 0, "no worker record is left"); + console.log("✓ reported with the record's path\n"); +} finally { + await runLocalPaseo(["daemon", "stop", "--home", paseoHome, "--force"], env); + for (const pid of trackedPids) { + if (isAlive(pid)) process.kill(pid, "SIGKILL"); + } + await rm(paseoHome, { recursive: true, force: true }); +} + +console.log("=== Daemon terminals survive restart passed ==="); diff --git a/packages/cli/tests/46-daemon-terminal-worker-upgrade.test.ts b/packages/cli/tests/46-daemon-terminal-worker-upgrade.test.ts new file mode 100644 index 0000000000..ff8ef6bc6f --- /dev/null +++ b/packages/cli/tests/46-daemon-terminal-worker-upgrade.test.ts @@ -0,0 +1,527 @@ +#!/usr/bin/env npx tsx + +/** + * A terminal worker survives an upgrade of the installation that started it. Two disposable + * installations A and B (built dist, real copies of @getpaseo/server, @getpaseo/cli and + * node-pty, every other dependency a symlink into this repo) each get their own CLI shim, so + * each daemon's PASEO_CLI is a distinct path: + * + * 1. A daemon from A runs its worker from a copy in $PASEO_HOME/terminal-worker/. After + * `stop --keep-terminals` and moving A's node-pty, worker bundle and shell integration away, + * a daemon from B adopts the same worker and opens a new, working terminal in it with B's + * environment and CLI, while the first terminal keeps A's. + * 2. Worker protocol versions (PASEO_TERMINAL_WORKER_PROTOCOL_OVERRIDE): a worker one version + * old drains (keeps its terminal, new ones open in a second worker, it exits after its last + * terminal and its copy is deleted); one two versions old is shut down; one from a newer + * Paseo is left running, reported as terminal_worker_incompatible, and closed by + * `daemon stop`. + * + * Every process it starts is stopped in `finally`. + */ + +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { + chmodSync, + cpSync, + existsSync, + mkdirSync, + readdirSync, + readFileSync, + renameSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { getAvailablePort } from "./helpers/network.ts"; + +if (process.platform === "win32") { + console.log("Skipping terminal worker upgrade CLI test on Windows (POSIX shell terminals)."); + process.exit(0); +} + +console.log("=== Terminal worker survives an upgrade ===\n"); + +const repoRoot = join(import.meta.dirname, "..", "..", ".."); +const root = await mkdtemp(join(tmpdir(), "paseo-worker-upgrade-")); +const paseoHome = join(root, "home"); +const cliHome = join(root, "os-home"); +const workDir = join(root, "work"); +for (const dir of [paseoHome, cliHome, workDir]) mkdirSync(dir, { recursive: true }); +const port = await getAvailablePort(); +const zsh = ["/bin/zsh", "/usr/bin/zsh"].find((candidate) => existsSync(candidate)) ?? null; + +interface Installation { + name: string; + dir: string; + version: string; + shim: string; + serverDir: string; +} + +// --------------------------------------------------------------------------------------------- +// Installation fixtures +// --------------------------------------------------------------------------------------------- + +function linkEntries(source: string, target: string, taken: Set): void { + if (!existsSync(source)) return; + for (const name of readdirSync(source)) { + if (name.startsWith(".")) continue; + if (name.startsWith("@")) { + mkdirSync(join(target, name), { recursive: true }); + for (const scoped of readdirSync(join(source, name))) { + const key = `${name}/${scoped}`; + if (taken.has(key)) continue; + taken.add(key); + symlinkSync(join(source, name, scoped), join(target, name, scoped)); + } + continue; + } + if (taken.has(name)) continue; + taken.add(name); + symlinkSync(join(source, name), join(target, name)); + } +} + +function createInstallation(name: string, version: string): Installation { + const dir = join(root, `install-${name}`); + const modules = join(dir, "node_modules"); + mkdirSync(join(modules, "@getpaseo"), { recursive: true }); + + // Real copies: the server (its version marks the installation), the CLI (so it resolves this + // installation's server), and node-pty (so moving it away breaks only this installation). + const serverDir = join(modules, "@getpaseo", "server"); + const serverPackage = JSON.parse( + readFileSync(join(repoRoot, "packages/server/package.json"), "utf8"), + ) as Record; + mkdirSync(join(serverDir, "dist"), { recursive: true }); + writeFileSync( + join(serverDir, "package.json"), + `${JSON.stringify({ ...serverPackage, version }, null, 2)}\n`, + ); + for (const part of ["server", "src", "scripts"]) { + cpSync(join(repoRoot, "packages/server/dist", part), join(serverDir, "dist", part), { + recursive: true, + }); + } + const cliDir = join(modules, "@getpaseo", "cli"); + mkdirSync(cliDir, { recursive: true }); + cpSync(join(repoRoot, "packages/cli/package.json"), join(cliDir, "package.json")); + cpSync(join(repoRoot, "packages/cli/dist"), join(cliDir, "dist"), { recursive: true }); + cpSync(join(repoRoot, "packages/server/node_modules/node-pty"), join(modules, "node-pty"), { + recursive: true, + }); + + const taken = new Set(["@getpaseo/server", "@getpaseo/cli", "node-pty"]); + for (const source of [ + "packages/server/node_modules", + "packages/cli/node_modules", + "node_modules", + ]) { + linkEntries(join(repoRoot, source), modules, taken); + } + + const shim = join(dir, "bin", "paseo"); + mkdirSync(join(dir, "bin")); + writeFileSync( + shim, + `#!/bin/sh\nexec "${process.execPath}" "${join(cliDir, "dist", "index.js")}" "$@"\n`, + ); + chmodSync(shim, 0o755); + return { name, dir, version, shim, serverDir }; +} + +// The runtime files an upgrade replaces: node-pty, the worker bundle, the shell integration. +function moveRuntimeAway(installation: Installation): () => void { + const away = join(installation.dir, "moved-away"); + const moves = [ + join(installation.dir, "node_modules", "node-pty"), + join(installation.serverDir, "dist", "server", "terminal-worker"), + join(installation.serverDir, "dist", "server", "terminal", "shell-integration"), + ].map((from, index) => ({ from, to: join(away, String(index)) })); + mkdirSync(away, { recursive: true }); + for (const move of moves) renameSync(move.from, move.to); + return () => { + for (const move of moves) renameSync(move.to, move.from); + }; +} + +// --------------------------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------------------------- + +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +async function waitFor( + probe: () => Promise | T | null | false, + message: () => string | Promise, + timeoutMs = 30_000, +): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const value = await probe(); + if (value) return value; + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error(await message()); +} + +interface RunResult { + exitCode: number; + stdout: string; + stderr: string; +} + +function run( + installation: Installation, + args: string[], + extraEnv: Record = {}, +): Promise { + const env = { + ...Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith("PASEO_"))), + HOME: cliHome, + USERPROFILE: cliHome, + PASEO_HOME: paseoHome, + PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD: "0", + PASEO_DICTATION_ENABLED: "0", + PASEO_VOICE_MODE_ENABLED: "0", + // Each daemon hands its terminals its own installation's CLI. + PASEO_CLI: installation.shim, + PASEO_TEST_INSTALL: installation.name, + SHELL: "/bin/sh", + ...extraEnv, + }; + return new Promise((resolve) => { + execFile(installation.shim, args, { env, timeout: 120_000 }, (error, stdout, stderr) => { + let exitCode = 0; + if (error) exitCode = typeof error.code === "number" ? error.code : 1; + resolve({ exitCode, stdout, stderr }); + }); + }); +} + +async function paseo( + installation: Installation, + args: string[], + extraEnv: Record = {}, +): Promise { + const result = await run(installation, args, extraEnv); + assert.equal( + result.exitCode, + 0, + `paseo ${args.join(" ")} (${installation.name}) failed:\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + return result; +} + +async function json(installation: Installation, args: string[]): Promise { + return JSON.parse((await paseo(installation, [...args, "--json"])).stdout) as T; +} + +async function capture(installation: Installation, terminalId: string): Promise { + const { lines } = await json<{ lines: string[] }>(installation, [ + "terminal", + "capture", + terminalId, + ]); + return lines.join("\n"); +} + +async function type(installation: Installation, terminalId: string, line: string) { + await paseo(installation, ["terminal", "send-keys", terminalId, "-l", line]); + await paseo(installation, ["terminal", "send-keys", terminalId, "Enter"]); +} + +async function waitForScreen(installation: Installation, terminalId: string, text: string) { + await waitFor( + async () => (await capture(installation, terminalId)).includes(text), + async () => + `terminal ${terminalId} never showed ${text}:\n${await capture(installation, terminalId)}`, + ); +} + +// Opens a terminal, records its shell's pid, and prints a marker. +async function openTerminal( + installation: Installation, + name: string, +): Promise<{ id: string; pid: number }> { + const terminal = await json<{ id: string }>(installation, [ + "terminal", + "create", + "--cwd", + workDir, + "--name", + name, + ]); + const pidFile = join(workDir, `${name}.pid`); + await type(installation, terminal.id, `echo $$ > '${pidFile}'; echo MARKER-${name}`); + const pid = await waitFor( + () => + existsSync(pidFile) && readFileSync(pidFile, "utf8").trim() + ? Number(readFileSync(pidFile, "utf8")) + : null, + () => `terminal ${name} never wrote its pid`, + ); + await waitForScreen(installation, terminal.id, `MARKER-${name}`); + trackedPids.push(pid); + return { id: terminal.id, pid }; +} + +// The value of `$variable` in the terminal's shell, written to a file (a long path would wrap +// on screen). +async function echoVariable( + installation: Installation, + terminalId: string, + label: string, + variable: string, +): Promise { + const valueFile = join(workDir, `${label}.value`); + await type( + installation, + terminalId, + `printf '%s' "$${variable}" > '${valueFile}.tmp' && mv '${valueFile}.tmp' '${valueFile}'`, + ); + await waitFor( + () => existsSync(valueFile), + async () => + `terminal ${terminalId} never wrote ${label}:\n${await capture(installation, terminalId)}`, + ); + return readFileSync(valueFile, "utf8"); +} + +interface WorkerRecordFile { + pid: number; + protocolVersion: unknown; + runtime?: string | null; + terminals: Array<{ id: string; pid: number }>; +} + +function workerRecords(): WorkerRecordFile[] { + const dir = join(paseoHome, "terminal-worker"); + if (!existsSync(dir)) return []; + return readdirSync(dir) + .filter((name) => /^\d+\.json$/.test(name)) + .map((name) => JSON.parse(readFileSync(join(dir, name), "utf8")) as WorkerRecordFile); +} + +function runtimeCopies(): string[] { + const dir = join(paseoHome, "terminal-worker"); + if (!existsSync(dir)) return []; + return readdirSync(dir).filter((name) => + existsSync(join(dir, name, "paseo-terminal-worker-runtime.json")), + ); +} + +function commandLineOf(pid: number): Promise { + return new Promise((resolve) => { + execFile("ps", ["-ww", "-o", "command=", "-p", String(pid)], (_error, stdout) => + resolve(stdout.trim()), + ); + }); +} + +async function terminalIds(installation: Installation): Promise { + return (await json>(installation, ["terminal", "ls", "--all"])).map( + (terminal) => terminal.id, + ); +} + +async function startDaemon(installation: Installation, extraEnv: Record = {}) { + await paseo(installation, ["daemon", "start"], extraEnv); +} + +async function stopDaemon(installation: Installation, keepTerminals = false) { + await paseo(installation, [ + "daemon", + "stop", + "--home", + paseoHome, + ...(keepTerminals ? ["--keep-terminals"] : []), + ]); +} + +// A terminal in a worker reporting `protocolVersion`, left running by `stop --keep-terminals`. +async function leaveWorkerWithProtocol( + installation: Installation, + protocolVersion: number, + name: string, +): Promise<{ terminal: { id: string; pid: number }; workerPid: number; runtime: string }> { + await startDaemon(installation, { + PASEO_TERMINAL_WORKER_PROTOCOL_OVERRIDE: String(protocolVersion), + }); + const terminal = await openTerminal(installation, name); + const [worker] = workerRecords(); + assert(worker, "the daemon started a worker"); + assert.equal(worker.protocolVersion, protocolVersion, "the worker reports the override"); + await stopDaemon(installation, true); + return { terminal, workerPid: worker.pid, runtime: worker.runtime! }; +} + +const trackedPids: number[] = []; +const a = createInstallation("A", "0.98.0"); +const b = createInstallation("B", "0.99.0"); +let restoreA: (() => void) | null = null; +let passed = false; + +try { + for (const [field, value] of [ + ["daemon.listen", `127.0.0.1:${port}`], + ["daemon.relay.enabled", "false"], + ["features.dictation.enabled", "false"], + ["features.voiceMode.enabled", "false"], + ] as const) { + await paseo(a, ["daemon", "config", "set", field, value]); + } + + console.log("Test 1: a daemon from A runs its worker from a copy in the home"); + await startDaemon(a); + const t1 = await openTerminal(a, "t1"); + const [firstWorker] = workerRecords(); + assert(firstWorker, "one worker runs"); + assert.match(firstWorker.runtime ?? "", /^0\.98\.0-[0-9a-f]{12}$/); + const copyA = join(paseoHome, "terminal-worker", firstWorker.runtime!); + assert( + (await commandLineOf(firstWorker.pid)).includes(join(copyA, "terminal-worker.mjs")), + "the worker runs the copy's bundle", + ); + assert.equal(await echoVariable(a, t1.id, "CLI1", "PASEO_CLI"), a.shim, "T1 has A's CLI"); + console.log("✓ worker runs from", copyA, "\n"); + + console.log("Test 2: B adopts the worker after A's runtime files are gone"); + await stopDaemon(a, true); + restoreA = moveRuntimeAway(a); + await startDaemon(b, zsh ? { SHELL: zsh } : {}); + assert.deepEqual( + workerRecords().map((record) => record.pid), + [firstWorker.pid], + "B adopted the same worker instead of starting one", + ); + assert((await terminalIds(b)).includes(t1.id), "T1 is listed by B"); + const t2 = await openTerminal(b, "t2"); + assert.equal(await echoVariable(b, t2.id, "INSTALL2", "PASEO_TEST_INSTALL"), "B"); + assert.equal(await echoVariable(b, t2.id, "CLI2", "PASEO_CLI"), b.shim, "T2 has B's CLI"); + if (zsh) { + // Creating a zsh terminal reads Paseo's integration from the worker's copy (A's is gone); + // a missing file would have failed the create. + const zshVersion = await echoVariable(b, t2.id, "ZSH", "ZSH_VERSION"); + assert(zshVersion.length > 0, "T2 runs zsh with Paseo's shell integration"); + } else { + console.log(" (zsh is not installed: the shell-integration check is skipped)"); + } + assert.equal(await echoVariable(b, t1.id, "CLI1AGAIN", "PASEO_CLI"), a.shim, "T1 keeps A's CLI"); + console.log("✓ same worker, T2 uses B's environment, T1 still works\n"); + + console.log("Test 3: daemon stop closes both terminals and deletes the copy"); + await stopDaemon(b); + await waitFor( + () => !isAlive(t1.pid) && !isAlive(t2.pid) && !isAlive(firstWorker.pid), + () => "terminals or worker outlived daemon stop", + ); + assert.deepEqual(workerRecords(), [], "no worker record is left"); + assert.deepEqual(runtimeCopies(), [], "no copy is left"); + restoreA(); + restoreA = null; + console.log("✓ closed and cleaned up\n"); + + console.log("Test 4: a worker one protocol version old drains"); + const old = await leaveWorkerWithProtocol(a, 0, "t3"); + await startDaemon(b); + const workers = workerRecords(); + assert.equal(workers.length, 2, "the old worker drains beside a new one"); + const current = workers.find((record) => record.pid !== old.workerPid)!; + assert.equal(current.protocolVersion, 1); + assert.match(current.runtime ?? "", /^0\.99\.0-/); + assert((await terminalIds(b)).includes(old.terminal.id), "the old worker's terminal is listed"); + await type(b, old.terminal.id, "echo still-served"); + await waitForScreen(b, old.terminal.id, "\nstill-served"); + const t4 = await openTerminal(b, "t4"); + assert.deepEqual( + workerRecords() + .find((record) => record.pid === current.pid)! + .terminals.map((terminal) => terminal.id), + [t4.id], + "new terminals open in the current worker", + ); + await paseo(b, ["terminal", "kill", old.terminal.id]); + await waitFor( + () => !isAlive(old.workerPid), + () => `the drained worker ${old.workerPid} did not exit`, + ); + await waitFor( + () => !runtimeCopies().includes(old.runtime), + () => `the drained worker's copy ${old.runtime} was not deleted: ${runtimeCopies().join(", ")}`, + ); + assert.deepEqual(runtimeCopies(), [current.runtime], "the current worker's copy stays"); + await type(b, t4.id, "echo after-drain"); + await waitForScreen(b, t4.id, "\nafter-drain"); + await stopDaemon(b); + console.log("✓ drained, exited, copy deleted\n"); + + console.log("Test 5: a worker two protocol versions old is shut down"); + const obsolete = await leaveWorkerWithProtocol(a, -1, "t5"); + await startDaemon(b); + await waitFor( + () => !isAlive(obsolete.workerPid) && !isAlive(obsolete.terminal.pid), + () => "the obsolete worker or its terminal survived", + ); + assert(!(await terminalIds(b)).includes(obsolete.terminal.id), "its terminal is gone"); + assert.equal(workerRecords().length, 1, "one current worker runs"); + await stopDaemon(b); + console.log("✓ shut down\n"); + + console.log("Test 6: a worker from a newer Paseo is left running and reported"); + const newer = await leaveWorkerWithProtocol(a, 2, "t6"); + await startDaemon(b); + const listed = await run(b, ["terminal", "ls", "--all", "--json"]); + assert.notEqual(listed.exitCode, 0, "terminal ls fails"); + assert.match(listed.stdout + listed.stderr, /held by a newer Paseo \(v0\.98\.0/); + const created = await run(b, ["terminal", "create", "--cwd", workDir, "--json"]); + assert.notEqual(created.exitCode, 0, "terminal create fails"); + assert.match(created.stdout + created.stderr, /held by a newer Paseo/); + const status = await paseo(b, ["daemon", "status", "--home", paseoHome]); + assert.match(status.stdout, /terminals: Terminals are held by a newer Paseo/); + assert(isAlive(newer.terminal.pid), "its program keeps running"); + assert.deepEqual( + workerRecords().map((record) => record.pid), + [newer.workerPid], + "no second worker starts", + ); + await stopDaemon(b); + await waitFor( + () => !isAlive(newer.workerPid) && !isAlive(newer.terminal.pid), + () => "daemon stop left the newer worker or its terminal running", + ); + assert.deepEqual(workerRecords(), [], "no worker record is left"); + assert.deepEqual(runtimeCopies(), [], "no copy is left"); + console.log("✓ reported, untouched, closed by stop\n"); + passed = true; +} finally { + if (!passed) { + for (const log of ["daemon.log", "terminal-worker.log"]) { + const logPath = join(paseoHome, log); + if (existsSync(logPath)) { + console.error(`--- ${log} (last 60 lines) ---`); + console.error(readFileSync(logPath, "utf8").split("\n").slice(-60).join("\n")); + } + } + } + restoreA?.(); + for (const installation of [b, a]) { + await run(installation, ["daemon", "stop", "--home", paseoHome, "--force"]); + } + for (const pid of trackedPids) { + if (isAlive(pid)) process.kill(pid, "SIGKILL"); + } + await rm(root, { recursive: true, force: true }); +} + +console.log("=== Terminal worker survives an upgrade passed ==="); diff --git a/packages/client/src/daemon-client.ts b/packages/client/src/daemon-client.ts index bca3d00301..d4b0f28a39 100644 --- a/packages/client/src/daemon-client.ts +++ b/packages/client/src/daemon-client.ts @@ -781,6 +781,9 @@ type ShutdownRequestedStatusPayload = z.infer, + target: string, +): void { + const serverInfo = client.getLastServerInfoMessage(); + if (serverInfo?.features?.terminalsSurviveRestart !== true) { + throw new TerminalKeepUnsupportedError(target, serverInfo?.version ?? null); + } +} + function buildFetchAgentsRequest( options: FetchAgentsOptions | undefined, requestId: string, @@ -4071,13 +4104,14 @@ export class DaemonClient { async restartServer( reason?: string, requestId?: string, - options?: { timeout?: number }, + options?: { timeout?: number; killTerminals?: boolean }, ): Promise { const resolvedRequestId = this.createRequestId(requestId); const message = SessionInboundMessageSchema.parse({ type: "restart_server_request", ...(reason && reason.trim().length > 0 ? { reason } : {}), requestId: resolvedRequestId, + ...(options?.killTerminals ? { killTerminals: true } : {}), }); return this.sendRequest({ requestId: resolvedRequestId, @@ -4105,6 +4139,7 @@ export class DaemonClient { const message = SessionInboundMessageSchema.parse({ type: "shutdown_server_request", requestId: resolvedRequestId, + ...(options?.keepTerminals ? { keepTerminals: true } : {}), }); return this.sendRequest({ requestId: resolvedRequestId, diff --git a/packages/desktop/src/daemon/cli/external.ts b/packages/desktop/src/daemon/cli/external.ts index 6c9deeccb1..4a55adff1c 100644 --- a/packages/desktop/src/daemon/cli/external.ts +++ b/packages/desktop/src/daemon/cli/external.ts @@ -4,6 +4,31 @@ import type { NodeEntrypointInvocation } from "../node-entrypoint-launcher.js"; import { createNodeEntrypointInvocation } from "../runtime-paths.js"; import { resolveExternalCliEntrypoint } from "./entrypoints.js"; +/** A CLI command that exited non-zero, with the `--json` error it printed when there was one. */ +export class ExternalCliCommandError extends Error { + readonly code: string | null; + readonly error: Record | null; + + constructor(message: string, error: Record | null) { + super(message); + this.name = "ExternalCliCommandError"; + this.error = error; + this.code = typeof error?.code === "string" ? error.code : null; + } +} + +// `--json` commands print `{ "error": { code, message, ... } }` on stderr when they fail. +function parseJsonCliError(stderr: string): Record | null { + try { + const parsed = JSON.parse(stderr) as { error?: unknown }; + return typeof parsed.error === "object" && parsed.error !== null + ? (parsed.error as Record) + : null; + } catch { + return null; + } +} + function createExternalCliInvocation(args: string[]): NodeEntrypointInvocation { return createNodeEntrypointInvocation({ entrypoint: resolveExternalCliEntrypoint(), @@ -85,7 +110,10 @@ export async function runExternalCliJsonCommand(args: string[]): Promise ({ paseoHome: "", @@ -73,7 +79,8 @@ vi.mock("./runtime-paths.js", () => ({ })), })); -vi.mock("./cli/external.js", () => ({ +vi.mock("./cli/external.js", async (importOriginal) => ({ + ...(await importOriginal>()), runExternalCliJsonCommand: mocks.runExternalCliJsonCommand, runExternalCliTextCommand: mocks.runExternalCliTextCommand, })); @@ -176,3 +183,107 @@ describe("daemon-manager commands", () => { expect(await handler({ listen: "localhost:6799" })).toBeNull(); }); }); + +describe("desktop stops and terminals", () => { + const reasons: DesktopDaemonStopReason[] = [ + "manual_ipc", + "settings", + "host_remove", + "quit", + "app_update", + "version_mismatch", + "restart", + ]; + + it("keeps terminals only for an app update or version-mismatch restart, never on Windows", () => { + for (const platform of ["darwin", "linux"] as const) { + expect(reasons.filter((reason) => keepsTerminalsOnStop(reason, platform))).toEqual([ + "app_update", + "version_mismatch", + ]); + } + expect(reasons.filter((reason) => keepsTerminalsOnStop(reason, "win32"))).toEqual([]); + }); + + function stopHarness(runCliJson: (args: string[]) => Promise) { + const stops: Array<{ keepTerminals: boolean | undefined }> = []; + const logged: string[] = []; + const instance = { + pid: 4242, + startedAt: "2026-10-21T00:00:00.000Z", + hostname: "host", + uid: 501, + listen: "127.0.0.1:6799", + }; + const stopInstance = (async ( + _home: string, + options: { + keepTerminals?: boolean; + requestShutdown?: (ready: { listen: string }) => Promise; + }, + ) => { + stops.push({ keepTerminals: options.keepTerminals }); + await options.requestShutdown?.({ listen: instance.listen }); + return { action: "stopped", pid: instance.pid, forced: false, usedLifecycleRpc: true }; + }) as unknown as NonNullable[0]["stopInstance"]>; + return { + stops, + logged, + input: { + home: "/tmp/paseo-home", + instance, + stopInstance, + runCliJson, + log: (message: string) => logged.push(message), + } as Omit[0], "reason" | "keepTerminals">, + }; + } + + it("asks the daemon to keep its terminals on an app update", async () => { + const calls: string[][] = []; + const harness = stopHarness(async (args) => { + calls.push(args); + return { action: "shutdown_requested" }; + }); + + await stopCapturedDaemon({ ...harness.input, reason: "app_update", keepTerminals: true }); + + expect(harness.stops).toEqual([{ keepTerminals: true }]); + expect(calls).toEqual([ + ["daemon", "stop", "--host", "127.0.0.1:6799", "--keep-terminals", "--json"], + ]); + }); + + it("stops a daemon without the capability with its terminals closed, and says so", async () => { + const calls: string[][] = []; + const harness = stopHarness(async (args) => { + calls.push(args); + if (args.includes("--keep-terminals")) { + throw new ExternalCliCommandError("refused", { + code: "TERMINALS_KEEP_UNSUPPORTED", + version: "0.11.0", + }); + } + return { action: "shutdown_requested" }; + }); + + await stopCapturedDaemon({ ...harness.input, reason: "app_update", keepTerminals: true }); + + expect(harness.stops).toEqual([{ keepTerminals: true }, { keepTerminals: false }]); + expect(calls[1]).toEqual(["daemon", "stop", "--host", "127.0.0.1:6799", "--json"]); + expect(harness.logged).toContain( + "daemon v0.11.0 cannot keep terminals; stopping it and closing them", + ); + }); + + it("does not retry a keep stop that failed for another reason", async () => { + const harness = stopHarness(async () => { + throw new ExternalCliCommandError("unreachable", { code: "DAEMON_UNREACHABLE" }); + }); + + await expect( + stopCapturedDaemon({ ...harness.input, reason: "app_update", keepTerminals: true }), + ).rejects.toThrow("unreachable"); + expect(harness.stops).toEqual([{ keepTerminals: true }]); + }); +}); diff --git a/packages/desktop/src/daemon/daemon-manager.ts b/packages/desktop/src/daemon/daemon-manager.ts index 5f3f5394c6..769b5513f5 100644 --- a/packages/desktop/src/daemon/daemon-manager.ts +++ b/packages/desktop/src/daemon/daemon-manager.ts @@ -37,7 +37,11 @@ import { closeLocalTransportSession, } from "./local-transport.js"; import { createNodeEntrypointInvocation, resolveDaemonRunnerEntrypoint } from "./runtime-paths.js"; -import { runExternalCliJsonCommand, runExternalCliTextCommand } from "./cli/external.js"; +import { + ExternalCliCommandError, + runExternalCliJsonCommand, + runExternalCliTextCommand, +} from "./cli/external.js"; import { createDesktopSettingsCommandHandlers, type DesktopCommandHandler, @@ -348,18 +352,84 @@ export async function stopDesktopDaemon( "Daemon changed since confirmation; inspect its current home and PID before stopping it.", ); if (!instance || (!owned && !explicit)) return resolveDesktopDaemonStatus(); - logDesktopDaemonLifecycle("stopping captured supervisor", { reason, pid: instance.pid, owned }); - await stopDaemonInstance(home, { - instance, - timeoutMs: 15_000, - requestShutdown: async (ready) => { - await runExternalCliJsonCommand(["daemon", "stop", "--host", ready.listen, "--json"]); - }, + const keepTerminals = keepsTerminalsOnStop(reason, process.platform); + logDesktopDaemonLifecycle("stopping captured supervisor", { + reason, + pid: instance.pid, + owned, + keepTerminals, }); + await stopCapturedDaemon({ home, instance, reason, keepTerminals }); if (owned) ownedLaunch = null; return resolveDesktopDaemonStatus(); } +export interface StopCapturedDaemonInput { + home: string; + instance: DaemonInstance; + reason: DesktopDaemonStopReason; + keepTerminals: boolean; + stopInstance?: typeof stopDaemonInstance; + runCliJson?: typeof runExternalCliJsonCommand; + log?: typeof logDesktopDaemonLifecycle; +} + +/** Stops the supervisor desktop captured, keeping its terminals when asked and supported. */ +export async function stopCapturedDaemon(input: StopCapturedDaemonInput): Promise { + const stopInstance = input.stopInstance ?? stopDaemonInstance; + const runCliJson = input.runCliJson ?? runExternalCliJsonCommand; + const logLifecycle = input.log ?? logDesktopDaemonLifecycle; + const stop = (keep: boolean) => + stopInstance(input.home, { + instance: input.instance, + timeoutMs: 15_000, + keepTerminals: keep, + requestShutdown: async (ready) => { + await runCliJson([ + "daemon", + "stop", + "--host", + ready.listen, + ...(keep ? ["--keep-terminals"] : []), + "--json", + ]); + }, + }); + try { + await stop(input.keepTerminals); + } catch (error) { + // The CLI refuses a keep stop before sending anything when the daemon predates the feature; + // that daemon closes terminals on every stop, as every upgrade did before. + if ( + !input.keepTerminals || + !(error instanceof ExternalCliCommandError) || + error.code !== "TERMINALS_KEEP_UNSUPPORTED" + ) { + throw error; + } + const version = typeof error.error?.version === "string" ? error.error.version : "unknown"; + logLifecycle(`daemon v${version} cannot keep terminals; stopping it and closing them`, { + reason: input.reason, + pid: input.instance.pid, + }); + await stop(false); + } +} + +/** + * Whether a desktop stop keeps the daemon's terminals running for the daemon that replaces it. + * Only when one comes straight back (an app update, a version-mismatch restart), and never on + * Windows: a surviving terminal worker runs Paseo.exe and would lock the install folder the + * update replaces. Quit, Settings, host removal and manual stops close terminals. + */ +export function keepsTerminalsOnStop( + reason: DesktopDaemonStopReason, + platform: NodeJS.Platform, +): boolean { + if (platform === "win32") return false; + return reason === "app_update" || reason === "version_mismatch"; +} + async function restartDaemon(): Promise { await runExternalCliJsonCommand(["daemon", "restart", "--home", getPaseoHome(), "--json"]); return resolveDesktopDaemonStatus(); diff --git a/packages/protocol/src/messages.ts b/packages/protocol/src/messages.ts index 92d6ec71e1..26a90016a7 100644 --- a/packages/protocol/src/messages.ts +++ b/packages/protocol/src/messages.ts @@ -1896,11 +1896,17 @@ export const RestartServerRequestMessageSchema = z.object({ type: z.literal("restart_server_request"), reason: z.string().optional(), requestId: z.string(), + // Restart keeps terminals unless this is true. Daemons that predate + // `features.terminalsSurviveRestart` ignore it and close terminals anyway. + killTerminals: z.boolean().optional(), }); export const ShutdownServerRequestMessageSchema = z.object({ type: z.literal("shutdown_server_request"), requestId: z.string(), + // Shutdown closes terminals unless this is true. Gate on + // `features.terminalsSurviveRestart`: older daemons drop the field. + keepTerminals: z.boolean().optional(), }); export const DaemonUpdateRequestMessageSchema = z.object({ @@ -3763,6 +3769,8 @@ export const ServerInfoStatusPayloadSchema = z daemonDiagnostics: z.boolean().optional(), // COMPAT(daemonSelfUpdate): added in v0.1.93, remove gate after 2026-12-13. daemonSelfUpdate: z.boolean().optional(), + // COMPAT(terminalsSurviveRestart): added in v0.12.0, remove gate after 2027-04-21. + terminalsSurviveRestart: z.boolean().optional(), // COMPAT(agentForkContext): added in v0.1.102, remove gate after 2026-12-28. agentForkContext: z.boolean().optional(), // COMPAT(agentForkContextCursor): added in v0.1.108, remove gate after 2027-01-14. @@ -5125,6 +5133,22 @@ export const DaemonGetStatusResponseSchema = z.object({ error: z.string().nullable().optional(), }), ), + // Processes a terminal cleanup on this host (a kill restart or stop) could not confirm + // gone; empty when every cleanup finished. Daemons without terminalsSurviveRestart have + // no such cleanup and omit it. + terminalCleanupUnconfirmed: z + .array( + z.object({ + pid: z.number(), + role: z.string(), + reason: z.string(), + detail: z.string(), + }), + ) + .optional(), + // Set when this host's terminals are held by a terminal worker from a newer Paseo, which + // this daemon leaves running but cannot drive: why, and what the user can do. + terminalsUnavailable: z.string().optional(), }) .passthrough(), }); diff --git a/packages/server/package.json b/packages/server/package.json index a6eb8eaa32..5572271cbb 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -73,7 +73,7 @@ "clean": "node ../../scripts/clean-package-dist.mjs", "build": "npm run build:lib && npm run build:scripts", "build:clean": "npm run clean && npm run build", - "build:lib": "tsc -p tsconfig.server.json --incremental false && node scripts/build-opencode-bridge-plugin.mjs && node -e \"const fs=require('node:fs'); fs.mkdirSync('dist/server/server/speech/providers/local/sherpa/assets',{recursive:true}); fs.copyFileSync('src/server/speech/providers/local/sherpa/assets/silero_vad.onnx','dist/server/server/speech/providers/local/sherpa/assets/silero_vad.onnx'); fs.cpSync('src/terminal/shell-integration','dist/server/terminal/shell-integration',{recursive:true}); fs.cpSync('src/terminal/shell-integration','dist/src/terminal/shell-integration',{recursive:true}); fs.copyFileSync('src/terminal/terminal-ts-loader.mjs','dist/server/terminal/terminal-ts-loader.mjs'); fs.rmSync('dist/server/skills',{recursive:true,force:true}); fs.cpSync('../../skills','dist/server/skills',{recursive:true}); fs.rmSync('dist/server/builtin-plugins',{recursive:true,force:true}); fs.cpSync('../../plugins','dist/server/builtin-plugins',{recursive:true,filter:(source)=>!source.split(require('node:path').sep).some((segment)=>segment==='node_modules'||segment==='test')&&!/\\.(test|spec)\\.[cm]?[jt]sx?$/.test(source)&&!source.endsWith('.tsbuildinfo')});\"", + "build:lib": "tsc -p tsconfig.server.json --incremental false && node scripts/build-opencode-bridge-plugin.mjs && node scripts/build-terminal-worker.mjs && node -e \"const fs=require('node:fs'); fs.mkdirSync('dist/server/server/speech/providers/local/sherpa/assets',{recursive:true}); fs.copyFileSync('src/server/speech/providers/local/sherpa/assets/silero_vad.onnx','dist/server/server/speech/providers/local/sherpa/assets/silero_vad.onnx'); fs.cpSync('src/terminal/shell-integration','dist/server/terminal/shell-integration',{recursive:true}); fs.cpSync('src/terminal/shell-integration','dist/src/terminal/shell-integration',{recursive:true}); fs.copyFileSync('src/terminal/terminal-ts-loader.mjs','dist/server/terminal/terminal-ts-loader.mjs'); fs.rmSync('dist/server/skills',{recursive:true,force:true}); fs.cpSync('../../skills','dist/server/skills',{recursive:true}); fs.rmSync('dist/server/builtin-plugins',{recursive:true,force:true}); fs.cpSync('../../plugins','dist/server/builtin-plugins',{recursive:true,filter:(source)=>!source.split(require('node:path').sep).some((segment)=>segment==='node_modules'||segment==='test')&&!/\\.(test|spec)\\.[cm]?[jt]sx?$/.test(source)&&!source.endsWith('.tsbuildinfo')});\"", "build:scripts": "tsc -p tsconfig.scripts.json --incremental false && node -e \"const fs=require('node:fs'); fs.mkdirSync('dist/scripts',{recursive:true}); fs.copyFileSync('scripts/mcp-stdio-socket-bridge-cli.mjs','dist/scripts/mcp-stdio-socket-bridge-cli.mjs');\"", "prepack": "npm run build:clean && npm --prefix ../.. run build:daemon-web-ui", "start": "node dist/scripts/supervisor-entrypoint.js", @@ -89,7 +89,7 @@ "speech:transcribe:local": "tsx scripts/transcribe-local-wav.ts", "test": "npm run test:unit && npm run test:integration", "test:unit": "vitest run --fileParallelism --exclude \"**/*.e2e.test.ts\"", - "test:integration": "vitest run --maxWorkers=1 src/server/git-filename-diffs.e2e.test.ts src/server/daemon-e2e/checkout-diff-content.e2e.test.ts src/server/daemon-e2e/filesystem.e2e.test.ts src/server/daemon-e2e/file-download.e2e.test.ts src/server/daemon-e2e/cli-logs-follow.e2e.test.ts src/server/creation/creation.e2e.test.ts src/server/daemon-e2e/terminal-workspace-sdk.e2e.test.ts src/server/workspace-create-worktree-source.e2e.test.ts src/server/daemon-e2e/models.e2e.test.ts src/server/daemon-e2e/live-preferences.e2e.test.ts src/server/daemon-e2e/oversized-message.e2e.test.ts src/server/agent/model-catalog.e2e.test.ts src/server/plugins/plugin-paseo-api.e2e.test.ts src/server/plugins/lifecycle.e2e.test.ts src/server/connected-hosts/hosts.e2e.test.ts && vitest run --maxWorkers=1 src/server/agent/providers/omp/history.e2e.test.ts src/server/daemon-e2e/git-operations.e2e.test.ts src/server/session.create-agent-worktree-autoarchive.e2e.test.ts src/server/agent/providers/opencode-bridge.local.e2e.test.ts src/server/daemon-e2e/relay-transport.e2e.test.ts -t \"merges into|creates a worktree and auto-archives both|versioned runtime|oversized attachments|connected hosts|password-protected daemon refuses|replays selected OMP history\"", + "test:integration": "vitest run --maxWorkers=1 src/server/git-filename-diffs.e2e.test.ts src/server/daemon-e2e/checkout-diff-content.e2e.test.ts src/server/daemon-e2e/filesystem.e2e.test.ts src/server/daemon-e2e/file-download.e2e.test.ts src/server/daemon-e2e/cli-logs-follow.e2e.test.ts src/server/creation/creation.e2e.test.ts src/server/daemon-e2e/terminal-workspace-sdk.e2e.test.ts src/server/daemon-e2e/terminal-worker-lifecycle.e2e.test.ts src/server/workspace-create-worktree-source.e2e.test.ts src/server/daemon-e2e/models.e2e.test.ts src/server/daemon-e2e/live-preferences.e2e.test.ts src/server/daemon-e2e/oversized-message.e2e.test.ts src/server/agent/model-catalog.e2e.test.ts src/server/plugins/plugin-paseo-api.e2e.test.ts src/server/plugins/lifecycle.e2e.test.ts src/server/connected-hosts/hosts.e2e.test.ts && vitest run --maxWorkers=1 src/server/agent/providers/omp/history.e2e.test.ts src/server/daemon-e2e/git-operations.e2e.test.ts src/server/session.create-agent-worktree-autoarchive.e2e.test.ts src/server/agent/providers/opencode-bridge.local.e2e.test.ts src/server/daemon-e2e/relay-transport.e2e.test.ts -t \"merges into|creates a worktree and auto-archives both|versioned runtime|oversized attachments|connected hosts|password-protected daemon refuses|replays selected OMP history\"", "test:hub-cli-contract": "vitest run --maxWorkers=1 src/server/hub/hub-cli-contract.test.ts", "test:integration:all": "npm run test:e2e", "test:integration:real": "vitest run real.e2e.test.ts", diff --git a/packages/server/scripts/build-terminal-worker.mjs b/packages/server/scripts/build-terminal-worker.mjs new file mode 100644 index 0000000000..3a5b290fc3 --- /dev/null +++ b/packages/server/scripts/build-terminal-worker.mjs @@ -0,0 +1,55 @@ +// Bundles the terminal worker into one file with its shell integration beside it. The daemon +// copies this directory (plus node-pty) into $PASEO_HOME/terminal-worker/ and launches the worker +// from the copy, so a worker that outlives an upgrade never loads files from the installation +// that replaced the one it came from. See docs/terminal-worker.md. +// +// Usage: node scripts/build-terminal-worker.mjs [--outdir ] +// Default output: dist/server/terminal-worker/ + +import { cp, mkdir, rm } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +import { build } from "esbuild"; + +const packageRoot = fileURLToPath(new URL("..", import.meta.url)); + +export const TERMINAL_WORKER_BUNDLE_NAME = "terminal-worker.mjs"; + +export async function buildTerminalWorkerBundle({ outDir }) { + await rm(outDir, { recursive: true, force: true }); + await mkdir(outDir, { recursive: true }); + await build({ + entryPoints: [path.join(packageRoot, "src/terminal/terminal-worker-process.ts")], + bundle: true, + format: "esm", + platform: "node", + target: "node20", + // Loaded from the copy's own node_modules/ at run time. + external: ["node-pty"], + // Bundled CommonJS dependencies call require() for Node built-ins. + banner: { + js: 'import { createRequire as __terminalWorkerCreateRequire } from "node:module"; const require = __terminalWorkerCreateRequire(import.meta.url);', + }, + outfile: path.join(outDir, TERMINAL_WORKER_BUNDLE_NAME), + logLevel: "warning", + }); + // terminal.ts resolves the zsh integration relative to its own module, which is the bundle. + await cp( + path.join(packageRoot, "src/terminal/shell-integration"), + path.join(outDir, "shell-integration"), + { recursive: true }, + ); +} + +function readOutDir(argv) { + const index = argv.indexOf("--outdir"); + if (index === -1) return path.join(packageRoot, "dist/server/terminal-worker"); + const value = argv[index + 1]; + if (!value) throw new Error("--outdir requires a directory"); + return path.resolve(value); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + await buildTerminalWorkerBundle({ outDir: readOutDir(process.argv.slice(2)) }); +} diff --git a/packages/server/scripts/supervisor-entrypoint.ts b/packages/server/scripts/supervisor-entrypoint.ts index ee3746accf..7d0e7f50aa 100644 --- a/packages/server/scripts/supervisor-entrypoint.ts +++ b/packages/server/scripts/supervisor-entrypoint.ts @@ -12,6 +12,7 @@ import { resolvePaseoHome } from "../src/server/paseo-home.js"; import { daemonLogPath } from "../src/server/daemon-instance.js"; import { PRIVATE_FILE_MODE } from "../src/server/private-files.js"; import { loadPersistedConfig } from "../src/server/persisted-config.js"; +import { reapTerminalWorkers } from "../src/terminal/terminal-worker-reaper.js"; import { runSupervisor } from "./supervisor.js"; import { resolveSupervisorLogFile } from "./supervisor-log-config.js"; @@ -175,6 +176,10 @@ async function main(): Promise { onWorkerExit: () => updatePidLock(paseoHome, { listen: null, serverId: null }, { ownerPid: process.pid }), onSupervisorExit: releaseLock, + cleanupTerminals: async (log) => { + const result = await reapTerminalWorkers({ paseoHome, log }); + log("Terminal worker cleanup result", { ...result }); + }, }); requestSupervisorShutdown = supervisor.requestShutdown; } diff --git a/packages/server/scripts/supervisor.terminal-cleanup.test.ts b/packages/server/scripts/supervisor.terminal-cleanup.test.ts new file mode 100644 index 0000000000..224a0788d3 --- /dev/null +++ b/packages/server/scripts/supervisor.terminal-cleanup.test.ts @@ -0,0 +1,180 @@ +import { spawn } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, test } from "vitest"; + +// A real supervisor with scripted workers. Each worker generation logs what it sees to an +// events file; `cleanupTerminals` logs "cleanup". The order of events is the contract: a kill +// intent cleans up before the next worker starts, a keep intent or a crash never cleans up. + +async function runSupervisorWith(workerScript: string, timeoutMs = 8_000) { + const home = await mkdtemp(path.join(tmpdir(), "paseo-terminal-cleanup-")); + const workerPath = path.join(home, "worker.mjs"); + const runnerPath = path.join(home, "runner.mjs"); + const eventsPath = path.join(home, "events.jsonl"); + const supervisor = new URL("./supervisor.ts", import.meta.url).href; + await writeFile( + workerPath, + ` + import { appendFileSync, existsSync, readFileSync, writeFileSync } from "node:fs"; + const counter = ${JSON.stringify(path.join(home, "generation"))}; + const generation = existsSync(counter) ? Number(readFileSync(counter, "utf8")) + 1 : 1; + writeFileSync(counter, String(generation)); + const record = (event) => + appendFileSync(${JSON.stringify(eventsPath)}, JSON.stringify(event) + "\\n"); + record({ started: generation }); + ${workerScript} + `, + ); + await writeFile( + runnerPath, + ` + import { appendFileSync } from "node:fs"; + import { runSupervisor } from ${JSON.stringify(supervisor)}; + runSupervisor({ + name: "TerminalCleanupTest", startupMessage: "starting", restartOnCrash: true, + resolveWorkerEntry: () => ${JSON.stringify(workerPath)}, workerArgs: [], workerExecArgv: [], + cleanupTerminals: async () => { + appendFileSync(${JSON.stringify(eventsPath)}, JSON.stringify("cleanup") + "\\n"); + }, + }); + `, + ); + const env = Object.fromEntries( + Object.entries(process.env).filter(([key]) => !key.startsWith("PASEO_")), + ); + const child = spawn(process.execPath, ["--import", "tsx", runnerPath], { + cwd: fileURLToPath(new URL("../../../", import.meta.url)), + env: { ...env, HOME: home, PASEO_HOME: home }, + stdio: ["ignore", "pipe", "pipe"], + }); + let output = ""; + child.stdout.on("data", (chunk) => (output += chunk)); + child.stderr.on("data", (chunk) => (output += chunk)); + try { + const code = await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + child.kill("SIGKILL"); + reject(new Error(output)); + }, timeoutMs); + child.once("error", reject); + child.once("close", (exitCode) => { + clearTimeout(timer); + resolve(exitCode); + }); + }); + const events = (await readFile(eventsPath, "utf8").catch(() => "")) + .trim() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as unknown); + return { code, output, events }; + } finally { + await rm(home, { recursive: true, force: true }); + } +} + +// Generation 1 asks for `firstIntent`; generation 2 (if any) stops with `secondIntent`. +function scriptedWorker(firstIntent: string, secondIntent = "") { + return ` + process.on("message", (message) => { + if (message.type !== "paseo:graceful-shutdown") return; + record({ graceful: generation, keepTerminals: message.keepTerminals }); + process.exit(0); + }); + process.send({ type: "paseo:ready", listen: "endpoint-" + generation, serverId: "srv" }); + if (generation === 1) { ${firstIntent} } + else { ${secondIntent} } + `; +} + +const stopKeeping = `process.send({ type: "paseo:shutdown", reason: "test", keepTerminals: true });`; + +describe("supervisor terminal cleanup", () => { + test("a keep restart passes keepTerminals and never cleans up", async () => { + const result = await runSupervisorWith( + scriptedWorker( + `process.send({ type: "paseo:restart", reason: "test", keepTerminals: true });`, + stopKeeping, + ), + ); + expect(result.code, result.output).toBe(0); + expect(result.events).toEqual([ + { started: 1 }, + { graceful: 1, keepTerminals: true }, + { started: 2 }, + { graceful: 2, keepTerminals: true }, + ]); + }); + + test("a kill restart cleans up after the worker exits and before the respawn", async () => { + const result = await runSupervisorWith( + scriptedWorker( + `process.send({ type: "paseo:restart", reason: "test", keepTerminals: false });`, + stopKeeping, + ), + ); + expect(result.code, result.output).toBe(0); + expect(result.events).toEqual([ + { started: 1 }, + { graceful: 1, keepTerminals: false }, + "cleanup", + { started: 2 }, + { graceful: 2, keepTerminals: true }, + ]); + }); + + test("a kill restart whose worker hangs cleans up after the forced kill, before the respawn", async () => { + const result = await runSupervisorWith( + ` + process.on("message", (message) => { + if (message.type !== "paseo:graceful-shutdown") return; + record({ graceful: generation, keepTerminals: message.keepTerminals }); + if (generation === 2) process.exit(0); + }); + process.send({ type: "paseo:ready", listen: "endpoint-" + generation, serverId: "srv" }); + if (generation === 1) { + process.send({ type: "paseo:restart", reason: "test", keepTerminals: false }); + setInterval(() => {}, 1000); + } else { + ${stopKeeping} + } + `, + 20_000, + ); + expect(result.code, result.output).toBe(0); + expect(result.events).toEqual([ + { started: 1 }, + { graceful: 1, keepTerminals: false }, + "cleanup", + { started: 2 }, + { graceful: 2, keepTerminals: true }, + ]); + }, 30_000); + + test("a crash restart keeps terminals", async () => { + const result = await runSupervisorWith(scriptedWorker(`process.exit(1);`, stopKeeping)); + expect(result.code, result.output).toBe(0); + expect(result.events).toEqual([ + { started: 1 }, + { started: 2 }, + { graceful: 2, keepTerminals: true }, + ]); + }); + + test("a kill stop cleans up when the supervisor exits", async () => { + const result = await runSupervisorWith( + scriptedWorker( + `process.send({ type: "paseo:shutdown", reason: "test", keepTerminals: false });`, + ), + ); + expect(result.code, result.output).toBe(0); + expect(result.events).toEqual([ + { started: 1 }, + { graceful: 1, keepTerminals: false }, + "cleanup", + ]); + }); +}); diff --git a/packages/server/scripts/supervisor.ts b/packages/server/scripts/supervisor.ts index ec56822619..fac28946d5 100644 --- a/packages/server/scripts/supervisor.ts +++ b/packages/server/scripts/supervisor.ts @@ -21,6 +21,7 @@ type WorkerLifecycleMessage = | { type: "paseo:shutdown"; reason?: string; + keepTerminals: boolean; } | { type: "paseo:ready"; @@ -30,6 +31,7 @@ type WorkerLifecycleMessage = | { type: "paseo:restart"; reason?: string; + keepTerminals: boolean; }; interface SupervisorHeartbeatMessage { @@ -39,8 +41,11 @@ interface SupervisorHeartbeatMessage { interface SupervisorGracefulShutdownMessage { type: "paseo:graceful-shutdown"; reason: string; + keepTerminals: boolean; } +export type SupervisorLifecycleLog = (message: string, fields?: Record) => void; + interface SupervisorOptions { name: string; startupMessage: string; @@ -57,11 +62,15 @@ interface SupervisorOptions { onWorkerExit?: () => Promise | void; restartOnCrash?: boolean; onSupervisorExit?: () => Promise | void; + // Closes every terminal the worker left. Runs after a kill-intent restart or stop once the + // worker has exited (also after a forced kill), before a respawn, and on any supervisor + // exit except a keep-terminals stop. A crash restart keeps terminals and skips it. + cleanupTerminals?: (log: SupervisorLifecycleLog) => Promise; logFile?: SupervisorLogFileOptions; } export interface SupervisorController { - requestShutdown(reason: string): void; + requestShutdown(reason: string, options?: { keepTerminals?: boolean }): void; } function describeExit(code: number | null, signal: NodeJS.Signals | null): string { @@ -74,10 +83,11 @@ function parseLifecycleMessage(msg: unknown): WorkerLifecycleMessage | null { } const type = (msg as { type?: unknown }).type; if (type === "paseo:shutdown") { - const reason = (msg as { reason?: unknown }).reason; + const { reason, keepTerminals } = msg as { reason?: unknown; keepTerminals?: unknown }; return { type: "paseo:shutdown", ...(typeof reason === "string" && reason.trim().length > 0 ? { reason } : {}), + keepTerminals: keepTerminals === true, }; } if (type === "paseo:ready") { @@ -91,10 +101,11 @@ function parseLifecycleMessage(msg: unknown): WorkerLifecycleMessage | null { return { type: "paseo:ready", listen, serverId }; } if (type === "paseo:restart") { - const reason = (msg as { reason?: unknown }).reason; + const { reason, keepTerminals } = msg as { reason?: unknown; keepTerminals?: unknown }; return { type: "paseo:restart", ...(typeof reason === "string" && reason.trim().length > 0 ? { reason } : {}), + keepTerminals: keepTerminals === true, }; } return null; @@ -185,6 +196,9 @@ export function runSupervisor(options: SupervisorOptions): SupervisorController let child: ChildProcess | null = null; let restarting = false; let shuttingDown = false; + // The resolved terminal decision of the pending restart or stop. It survives the force-kill + // path, so a hung kill restart still closes terminals before the respawn. + let pendingKeepTerminals = false; let exiting = false; let lifecycleFailed = false; let publication = Promise.resolve(); @@ -217,12 +231,31 @@ export function runSupervisor(options: SupervisorOptions): SupervisorController const closeLogStream = (): Promise => durableLog.close(); - const exitSupervisor = (code: number): void => { + const runTerminalCleanup = async (when: string): Promise => { + if (!options.cleanupTerminals) { + return; + } + writeLifecycleLog("Closing terminals", { when }); + try { + await options.cleanupTerminals(writeLifecycleLog); + writeLifecycleLog("Terminal cleanup finished", { when }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + log(`Terminal cleanup not confirmed (${when}): ${message}`); + } + }; + + const exitSupervisor = (code: number, exitOptions: { keepTerminals?: boolean } = {}): void => { if (exiting) { return; } exiting = true; - Promise.resolve(options.onSupervisorExit?.()) + const cleanup = + exitOptions.keepTerminals === true + ? Promise.resolve() + : runTerminalCleanup("supervisor_exit"); + cleanup + .then(() => options.onSupervisorExit?.()) .catch((error) => { const message = error instanceof Error ? error.message : String(error); log(`Supervisor exit cleanup failed: ${message}`); @@ -357,14 +390,20 @@ export function runSupervisor(options: SupervisorOptions): SupervisorController if (lifecycleMessage.type === "paseo:shutdown") { const reason = lifecycleMessage.reason ?? "worker_requested_shutdown"; - writeLifecycleLog("Worker requested shutdown", { reason }); - requestShutdown(reason); + writeLifecycleLog("Worker requested shutdown", { + reason, + keepTerminals: lifecycleMessage.keepTerminals, + }); + requestShutdown(reason, { keepTerminals: lifecycleMessage.keepTerminals }); return; } const reason = lifecycleMessage.reason ?? "worker_requested_restart"; - writeLifecycleLog("Worker requested restart", { reason }); - requestRestart(reason); + writeLifecycleLog("Worker requested restart", { + reason, + keepTerminals: lifecycleMessage.keepTerminals, + }); + requestRestart(reason, lifecycleMessage.keepTerminals); }); child.on("error", (error) => { @@ -395,7 +434,7 @@ export function runSupervisor(options: SupervisorOptions): SupervisorController if (shuttingDown) { log(`Worker exited (${exitDescriptor}). Supervisor shutting down.`); - exitSupervisor(0); + exitSupervisor(0, { keepTerminals: pendingKeepTerminals }); return; } @@ -404,13 +443,25 @@ export function runSupervisor(options: SupervisorOptions): SupervisorController ((code !== 0 && code !== null) || (signal !== null && signal !== "SIGTERM")); if (restarting || crashed) { + const closeTerminals = restarting && !pendingKeepTerminals; restarting = false; + pendingKeepTerminals = false; log( - crashed + crashed && !closeTerminals ? `Worker crashed (${exitDescriptor}). Restarting worker...` : `Worker exited (${exitDescriptor}). Restarting worker...`, ); - spawnWorker(); + if (!closeTerminals) { + spawnWorker(); + return; + } + // The replacement daemon must never adopt terminals the user asked to close. + void runTerminalCleanup("before_respawn").then(() => { + if (!shuttingDown) { + spawnWorker(); + } + return; + }); return; } @@ -429,9 +480,11 @@ export function runSupervisor(options: SupervisorOptions): SupervisorController const message: SupervisorGracefulShutdownMessage = { type: "paseo:graceful-shutdown", reason, + keepTerminals: pendingKeepTerminals, }; writeLifecycleLog("Supervisor requesting graceful worker shutdown", { reason, + keepTerminals: pendingKeepTerminals, supervisorPid: process.pid, workerPid: currentChild.pid ?? null, }); @@ -455,28 +508,33 @@ export function runSupervisor(options: SupervisorOptions): SupervisorController }); }; - const requestRestart = (reason: string) => { + const requestRestart = (reason: string, keepTerminals: boolean) => { if (!child || restarting || shuttingDown) { return; } restarting = true; - writeLifecycleLog("Restart requested", { reason }); + pendingKeepTerminals = keepTerminals; + writeLifecycleLog("Restart requested", { reason, keepTerminals }); log(`${reason}. Stopping worker for restart...`); requestWorkerShutdown(reason); scheduleForceKill(reason); }; - const requestShutdown = (reason: string) => { + const requestShutdown = (reason: string, shutdownOptions: { keepTerminals?: boolean } = {}) => { if (shuttingDown) { return; } shuttingDown = true; restarting = false; - writeLifecycleLog("Supervisor shutdown requested", { reason }); + pendingKeepTerminals = shutdownOptions.keepTerminals === true; + writeLifecycleLog("Supervisor shutdown requested", { + reason, + keepTerminals: pendingKeepTerminals, + }); log(`${reason}. Stopping worker...`); if (!child) { void publication.then(() => { - exitSupervisor(lifecycleFailed ? 1 : 0); + exitSupervisor(lifecycleFailed ? 1 : 0, { keepTerminals: pendingKeepTerminals }); return; }); return; diff --git a/packages/server/src/executable-resolution/executable-resolution.ts b/packages/server/src/executable-resolution/executable-resolution.ts index 97bb8af3a3..064dcec64e 100644 --- a/packages/server/src/executable-resolution/executable-resolution.ts +++ b/packages/server/src/executable-resolution/executable-resolution.ts @@ -1,17 +1,10 @@ -import { createRequire } from "node:module"; import { existsSync } from "node:fs"; import { delimiter, resolve } from "node:path"; import type { ProcessEnvRecord } from "../server/paseo-env.js"; import { execCommand } from "../utils/spawn.js"; +import which from "which"; import { windowsExecutableResolution } from "./windows.js"; -type Which = ( - command: string, - options: { all: true; path?: string; pathExt?: string }, -) => Promise; - -const require = createRequire(import.meta.url); -const which = require("which") as Which; const PROBE_TIMEOUT_MS = 2000; function hasPathSeparator(value: string): boolean { diff --git a/packages/server/src/executable-resolution/which.d.ts b/packages/server/src/executable-resolution/which.d.ts new file mode 100644 index 0000000000..e7fd869f25 --- /dev/null +++ b/packages/server/src/executable-resolution/which.d.ts @@ -0,0 +1,10 @@ +// `which` ships no types. Imported statically (not through createRequire) so the terminal +// worker bundle includes it: the worker runs from a copy that has only node-pty beside it. +// Listed in the tsconfig includes, since nothing imports a declaration file. +declare module "which" { + function which( + command: string, + options: { all: true; path?: string; pathExt?: string }, + ): Promise; + export default which; +} diff --git a/packages/server/src/server/agent/mcp-server.test.ts b/packages/server/src/server/agent/mcp-server.test.ts index f2c7da8574..3b72c8805a 100644 --- a/packages/server/src/server/agent/mcp-server.test.ts +++ b/packages/server/src/server/agent/mcp-server.test.ts @@ -76,7 +76,7 @@ import type { GeneratedWorkspaceName } from "../worktree-branch-name-generator.j import type { ForgeService } from "../../services/forge-service.js"; import { areEquivalentPaths } from "../../utils/path.js"; import { createTerminalManager, type TerminalManager } from "../../terminal/terminal-manager.js"; -import { createWorkerTerminalManager } from "../../terminal/worker-terminal-manager.js"; +import { startWorkerTerminalManager } from "../../terminal/worker-terminal-manager.js"; import { PARENT_AGENT_ID_LABEL } from "@getpaseo/protocol/agent-labels"; import { PASEO_TOOL_NAMES } from "@getpaseo/protocol/paseo-tool-presentation/index"; import { MutableDaemonConfigSchema, type AgentProfile } from "@getpaseo/protocol/messages"; @@ -1384,13 +1384,24 @@ describe("terminal MCP tools", () => { ); it.each([ - ["in-process", createTerminalManager], - ["worker", createWorkerTerminalManager], + ["in-process", async () => ({ manager: createTerminalManager(), home: null })], + [ + "worker", + async () => { + const home = await mkdtemp(join(tmpdir(), "mcp-terminal-keys-home-")); + const manager = await startWorkerTerminalManager({ + paseoHome: home, + paseoVersion: "0.0.0-test", + logger, + }); + return { manager, home }; + }, + ], ] as const)( "delivers MCP key tokens through %s after terminal input modes change", async (_name, createManager) => { const { agentManager, agentStorage } = createTestDeps(); - const terminalManager = createManager(); + const { manager: terminalManager, home } = await createManager(); const terminal = await terminalManager.createTerminal({ cwd: process.cwd(), workspaceId: "ws-terminal-keys", @@ -1420,15 +1431,15 @@ describe("terminal MCP tools", () => { logger, }); + // Every step is a round trip through a real PTY. A loaded Windows runner (ConPTY) has + // taken over 3 s to reach READY, so each step gets the same window rather than 1 s. + const roundTrip = { timeout: 5000 }; let output = ""; const unsubscribe = terminal.subscribe((message) => { if (message.type === "output") { output += message.data; } }); - // Every key travels through the PTY and back; on loaded Windows runners one round - // trip can take seconds, so each wait gets the same budget as the first. - const roundTrip = { timeout: 5000 }; try { await vi.waitFor(async () => { const capture = await terminalManager.captureTerminal(terminal.id); @@ -1465,6 +1476,7 @@ describe("terminal MCP tools", () => { await server.close(); await terminalManager.killTerminalAndWait(terminal.id); terminalManager.killAll(); + if (home) await rm(home, { recursive: true, force: true }); } }, ); diff --git a/packages/server/src/server/agent/providers/opencode-bridge.local.e2e.test.ts b/packages/server/src/server/agent/providers/opencode-bridge.local.e2e.test.ts index 843ef21ebb..aeb87388fd 100644 --- a/packages/server/src/server/agent/providers/opencode-bridge.local.e2e.test.ts +++ b/packages/server/src/server/agent/providers/opencode-bridge.local.e2e.test.ts @@ -455,9 +455,15 @@ test.each([ let original: Awaited> | undefined; let resumed: Awaited> | undefined; try { + // Windows invokes the platform binary directly. Installing its wrapper also runs an + // unused postinstall binary probe, which has no timeout and can hold the fixture open. + const installPackage = + process.platform === "win32" + ? `${windowsPackage}-${process.arch}@${cliPackage.slice(cliPackage.lastIndexOf("@") + 1)}` + : cliPackage; await execCommand( "npm", - ["install", "--prefix", root, "--no-audit", "--no-fund", cliPackage], + ["install", "--prefix", root, "--no-audit", "--no-fund", installPackage], { cwd: root, timeout: 180_000, diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index b3eb6f0f66..36ba9e2b06 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -180,7 +180,7 @@ import { import { setupAutoArchiveOnMerge } from "./auto-archive-on-merge/index.js"; import { wrapSessionMessage, type SessionOutboundMessage } from "./messages.js"; import type { TerminalManager } from "../terminal/terminal-manager.js"; -import { createConfiguredTerminalManager } from "../terminal/terminal-manager-factory.js"; +import { startWorkerTerminalManager } from "../terminal/worker-terminal-manager.js"; import { applyTerminalAgentHookSetting } from "../terminal/agent-hooks/terminal-agent-hook-setting.js"; import { loadOrCreateDaemonKeyPair } from "./daemon-keypair.js"; import { createRelayRuntime, type RelayRuntime } from "./relay-runtime.js"; @@ -396,12 +396,14 @@ export type DaemonLifecycleIntent = clientId: string; requestId: string; reason: string; + keepTerminals: boolean; } | { type: "restart"; clientId: string; requestId: string; reason: string; + keepTerminals: boolean; }; export interface PaseoDaemonConfig { @@ -493,7 +495,8 @@ export interface PaseoDaemon { scriptRuntimeStore: WorkspaceScriptRuntimeStore; browserToolsBroker: BrowserToolsBroker; start(): Promise; - stop(): Promise; + /** Stops the daemon. Terminals keep running for the next daemon only with keepTerminals. */ + stop(options?: { keepTerminals?: boolean }): Promise; getListenTarget(): ListenTarget | null; getServerId(): string; } @@ -709,7 +712,10 @@ export async function createPaseoDaemon( }); let boundListenTarget: ListenTarget | null = null; let workspaceRegistry: FileBackedWorkspaceRegistry | null = null; - const terminalManager = createConfiguredTerminalManager({ + const terminalManager = await startWorkerTerminalManager({ + paseoHome: config.paseoHome, + paseoVersion: daemonVersion, + logger, getTerminalActivityUrl: () => createTerminalActivityUrl(boundListenTarget), }); applyTerminalAgentHookSetting({ store: daemonConfigStore, logger }); @@ -1936,7 +1942,9 @@ export async function createPaseoDaemon( } }; - const stop = async () => { + const stop = async (options?: { keepTerminals?: boolean }) => { + const keepTerminals = options?.keepTerminals === true; + let terminalShutdownError: unknown = null; localCredential = null; await deleteLocalCredential(config.paseoHome); // Stop tracking plugin provider registrations before anything tears plugins @@ -1966,7 +1974,19 @@ export async function createPaseoDaemon( await agentStorage.flush().catch(() => undefined); await agentProviderRuntime.shutdown(); await pluginRuntime.stopAllPlugins(); - terminalManager.killAll(); + try { + if (keepTerminals) { + await terminalManager.detach(); + logger.info("Terminals left running for the next daemon"); + } else { + await terminalManager.closeTerminals(); + } + } catch (error) { + // Finish the rest of the shutdown, then report it: the supervisor and `daemon stop` + // retry the cleanup from the record the reaper kept. + terminalShutdownError = error; + logger.error({ err: error }, "Terminal shutdown failed"); + } await speechService.stop(); await scheduleService.stop().catch(() => undefined); await relayRuntime?.stop().catch(() => undefined); @@ -1989,6 +2009,9 @@ export async function createPaseoDaemon( if (listenTarget.type === "socket" && existsSync(listenTarget.path)) { unlinkSync(listenTarget.path); } + if (terminalShutdownError) { + throw terminalShutdownError; + } }; return { diff --git a/packages/server/src/server/daemon-e2e/terminal-worker-lifecycle.e2e.test.ts b/packages/server/src/server/daemon-e2e/terminal-worker-lifecycle.e2e.test.ts new file mode 100644 index 0000000000..d606b4d4d9 --- /dev/null +++ b/packages/server/src/server/daemon-e2e/terminal-worker-lifecycle.e2e.test.ts @@ -0,0 +1,688 @@ +import { afterEach, describe, expect, test } from "vitest"; +import { spawn, type ChildProcess } from "node:child_process"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { createServer } from "node:net"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { DaemonClient } from "../test-utils/daemon-client.js"; +import { resolveDaemonVersion } from "../daemon-version.js"; +import { readDaemonInstance, stopDaemonInstance } from "../daemon-instance.js"; +import { currentBootId } from "../pid-lock.js"; +import { connectTerminalWorker } from "../../terminal/terminal-worker-client.js"; +import { createTerminalWorkerProcessInspector } from "../../terminal/terminal-worker-processes.js"; +import { reapTerminalWorkers } from "../../terminal/terminal-worker-reaper.js"; +import { launchTerminalWorker } from "../../terminal/terminal-worker-launch.js"; +import { prepareTerminalWorkerRuntime } from "../../terminal/terminal-worker-runtime.js"; +import { createWorkerTerminalManager } from "../../terminal/worker-terminal-manager.js"; +import { + listTerminalWorkerRecords, + writeTerminalWorkerRecord, + type TerminalWorkerRecord, +} from "../../terminal/terminal-worker-record.js"; + +// Real supervisors started through scripts/supervisor-entrypoint.ts, driven by DaemonClient the +// way the app drives them. Every daemon, worker and program here lives in a throwaway home and +// is killed in afterEach; nothing touches another daemon. + +const SERVER_ROOT = path.resolve(import.meta.dirname, "../../.."); +const SUPERVISOR_ENTRY = path.join(SERVER_ROOT, "scripts/supervisor-entrypoint.ts"); +const READY_TIMEOUT_MS = 90_000; +const POSIX = process.platform !== "win32"; +const inspector = createTerminalWorkerProcessInspector(); + +interface Harness { + home: string; + workDir: string; + port: number; + supervisor: ChildProcess; + output: () => string; +} + +const harnesses: Harness[] = []; +const extraPids = new Set(); +const extraDirs: string[] = []; + +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +async function waitFor( + probe: () => Promise | T | null | undefined | false, + message: string, + timeoutMs = READY_TIMEOUT_MS, +): Promise { + const deadline = Date.now() + timeoutMs; + let lastError: unknown = null; + while (Date.now() < deadline) { + try { + const value = await probe(); + if (value) return value; + } catch (error) { + lastError = error; + } + await delay(100); + } + throw new Error(`${message}${lastError ? ` (last error: ${String(lastError)})` : ""}`); +} + +async function getAvailablePort(): Promise { + const server = createServer(); + await new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve())); + const address = server.address(); + await new Promise((resolve) => server.close(() => resolve())); + if (!address || typeof address === "string") throw new Error("no port"); + return address.port; +} + +async function startSupervisor( + options: { home?: string; port?: number; pathPrefix?: string } = {}, +): Promise { + const home = options.home ?? mkdtempSync(path.join(tmpdir(), "tw-life-")); + const workDir = path.join(home, "work"); + mkdirSync(workDir, { recursive: true }); + const port = options.port ?? (await getAvailablePort()); + const env: NodeJS.ProcessEnv = { + ...Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith("PASEO_"))), + HOME: home, + USERPROFILE: home, + PASEO_HOME: home, + PASEO_LISTEN: `127.0.0.1:${port}`, + PASEO_RELAY_ENABLED: "false", + PASEO_NODE_INSPECT: "0", + PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD: "0", + PASEO_DICTATION_ENABLED: "0", + PASEO_VOICE_MODE_ENABLED: "0", + CI: "true", + }; + if (options.pathPrefix) { + env.PATH = `${options.pathPrefix}${path.delimiter}${env.PATH ?? ""}`; + } + let output = ""; + const supervisor = spawn( + process.execPath, + ["--import", "tsx", SUPERVISOR_ENTRY, "--dev", "--no-relay"], + { cwd: SERVER_ROOT, env, stdio: ["ignore", "pipe", "pipe"] }, + ); + supervisor.stdout?.on("data", (chunk) => (output = (output + String(chunk)).slice(-20_000))); + supervisor.stderr?.on("data", (chunk) => (output = (output + String(chunk)).slice(-20_000))); + const harness: Harness = { home, workDir, port, supervisor, output: () => output }; + harnesses.push(harness); + await waitFor(async () => { + const instance = await readDaemonInstance(home); + return instance?.pid === supervisor.pid && instance.listen ? instance : null; + }, `supervisor did not become ready:\n${output}`); + return harness; +} + +async function connect(harness: Harness): Promise { + return waitFor(async () => { + const client = new DaemonClient({ + url: `ws://127.0.0.1:${harness.port}/ws`, + connectTimeoutMs: 5_000, + }); + try { + await client.connect(); + return client; + } catch (error) { + await client.close().catch(() => undefined); + throw error; + } + }, "cannot connect to the daemon"); +} + +// A plain `paseo daemon stop`: SIGTERM on POSIX, the shutdown RPC where there is no signal +// for it (Windows refuses a graceful stop without one). +async function stopDaemon(harness: Harness): Promise { + await stopDaemonInstance(harness.home, { + requestShutdown: async () => { + const client = await connect(harness); + try { + await client.shutdownServer(); + } finally { + await client.close(); + } + }, + }); +} + +async function daemonWorkerPid(client: DaemonClient): Promise { + return (await client.getDaemonStatus()).pid; +} + +async function waitForReplacement(harness: Harness, previousPid: number): Promise { + return waitFor(async () => { + const client = await connect(harness); + const pid = await daemonWorkerPid(client).catch(() => previousPid); + if (pid !== previousPid) return client; + await client.close(); + return null; + }, "the daemon worker was not replaced"); +} + +async function createWorkspace(client: DaemonClient, harness: Harness): Promise { + const result = await client.createWorkspace({ + source: { kind: "directory", path: harness.workDir }, + title: "lifecycle", + }); + if (!result.workspace) throw new Error(result.error ?? "workspace creation failed"); + return result.workspace.id; +} + +interface MarkedTerminal { + id: string; + programPid: number; +} + +// A terminal whose program writes its pid, prints a marker and echoes each input line. +async function createMarkedTerminal( + client: DaemonClient, + harness: Harness, + workspaceId: string, + marker: string, +): Promise { + const pidFile = path.join(harness.workDir, `${marker}.pid`); + const created = await client.createTerminal(harness.workDir, marker, undefined, { + workspaceId, + command: process.execPath, + args: [ + "-e", + `require("node:fs").writeFileSync(${JSON.stringify(pidFile)}, String(process.pid)); + console.log(${JSON.stringify(marker)}); + process.stdin.on("data", (data) => console.log("ECHO:" + data.toString().trim()));`, + ], + }); + if (!created.terminal) throw new Error(created.error ?? "terminal creation failed"); + const programPid = await waitFor( + () => (existsSync(pidFile) ? Number(readFileSync(pidFile, "utf8")) : null), + "terminal program did not start", + 10_000, + ); + await waitForScreen(client, created.terminal.id, marker); + return { id: created.terminal.id, programPid }; +} + +// A shell program that ignores SIGHUP. With `exec`, the shell becomes `sleep`: same pid and +// birth time, another command line. Without it, `sleep` runs as the shell's child. +async function createHupResistantTerminal( + client: DaemonClient, + harness: Harness, + workspaceId: string, + name: string, + mode: "exec" | "child", +): Promise<{ id: string; shellPid: number; sleepPid: number }> { + const shellPidFile = path.join(harness.workDir, `${name}.shell.pid`); + const sleepPidFile = path.join(harness.workDir, `${name}.sleep.pid`); + const script = + mode === "exec" + ? `trap '' HUP; echo $$ > '${shellPidFile}'; echo $$ > '${sleepPidFile}'; exec sleep 100000` + : `trap '' HUP; echo $$ > '${shellPidFile}'; sleep 100000 & echo $! > '${sleepPidFile}'; wait`; + const created = await client.createTerminal(harness.workDir, name, undefined, { + workspaceId, + command: "/bin/sh", + args: ["-c", script], + }); + if (!created.terminal) throw new Error(created.error ?? "terminal creation failed"); + const read = (file: string) => + existsSync(file) && readFileSync(file, "utf8").trim() + ? Number(readFileSync(file, "utf8").trim()) + : null; + const shellPid = await waitFor(() => read(shellPidFile), "shell did not start", 10_000); + const sleepPid = await waitFor(() => read(sleepPidFile), "sleep did not start", 10_000); + extraPids.add(shellPid); + extraPids.add(sleepPid); + // `exec` replaces the shell; wait until the process really is `sleep`. + await waitFor( + async () => { + const inspection = await inspector.inspect(sleepPid); + return inspection.status === "alive" && inspection.snapshot.commandLine?.startsWith("sleep"); + }, + "sleep did not start", + 10_000, + ); + return { id: created.terminal.id, shellPid, sleepPid }; +} + +async function screen(client: DaemonClient, terminalId: string): Promise { + return (await client.captureTerminal(terminalId, { stripAnsi: true })).lines.join("\n"); +} + +async function waitForScreen(client: DaemonClient, terminalId: string, text: string) { + await waitFor( + async () => (await screen(client, terminalId)).includes(text), + `terminal ${terminalId} never showed ${text}`, + 15_000, + ); +} + +async function expectKept( + client: DaemonClient, + terminal: MarkedTerminal, + marker: string, + trigger: string, +) { + const listed = await client.listTerminals(undefined, undefined, {}); + expect(listed.terminals.map((entry) => entry.id)).toContain(terminal.id); + expect(isAlive(terminal.programPid)).toBe(true); + expect(await screen(client, terminal.id)).toContain(marker); + client.sendTerminalInput(terminal.id, { type: "input", data: `after-${trigger}\r` }); + await waitForScreen(client, terminal.id, `ECHO:after-${trigger}`); +} + +async function waitUntilGone(pid: number, what: string) { + await waitFor(() => !isAlive(pid), `${what} (pid ${pid}) is still alive`, 15_000); +} + +async function liveRecords(home: string): Promise { + return (await listTerminalWorkerRecords(home)).flatMap((file) => + "record" in file ? [file.record] : [], + ); +} + +async function currentWorkerRecord(home: string): Promise { + const records = await liveRecords(home); + if (records.length !== 1) throw new Error(`expected one worker record, found ${records.length}`); + return records[0]!; +} + +async function expectAllClosed(home: string, pids: number[]) { + for (const pid of pids) { + await waitUntilGone(pid, "a process that should be closed"); + } + expect(await liveRecords(home)).toEqual([]); +} + +function spawnUnrelated(): ChildProcess { + const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1e9)"], { + detached: true, + stdio: "ignore", + }); + child.unref(); + extraPids.add(child.pid!); + return child; +} + +async function birthOf(pid: number): Promise { + const inspection = await inspector.inspect(pid); + if (inspection.status !== "alive" || !inspection.snapshot.startedAt) { + throw new Error(`no birth time for ${pid}`); + } + return inspection.snapshot.startedAt; +} + +afterEach(async () => { + for (const harness of harnesses.splice(0)) { + const pid = harness.supervisor.pid; + if (pid && isAlive(pid)) { + try { + process.kill(pid, "SIGCONT"); + } catch { + // gone + } + } + await stopDaemonInstance(harness.home, { force: true, timeoutMs: 15_000 }).catch( + () => undefined, + ); + if (pid && isAlive(pid)) process.kill(pid, "SIGKILL"); + for (const record of await liveRecords(harness.home)) { + if (isAlive(record.pid)) { + try { + process.kill(record.pid, "SIGCONT"); + } catch { + // gone + } + } + } + await reapTerminalWorkers({ paseoHome: harness.home, gracefulMs: 0 }).catch(() => undefined); + rmSync(harness.home, { recursive: true, force: true }); + } + for (const pid of extraPids) { + if (isAlive(pid)) { + try { + process.kill(pid, "SIGKILL"); + } catch { + // gone + } + } + } + extraPids.clear(); + for (const dir of extraDirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +describe("terminals across daemon restarts", { timeout: 240_000 }, () => { + test("Settings restart keeps the terminal; Settings restart and close terminals kills it", async () => { + const harness = await startSupervisor(); + let client = await connect(harness); + const workspaceId = await createWorkspace(client, harness); + const terminal = await createMarkedTerminal(client, harness, workspaceId, "MARKER-SETTINGS"); + const workerPid = (await currentWorkerRecord(harness.home)).pid; + + // Exactly what the app sends from Settings: no killTerminals. + let daemonPid = await daemonWorkerPid(client); + await client.restartServer("settings_update"); + await client.close(); + client = await waitForReplacement(harness, daemonPid); + await expectKept(client, terminal, "MARKER-SETTINGS", "settings-restart"); + expect((await currentWorkerRecord(harness.home)).pid).toBe(workerPid); + + daemonPid = await daemonWorkerPid(client); + await client.restartServer("settings_update", undefined, { killTerminals: true }); + await client.close(); + client = await waitForReplacement(harness, daemonPid); + await waitUntilGone(terminal.programPid, "terminal program"); + await waitUntilGone(workerPid, "old terminal worker"); + expect((await client.listTerminals(undefined, undefined, {})).terminals).toEqual([]); + expect((await liveRecords(harness.home)).every((record) => record.pid !== workerPid)).toBe( + true, + ); + await client.close(); + }); + + test("a daemon crash keeps the terminal; the restarted daemon lists it", async () => { + const harness = await startSupervisor(); + let client = await connect(harness); + const workspaceId = await createWorkspace(client, harness); + const terminal = await createMarkedTerminal(client, harness, workspaceId, "MARKER-CRASH"); + const daemonPid = await daemonWorkerPid(client); + process.kill(daemonPid, "SIGKILL"); + await client.close(); + client = await waitForReplacement(harness, daemonPid); + await expectKept(client, terminal, "MARKER-CRASH", "crash"); + await client.close(); + }); + + test("daemon.update.request restarts the daemon and keeps the terminal", async () => { + // Fake npm only: `ls` reports this checkout as the global install at the daemon's + // version, `install -g` does nothing, and anything else fails so real npm never runs. + const fakeBin = mkdtempSync(path.join(tmpdir(), "tw-npm-")); + extraDirs.push(fakeBin); + const listing = JSON.stringify({ + path: path.join(fakeBin, "global"), + dependencies: { + "@getpaseo/cli": { + version: resolveDaemonVersion(import.meta.url), + path: SERVER_ROOT, + }, + }, + }); + const fakeNpm = path.join(fakeBin, "fake-npm.cjs"); + writeFileSync( + fakeNpm, + `const args = process.argv.slice(2); +const has = (...tokens) => tokens.every((token) => args.includes(token)); +if (has("-g", "ls", "@getpaseo/cli", "--json", "--depth=0", "--long")) { + process.stdout.write(${JSON.stringify(listing)} + "\\n"); +} else if (args[0] === "install" && has("-g", "@getpaseo/cli@latest")) { + // The installed version is already the running one: nothing to do. +} else { + process.stderr.write("unexpected npm call: " + args.join(" ") + "\\n"); + process.exit(1); +} +`, + ); + const npmScript = path.join(fakeBin, "npm"); + writeFileSync(npmScript, `#!/bin/sh\nexec "${process.execPath}" "${fakeNpm}" "$@"\n`); + chmodSync(npmScript, 0o755); + writeFileSync(path.join(fakeBin, "npm.cmd"), `@"${process.execPath}" "${fakeNpm}" %*\r\n`); + const harness = await startSupervisor({ pathPrefix: fakeBin }); + let client = await connect(harness); + const workspaceId = await createWorkspace(client, harness); + const terminal = await createMarkedTerminal(client, harness, workspaceId, "MARKER-UPDATE"); + const daemonPid = await daemonWorkerPid(client); + + const update = await client.updateDaemon(); + expect(update).toMatchObject({ success: true }); + await client.close(); + client = await waitForReplacement(harness, daemonPid); + await expectKept(client, terminal, "MARKER-UPDATE", "update"); + await client.close(); + }); + + test.skipIf(!POSIX)( + "POSIX: a kill restart whose shutdown hangs closes terminals before the respawn", + async () => { + const harness = await startSupervisor(); + let client = await connect(harness); + const workspaceId = await createWorkspace(client, harness); + const terminal = await createMarkedTerminal(client, harness, workspaceId, "MARKER-HUNG"); + const daemonPid = await daemonWorkerPid(client); + const supervisorPid = harness.supervisor.pid!; + const logPath = path.join(harness.home, "daemon.log"); + + // Barrier: the supervisor is paused, so the restart intent waits in its IPC channel. + // The daemon's own log line marks the intent dispatched; then the daemon is paused, so + // its graceful shutdown can never run and the supervisor's force path must. + process.kill(supervisorPid, "SIGSTOP"); + await client.restartServer("settings_update", undefined, { killTerminals: true }); + await waitFor( + () => readFileSync(logPath, "utf8").includes("Lifecycle intent dispatched"), + "the daemon never dispatched the restart intent", + 15_000, + ); + process.kill(daemonPid, "SIGSTOP"); + expect(isAlive(terminal.programPid)).toBe(true); + process.kill(supervisorPid, "SIGCONT"); + await client.close(); + + client = await waitForReplacement(harness, daemonPid); + const log = readFileSync(logPath, "utf8"); + const forced = log.indexOf("forcing process tree kill"); + const cleaned = log.indexOf('"msg":"Terminal cleanup finished","when":"before_respawn"'); + const respawned = log.indexOf('"msg":"Spawning worker"', forced); + expect(forced).toBeGreaterThan(-1); + expect(cleaned).toBeGreaterThan(forced); + expect(respawned).toBeGreaterThan(cleaned); + expect(isAlive(terminal.programPid)).toBe(false); + expect((await client.listTerminals(undefined, undefined, {})).terminals).toEqual([]); + await client.close(); + }, + ); +}); + +describe("terminals when the daemon stops", { timeout: 240_000 }, () => { + test.skipIf(!POSIX)( + "POSIX: stop closes a terminal whose program ignores HUP, child process included", + async () => { + const harness = await startSupervisor(); + const client = await connect(harness); + const workspaceId = await createWorkspace(client, harness); + const terminal = await createHupResistantTerminal( + client, + harness, + workspaceId, + "hup", + "child", + ); + const workerPid = (await currentWorkerRecord(harness.home)).pid; + await client.close(); + await stopDaemon(harness); + await expectAllClosed(harness.home, [terminal.shellPid, terminal.sleepPid, workerPid]); + }, + ); + + for (const force of [false, true]) { + test.skipIf(!POSIX)( + `POSIX: worker death reaps an exec'd HUP-resistant program and ${force ? "stop --force" : "stop"} closes its replacement`, + async () => { + const harness = await startSupervisor(); + const client = await connect(harness); + const workspaceId = await createWorkspace(client, harness); + const terminal = await createHupResistantTerminal( + client, + harness, + workspaceId, + "orphan", + "exec", + ); + const workerPid = (await currentWorkerRecord(harness.home)).pid; + const daemonPid = await daemonWorkerPid(client); + process.kill(workerPid, "SIGKILL"); + await waitUntilGone(workerPid, "terminal worker"); + await waitUntilGone(terminal.sleepPid, "orphaned terminal program"); + await waitFor( + async () => (await client.listTerminals(undefined, undefined, {})).terminals.length === 0, + "dead terminal was not removed", + ); + expect(await daemonWorkerPid(client)).toBe(daemonPid); + const replacement = await createMarkedTerminal(client, harness, workspaceId, "RECOVERED"); + await expectKept(client, replacement, "RECOVERED", "worker-death"); + await client.close(); + await stopDaemonInstance(harness.home, { force }); + await expectAllClosed(harness.home, [terminal.sleepPid, replacement.programPid]); + }, + ); + } + + test.skipIf(!POSIX)( + "POSIX: stop --force kills a stopped (SIGSTOP) worker and its terminal", + async () => { + const harness = await startSupervisor(); + const client = await connect(harness); + const workspaceId = await createWorkspace(client, harness); + const terminal = await createMarkedTerminal(client, harness, workspaceId, "MARKER-STOPPED"); + const workerPid = (await currentWorkerRecord(harness.home)).pid; + await client.close(); + process.kill(workerPid, "SIGSTOP"); + await stopDaemonInstance(harness.home, { force: true }); + await expectAllClosed(harness.home, [workerPid, terminal.programPid]); + }, + ); + + test("the worker socket rejects a wrong secret and leaves its terminals alone", async () => { + const harness = await startSupervisor(); + const client = await connect(harness); + const workspaceId = await createWorkspace(client, harness); + const terminal = await createMarkedTerminal(client, harness, workspaceId, "MARKER-AUTH"); + const record = await currentWorkerRecord(harness.home); + await expect( + connectTerminalWorker({ socketPath: record.socketPath, secret: "not-the-secret" }), + ).rejects.toThrow(); + await expectKept(client, terminal, "MARKER-AUTH", "auth"); + await client.close(); + }); + + test("stop and stop --force never signal processes a stale or earlier-boot record names", async () => { + const harness = await startSupervisor(); + const client = await connect(harness); + await client.close(); + const reused = spawnUnrelated(); + const earlierBoot = spawnUnrelated(); + const base = await currentWorkerRecord(harness.home); + // Same pid, another birth time: what a reused pid looks like, whatever its command line. + writeTerminalWorkerRecord(harness.home, { + ...base, + recordId: "stale-record", + pid: reused.pid!, + identity: { startedAt: "Thu Jan 1 00:00:00 1970", commandLine: "node -e setInterval" }, + terminals: [{ id: "t-stale", pid: reused.pid!, startedAt: "Thu Jan 1 00:00:00 1970" }], + }); + // The right birth time, but written before this boot. + writeTerminalWorkerRecord(harness.home, { + ...base, + recordId: "earlier-boot-record", + pid: earlierBoot.pid!, + bootId: currentBootId() ? "another-boot" : null, + createdAt: "2000-01-01T00:00:00.000Z", + identity: { + startedAt: await birthOf(earlierBoot.pid!), + commandLine: `node --paseo-terminal-worker earlier-boot-record`, + }, + terminals: [ + { id: "t-boot", pid: earlierBoot.pid!, startedAt: await birthOf(earlierBoot.pid!) }, + ], + }); + + await stopDaemon(harness); + expect(isAlive(reused.pid!)).toBe(true); + expect(isAlive(earlierBoot.pid!)).toBe(true); + expect(await liveRecords(harness.home)).toEqual([]); + + // A second round through --force, with the stale record put back. + writeTerminalWorkerRecord(harness.home, { + ...base, + recordId: "stale-record", + pid: reused.pid!, + identity: { startedAt: "Thu Jan 1 00:00:00 1970", commandLine: "node -e setInterval" }, + terminals: [{ id: "t-stale", pid: reused.pid!, startedAt: "Thu Jan 1 00:00:00 1970" }], + }); + await stopDaemonInstance(harness.home, { force: true }); + expect(isAlive(reused.pid!)).toBe(true); + expect(await liveRecords(harness.home)).toEqual([]); + }); +}); + +describe("a worker launched from its own copy", { timeout: 240_000 }, () => { + // What a built daemon does: bundle the worker, copy bundle + shell integration + node-pty into + // the home, launch from the copy. Here the copy's node-pty (ConPTY on Windows) opens a real + // terminal; nothing is loaded from this checkout once the worker runs. + test("opens a terminal and runs a command from the copied bundle and node-pty", async () => { + const home = mkdtempSync(path.join(tmpdir(), "tw-copy-home-")); + const bundleDirectory = mkdtempSync(path.join(tmpdir(), "tw-copy-bundle-")); + extraDirs.push(home, bundleDirectory); + const buildScript = path.join(SERVER_ROOT, "scripts/build-terminal-worker.mjs"); + const { buildTerminalWorkerBundle } = (await import(buildScript)) as { + buildTerminalWorkerBundle(input: { outDir: string }): Promise; + }; + await buildTerminalWorkerBundle({ outDir: bundleDirectory }); + const nodePtyDirectory = path.dirname( + createRequire(import.meta.url).resolve("node-pty/package.json"), + ); + const entry = prepareTerminalWorkerRuntime({ + paseoHome: home, + paseoVersion: "0.0.0-copy", + bundleDirectory, + nodePtyDirectory, + }); + + const launched = await launchTerminalWorker({ + paseoHome: home, + paseoVersion: "0.0.0-copy", + entry, + }); + extraPids.add(launched.record.pid); + expect(launched.record.runtime).toBe(entry.runtime); + const manager = createWorkerTerminalManager({ + worker: await connectTerminalWorker({ + socketPath: launched.record.socketPath, + secret: launched.record.secret, + }), + closeTerminals: async () => { + await reapTerminalWorkers({ paseoHome: home }); + }, + }); + try { + const session = await manager.createTerminal({ + workspaceId: "ws-copy", + cwd: home, + command: process.execPath, + args: ["-e", "console.log('COPY-' + 'OK'); setInterval(() => {}, 1000);"], + }); + await waitFor( + async () => + (await manager.captureTerminal(session.id, { stripAnsi: true })).lines + .join("\n") + .includes("COPY-OK"), + "the terminal in the copied worker never printed its marker", + 30_000, + ); + } finally { + await manager.closeTerminals(); + } + expect(isAlive(launched.record.pid)).toBe(false); + expect(existsSync(path.dirname(entry.scriptPath))).toBe(false); + }); +}); diff --git a/packages/server/src/server/daemon-instance.ts b/packages/server/src/server/daemon-instance.ts index 02f4446795..b3b07e7ff0 100644 --- a/packages/server/src/server/daemon-instance.ts +++ b/packages/server/src/server/daemon-instance.ts @@ -11,6 +11,7 @@ import { } from "./pid-lock.js"; import { daemonLaunchEnvironment } from "./config-environment.js"; import { readPersistedConfig } from "./persisted-config.js"; +import { reapTerminalWorkers } from "../terminal/terminal-worker-reaper.js"; import treeKill from "tree-kill"; const killTree = (pid: number, signal: string): Promise => new Promise((resolve, reject) => @@ -29,6 +30,10 @@ export { resolvePaseoHome } from "./paseo-home.js"; export { ensurePrivateDirectory } from "./private-files.js"; export { daemonLaunchEnvironment } from "./config-environment.js"; export { readLocalCredentialForTarget } from "./local-credential.js"; +export { + TerminalCleanupNotConfirmedError, + retryUnconfirmedTerminalCleanup, +} from "../terminal/terminal-worker-reaper.js"; export { isSamePidLock as isSameDaemonInstance, type PidLockInfo as DaemonInstance, @@ -97,11 +102,23 @@ async function requestInstanceStop( instance: PidLockInfo, options: { force?: boolean; + keepTerminals?: boolean; requestShutdown?: (instance: PidLockInfo & { listen: string }) => Promise; }, ) { let forced = false; let usedLifecycleRpc = false; + if (options.keepTerminals) { + // SIGTERM means "stop and close terminals"; keeping them always goes over the RPC. + if (!instance.listen || !options.requestShutdown) { + throw new DaemonInstanceError( + "STOP_NO_GRACEFUL_CHANNEL", + `PID ${instance.pid} for ${home} has no listen address, so it cannot be asked to keep its terminals.`, + ); + } + await options.requestShutdown({ ...instance, listen: instance.listen }); + return { forced, usedLifecycleRpc: true }; + } if (process.platform === "win32") { if (instance.listen && options.requestShutdown) { try { @@ -142,6 +159,8 @@ export async function stopDaemonInstance( options: { instance?: PidLockInfo; force?: boolean; + // Leave the terminal worker running so the next daemon adopts it. Not valid with force. + keepTerminals?: boolean; timeoutMs?: number; killTimeoutMs?: number; requestShutdown?: (instance: PidLockInfo & { listen: string }) => Promise; @@ -153,6 +172,7 @@ export async function stopDaemonInstance( usedLifecycleRpc: boolean; }> { const { timeoutMs = 15_000, killTimeoutMs = 3_000 } = options; + assertStopOptions(options); const deadline = Date.now() + timeoutMs; const instance = await getPidLockInfo(home); if (options.instance && instance && !isSamePidLock(instance, options.instance)) { @@ -164,6 +184,9 @@ export async function stopDaemonInstance( if (!instance || !isPidLockOwnerRunning(instance)) { if (instance) await releasePidLock(home, { ownerPid: instance.pid, startedAt: instance.startedAt }); + // Terminals a `stop --keep-terminals` left behind belong to no daemon now; a plain stop + // closes them. + await closeLeftoverTerminals(home, { keepTerminals: options.keepTerminals, forced: false }); return { action: "not_running", pid: instance?.pid ?? null, @@ -200,9 +223,29 @@ export async function stopDaemonInstance( `Timed out waiting for supervisor PID ${instance.pid} in ${home} to exit${options.force ? "" : "; use --force to permit forced cleanup"}.`, ); await releasePidLock(home, { ownerPid: instance.pid, startedAt: instance.startedAt }); + // The supervisor already closed the terminals unless it was killed; this confirms it, and + // after --force (no graceful shutdown happened) does it at once. + await closeLeftoverTerminals(home, { keepTerminals: options.keepTerminals, forced }); return { action: "stopped", pid: instance.pid, forced, usedLifecycleRpc }; } +function assertStopOptions(options: { force?: boolean; keepTerminals?: boolean }): void { + if (options.force && options.keepTerminals) { + throw new DaemonInstanceError( + "INVALID_STOP_OPTIONS", + "--force kills terminals; it cannot be combined with --keep-terminals.", + ); + } +} + +async function closeLeftoverTerminals( + home: string, + options: { keepTerminals?: boolean; forced: boolean }, +): Promise { + if (options.keepTerminals) return; + await reapTerminalWorkers({ paseoHome: home, gracefulMs: options.forced ? 0 : undefined }); +} + export async function startDaemonInstance(input: { home: string; command: string; diff --git a/packages/server/src/server/daemon-worker.ts b/packages/server/src/server/daemon-worker.ts index bc5acbc404..317903a592 100644 --- a/packages/server/src/server/daemon-worker.ts +++ b/packages/server/src/server/daemon-worker.ts @@ -13,6 +13,7 @@ type SupervisorLifecycleMessage = | { type: "paseo:shutdown"; reason: string; + keepTerminals: boolean; } | { type: "paseo:ready"; @@ -22,6 +23,7 @@ type SupervisorLifecycleMessage = | { type: "paseo:restart"; reason?: string; + keepTerminals: boolean; }; interface BootstrapResult { @@ -151,12 +153,15 @@ async function main() { options?: { reason?: string; successExitCode?: number; + keepTerminals?: boolean; }, ) => { const reason = options?.reason ?? `worker_received_${signal}`; + // Signals mean "stop and close terminals"; only an explicit keep request keeps them. + const keepTerminals = options?.keepTerminals === true; if (!shutdownPromise) { logger.info( - { signal, reason, ...getProcessDiagnostics() }, + { signal, reason, keepTerminals, ...getProcessDiagnostics() }, `${signal} received, shutting down gracefully...`, ); @@ -175,7 +180,7 @@ async function main() { clearTimeout(forceExit); return 1; } - await daemon.stop(); + await daemon.stop({ keepTerminals }); clearTimeout(forceExit); logger.info("Server closed"); return options?.successExitCode ?? 0; @@ -209,36 +214,53 @@ async function main() { }; const handleLifecycleIntent = (intent: DaemonLifecycleIntent) => { + const fields = { + clientId: intent.clientId, + requestId: intent.requestId, + reason: intent.reason, + keepTerminals: intent.keepTerminals, + }; if (intent.type === "shutdown") { - logger.warn( - { clientId: intent.clientId, requestId: intent.requestId, reason: intent.reason }, - "Shutdown requested via websocket", - ); - if (sendSupervisorLifecycleMessage({ type: "paseo:shutdown", reason: intent.reason })) { + logger.warn(fields, "Shutdown requested via websocket"); + const sent = sendSupervisorLifecycleMessage({ + type: "paseo:shutdown", + reason: intent.reason, + keepTerminals: intent.keepTerminals, + }); + recordLifecycleIntentDispatched(intent.type, fields); + if (sent) { return; } - beginShutdown("shutdown lifecycle intent", { reason: intent.reason }); + beginShutdown("shutdown lifecycle intent", { + reason: intent.reason, + keepTerminals: intent.keepTerminals, + }); return; } - logger.warn( - { clientId: intent.clientId, requestId: intent.requestId, reason: intent.reason }, - "Restart requested via websocket", - ); - if ( - sendSupervisorLifecycleMessage({ - type: "paseo:restart", - ...(intent.reason ? { reason: intent.reason } : {}), - }) - ) { + logger.warn(fields, "Restart requested via websocket"); + const sent = sendSupervisorLifecycleMessage({ + type: "paseo:restart", + ...(intent.reason ? { reason: intent.reason } : {}), + keepTerminals: intent.keepTerminals, + }); + recordLifecycleIntentDispatched(intent.type, fields); + if (sent) { return; } beginShutdown("restart lifecycle intent", { reason: intent.reason, successExitCode: 0, + keepTerminals: intent.keepTerminals, }); }; + // Written straight to daemon.log: the pino stream goes through the supervisor's pipe, which + // a paused or busy supervisor does not drain. Tests synchronize on this line. + const recordLifecycleIntentDispatched = (type: string, fields: Record) => { + writeWorkerLifecycleLog(paseoHome, "Lifecycle intent dispatched", { type, ...fields }); + }; + const installSupervisorLivenessGuard = () => { if (typeof process.send !== "function") { return; @@ -278,9 +300,10 @@ async function main() { return; } if (type === "paseo:graceful-shutdown") { - const reason = (message as { reason?: unknown }).reason; + const { reason, keepTerminals } = message as { reason?: unknown; keepTerminals?: unknown }; beginShutdown("Supervisor shutdown request", { reason: typeof reason === "string" ? reason : "supervisor_requested_shutdown", + keepTerminals: keepTerminals === true, }); } }); diff --git a/packages/server/src/server/managed-processes/managed-processes.test.ts b/packages/server/src/server/managed-processes/managed-processes.test.ts index e247db027f..b4be4d16bd 100644 --- a/packages/server/src/server/managed-processes/managed-processes.test.ts +++ b/packages/server/src/server/managed-processes/managed-processes.test.ts @@ -12,6 +12,7 @@ import { type ManagedProcessInspection, type ManagedProcessSnapshot, type ManagedProcessTable, + windowsPowerShellPath, } from "./managed-processes.js"; import { spawnProcess } from "../../utils/spawn.js"; import { @@ -333,7 +334,7 @@ describe("system managed process table", () => { }); expect(commandRunner.commands).toEqual([ { - command: "powershell.exe", + command: windowsPowerShellPath(), args: [ "-NoProfile", "-NonInteractive", @@ -345,6 +346,17 @@ describe("system managed process table", () => { }); }); +describe("windowsPowerShellPath", () => { + test("names Windows PowerShell under SystemRoot, whatever PATH holds", () => { + expect(windowsPowerShellPath({ SystemRoot: "D:\\Win", PATH: "" })).toBe( + "D:\\Win\\System32\\WindowsPowerShell\\v1.0\\powershell.exe", + ); + expect(windowsPowerShellPath({})).toBe( + "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe", + ); + }); +}); + class FakeProcessTable implements ManagedProcessTable { private readonly snapshots: Map; private readonly errorPids: Set; diff --git a/packages/server/src/server/managed-processes/managed-processes.ts b/packages/server/src/server/managed-processes/managed-processes.ts index 0031080c67..43c1b3c810 100644 --- a/packages/server/src/server/managed-processes/managed-processes.ts +++ b/packages/server/src/server/managed-processes/managed-processes.ts @@ -7,6 +7,20 @@ import { writeJsonFileAtomic } from "../atomic-file.js"; import { execCommand } from "../../utils/spawn.js"; import type { ProcessTerminator, TreeKillTarget } from "../../utils/tree-kill.js"; +/** + * Windows PowerShell by its install path. Process identity must not depend on the caller's + * PATH: a daemon or terminal worker can run with a PATH that does not reach System32. + */ +export function windowsPowerShellPath(env: NodeJS.ProcessEnv = process.env): string { + return path.win32.join( + env.SystemRoot ?? env.SYSTEMROOT ?? "C:\\Windows", + "System32", + "WindowsPowerShell", + "v1.0", + "powershell.exe", + ); +} + const MANAGED_PROCESS_GRACEFUL_SHUTDOWN_TIMEOUT_MS = 5_000; const MANAGED_PROCESS_FORCE_SHUTDOWN_TIMEOUT_MS = 1_000; const MANAGED_PROCESS_EXIT_POLL_INTERVAL_MS = 50; @@ -183,7 +197,7 @@ class SystemManagedProcessTable implements ManagedProcessTable { `$process = Get-CimInstance Win32_Process -Filter 'ProcessId = ${pid}';`, "if ($process) { $process | Select-Object ProcessId,CommandLine,CreationDate | ConvertTo-Json -Compress }", ].join(" "); - const { stdout } = await this.commandRunner.exec("powershell.exe", [ + const { stdout } = await this.commandRunner.exec(windowsPowerShellPath(), [ "-NoProfile", "-NonInteractive", "-Command", diff --git a/packages/server/src/server/pid-lock.ts b/packages/server/src/server/pid-lock.ts index 314b44ea47..6f3e7135d9 100644 --- a/packages/server/src/server/pid-lock.ts +++ b/packages/server/src/server/pid-lock.ts @@ -30,13 +30,16 @@ function isErrnoException(err: unknown): err is NodeJS.ErrnoException { return err instanceof Error && "code" in err; } +// The field is assigned explicitly rather than declared as a constructor parameter property: +// the terminal worker loads this module under --experimental-strip-types, which cannot emit +// the assignment a parameter property implies. export class PidLockError extends Error { - constructor( - message: string, - public readonly existingLock?: PidLockInfo, - ) { + readonly existingLock?: PidLockInfo; + + constructor(message: string, existingLock?: PidLockInfo) { super(message); this.name = "PidLockError"; + this.existingLock = existingLock; } } @@ -69,7 +72,7 @@ let cachedBootId: string | null | undefined; // Linux names each boot. The wall-clock boot instant is not enough there: a VM paused while // its host sleeps keeps its uptime, so after it resumes the instant lands after locks its // running supervisor wrote. -function currentBootId(): string | null { +export function currentBootId(): string | null { if (cachedBootId === undefined) { try { cachedBootId = readFileSync("/proc/sys/kernel/random/boot_id", "utf-8").trim() || null; @@ -80,10 +83,17 @@ function currentBootId(): string | null { return cachedBootId; } -function writtenDuringThisBoot(lock: PidLockInfo): boolean { +/** + * Whether a record stamped with `startedAt` (and, on Linux, `bootId`) was written during the + * current boot. Daemon locks and terminal worker records share this rule. + */ +export function writtenDuringThisBoot(stamp: { + startedAt: string; + bootId?: string | null; +}): boolean { const thisBoot = currentBootId(); - if (lock.bootId && thisBoot) return lock.bootId === thisBoot; - return !precedesThisBoot(lock.startedAt); + if (stamp.bootId && thisBoot) return stamp.bootId === thisBoot; + return !precedesThisBoot(stamp.startedAt); } /** diff --git a/packages/server/src/server/plugins/index.posix.test.ts b/packages/server/src/server/plugins/index.posix.test.ts index 4ca5aeefd5..17f25fce3b 100644 --- a/packages/server/src/server/plugins/index.posix.test.ts +++ b/packages/server/src/server/plugins/index.posix.test.ts @@ -15,7 +15,7 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; import pino from "pino"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, beforeAll, describe, expect, it } from "vitest"; import { DaemonConfigStore } from "../daemon-config-store.js"; import { PluginService } from "./index.js"; import { BuiltinPluginLoader } from "./builtin/index.js"; @@ -1321,7 +1321,11 @@ export default function contribute(plugin: unknown) { }); describe("npm plugin installation", () => { - it("preserves dependencies and metadata, and reloads without npm", async () => { + let registry: Awaited>; + + // Packing the registry starts eleven real npm processes. Give fixture construction its + // own setup budget so Windows process startup does not consume the installation journey. + beforeAll(async () => { const packages = npmPluginPackages(); const fixture = packages[1]; packages.push({ ...fixture, version: "2.0.0", tags: ["future"] }); @@ -1356,7 +1360,7 @@ describe("npm plugin installation", () => { for (const [name, overrides] of failures) { failedPackages.push({ ...fixture, name, files: { ...fixture.files, ...overrides } }); } - const registry = await startNpmRegistry([ + registry = await startNpmRegistry([ ...packages, { name: "paseo-prepared-plugin", @@ -1386,6 +1390,9 @@ export default function contribute() { }, ...failedPackages, ]); + }, 120_000); + + it("preserves dependencies and metadata, and reloads without npm", async () => { const home = await mkdtemp(path.join(tmpdir(), "paseo-npm-service-")); roots.push(home); const previousConfig = process.env.npm_config_userconfig; diff --git a/packages/server/src/server/session.test.ts b/packages/server/src/server/session.test.ts index 3cdef95a83..d1408f9880 100644 --- a/packages/server/src/server/session.test.ts +++ b/packages/server/src/server/session.test.ts @@ -8,7 +8,15 @@ import { createTestCreationService, } from "./test-utils/session-stubs.js"; import { execSync } from "child_process"; -import { existsSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from "fs"; import { tmpdir } from "os"; import { join, resolve as resolvePath } from "path"; import pino from "pino"; @@ -28,6 +36,11 @@ import { type FileTransferFrame, } from "@getpaseo/protocol/binary-frames/index"; import { Session } from "./session.js"; +import { + terminalCleanupRecordPath, + terminalWorkerDirectory, + writeTerminalCleanupRecord, +} from "../terminal/terminal-worker-record.js"; import { OWNER_PERMISSIONS, type DaemonPermission } from "./authorization/index.js"; import { DownloadTokenStore } from "./file-download/token-store.js"; import { StructuredAgentFallbackError } from "./agent/agent-response-loop.js"; @@ -2071,11 +2084,84 @@ describe("daemon status + pairing RPC", () => { { provider: "claude", available: true, error: null }, { provider: "codex", available: false, error: "boom" }, ], + terminalCleanupUnconfirmed: [], }, }, ]); }); + test("daemon.get_status.request reports what a terminal cleanup could not confirm gone", async () => { + const messages: unknown[] = []; + const paseoHome = makeHome(); + const unresolved = { + pid: 4242, + role: "terminal program" as const, + reason: "still alive" as const, + detail: "survived SIGKILL", + }; + await writeTerminalCleanupRecord(paseoHome, { + recordId: "closing-record", + pid: 4241, + bootId: null, + createdAt: new Date().toISOString(), + identity: { startedAt: "Thu Jan 1 00:00:00 1970", commandLine: "worker" }, + protocolVersion: 1, + paseoVersion: "0.0.0-test", + socketPath: join(paseoHome, "closing.sock"), + secret: "closing-secret", + terminals: [{ id: "t1", pid: 4242, startedAt: "Thu Jan 1 00:00:00 1970" }], + closing: { unresolved: [unresolved] }, + }); + const session = createSessionForTest({ + messages, + paseoHome, + serverId: "srv-test", + daemonVersion: "9.9.9", + daemonRuntimeConfig: { listen: "127.0.0.1:6767", getRelayConfig: () => null }, + agentManager: { listProviderAvailability: vi.fn().mockResolvedValue([]) }, + }); + + await session.handleMessage({ type: "daemon.get_status.request", requestId: "status-2" }); + + expect(messages).toEqual([ + expect.objectContaining({ + type: "daemon.get_status.response", + payload: expect.objectContaining({ terminalCleanupUnconfirmed: [unresolved] }), + }), + ]); + }); + + test("daemon.get_status.request fails when a terminal cleanup record cannot be read", async () => { + const messages: unknown[] = []; + const paseoHome = makeHome(); + mkdirSync(terminalWorkerDirectory(paseoHome), { recursive: true }); + const cleanupPath = terminalCleanupRecordPath(paseoHome, "closing-record"); + writeFileSync(cleanupPath, "{ not json"); + const session = createSessionForTest({ + messages, + paseoHome, + serverId: "srv-test", + daemonVersion: "9.9.9", + daemonRuntimeConfig: { listen: "127.0.0.1:6767", getRelayConfig: () => null }, + agentManager: { listProviderAvailability: vi.fn().mockResolvedValue([]) }, + }); + + await session.handleMessage({ type: "daemon.get_status.request", requestId: "status-3" }); + + expect(messages).toContainEqual( + expect.objectContaining({ + type: "rpc_error", + payload: expect.objectContaining({ + requestId: "status-3", + error: expect.stringContaining(cleanupPath), + }), + }), + ); + expect(messages).not.toContainEqual( + expect.objectContaining({ type: "daemon.get_status.response" }), + ); + }); + test("daemon.get_status.request falls back to a null/empty status when provider listing fails", async () => { const messages: unknown[] = []; const session = createSessionForTest({ @@ -2107,6 +2193,7 @@ describe("daemon status + pairing RPC", () => { listen: null, relay: null, providers: [], + terminalCleanupUnconfirmed: [], }, }, ]); diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 2ff8e917ab..3eac37cb7b 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -577,12 +577,14 @@ export type SessionLifecycleIntent = clientId: string; requestId: string; reason: string; + keepTerminals: boolean; } | { type: "restart"; clientId: string; requestId: string; reason: string; + keepTerminals: boolean; }; function parseClientCapabilities( @@ -1105,6 +1107,7 @@ export class Session { "Terminal output": this.terminalController.getMetrics().streamSubscriptionCount, }), listProviderAvailability: () => this.agentManager.listProviderAvailability(), + getTerminalsUnavailable: () => terminalManager?.getUnavailableError?.()?.message ?? null, listAgents: () => this.agentManager.listAgents(), listProjects: () => this.projectRegistry.list(), listWorkspaces: () => this.workspaceRegistry.list(), @@ -2728,9 +2731,13 @@ export class Session { case "dictation_stream_cancel": return this.voiceSessions.handleMessage(msg); case "restart_server_request": - return this.handleRestartServerRequest(msg.requestId, msg.reason); + return this.handleRestartServerRequest(msg.requestId, msg.reason, { + keepTerminals: msg.killTerminals !== true, + }); case "shutdown_server_request": - return this.handleShutdownServerRequest(msg.requestId); + return this.handleShutdownServerRequest(msg.requestId, { + keepTerminals: msg.keepTerminals === true, + }); case "client_heartbeat": this.handleClientHeartbeat(msg); return undefined; @@ -3256,7 +3263,11 @@ export class Session { this.terminalController.handleBinaryFrame(binaryFrame.frame, source); } - private async handleRestartServerRequest(requestId: string, reason?: string): Promise { + private async handleRestartServerRequest( + requestId: string, + reason: string | undefined, + options: { keepTerminals: boolean }, + ): Promise { const lifecycleReason = normalizeClientRestartRpcReason(reason); const payload: { status: string } & Record = { status: "restart_requested", @@ -3267,7 +3278,10 @@ export class Session { } payload.requestId = requestId; - this.sessionLogger.warn({ reason: lifecycleReason }, "Restart requested via websocket"); + this.sessionLogger.warn( + { reason: lifecycleReason, keepTerminals: options.keepTerminals }, + "Restart requested via websocket", + ); this.emit({ type: "status", payload, @@ -3278,12 +3292,19 @@ export class Session { clientId: this.clientId, requestId, reason: lifecycleReason, + keepTerminals: options.keepTerminals, }); } - private async handleShutdownServerRequest(requestId: string): Promise { + private async handleShutdownServerRequest( + requestId: string, + options: { keepTerminals: boolean }, + ): Promise { const reason = CLIENT_SHUTDOWN_RPC_REASON; - this.sessionLogger.warn({ reason }, "Shutdown requested via websocket"); + this.sessionLogger.warn( + { reason, keepTerminals: options.keepTerminals }, + "Shutdown requested via websocket", + ); this.emit({ type: "status", payload: { @@ -3298,6 +3319,7 @@ export class Session { clientId: this.clientId, requestId, reason, + keepTerminals: options.keepTerminals, }); } diff --git a/packages/server/src/server/session/daemon/daemon-self-update-session-controller.test.ts b/packages/server/src/server/session/daemon/daemon-self-update-session-controller.test.ts index 81c41f3421..e9bcf94b65 100644 --- a/packages/server/src/server/session/daemon/daemon-self-update-session-controller.test.ts +++ b/packages/server/src/server/session/daemon/daemon-self-update-session-controller.test.ts @@ -118,6 +118,7 @@ describe("DaemonSelfUpdateSessionController", () => { clientId: "client-1", requestId: "update-1", reason: "daemon_update", + keepTerminals: true, }, ]); }); diff --git a/packages/server/src/server/session/daemon/daemon-self-update-session-controller.ts b/packages/server/src/server/session/daemon/daemon-self-update-session-controller.ts index 461a97277b..bb75f008bd 100644 --- a/packages/server/src/server/session/daemon/daemon-self-update-session-controller.ts +++ b/packages/server/src/server/session/daemon/daemon-self-update-session-controller.ts @@ -19,6 +19,7 @@ interface DaemonSelfUpdateRestartIntent { clientId: string; requestId: string; reason: string; + keepTerminals: boolean; } export interface DaemonSelfUpdateSessionControllerOptions { @@ -88,6 +89,8 @@ export class DaemonSelfUpdateSessionController { clientId: this.clientId, requestId: msg.requestId, reason: "daemon_update", + // A self-update restart keeps terminals: the worker outlives the daemon. + keepTerminals: true, }); } catch (error) { if (error instanceof DaemonSelfUpdateInProgressError) { diff --git a/packages/server/src/server/session/daemon/daemon-session.test.ts b/packages/server/src/server/session/daemon/daemon-session.test.ts index e96256c4cf..cb6da8777b 100644 --- a/packages/server/src/server/session/daemon/daemon-session.test.ts +++ b/packages/server/src/server/session/daemon/daemon-session.test.ts @@ -212,6 +212,7 @@ describe("DaemonSession", () => { { provider: "claude", available: true, error: null }, { provider: "codex", available: false, error: "boom" }, ], + terminalCleanupUnconfirmed: [], }, }, ]); @@ -242,6 +243,7 @@ describe("DaemonSession", () => { listen: null, relay: null, providers: [], + terminalCleanupUnconfirmed: [], }, }, ]); diff --git a/packages/server/src/server/session/daemon/daemon-session.ts b/packages/server/src/server/session/daemon/daemon-session.ts index ccc9e66e6d..585fa45573 100644 --- a/packages/server/src/server/session/daemon/daemon-session.ts +++ b/packages/server/src/server/session/daemon/daemon-session.ts @@ -2,6 +2,7 @@ import type pino from "pino"; import type { ProviderAvailability } from "../../agent/agent-manager.js"; import type { SessionInboundMessage, SessionOutboundMessage } from "../../messages.js"; import { getPidLockInfo } from "../../pid-lock.js"; +import { readUnconfirmedTerminalCleanup } from "../../../terminal/terminal-worker-reaper.js"; import { generateLocalPairingOffer } from "../../pairing-offer.js"; import { collectDaemonDiagnostics, @@ -35,6 +36,7 @@ export interface DaemonSessionHost { clientId: string; requestId: string; reason: string; + keepTerminals: boolean; }): void; } @@ -49,6 +51,8 @@ export interface DaemonSessionOptions { listProjects: () => Promise; listWorkspaces: () => Promise; listProviderAvailability: () => Promise; + /** Why this host's terminals cannot be served (held by a newer Paseo's worker), if so. */ + getTerminalsUnavailable?: () => string | null; getWebSocketRuntimeMetrics?: () => DaemonWebSocketRuntimeDiagnosticSnapshot | null; getObservationMetrics?: () => Record; logger: pino.Logger; @@ -74,6 +78,7 @@ export class DaemonSession { private readonly listProjects: () => Promise; private readonly listWorkspaces: () => Promise; private readonly listProviderAvailability: () => Promise; + private readonly getTerminalsUnavailable: () => string | null; private readonly getWebSocketRuntimeMetrics: () => DaemonWebSocketRuntimeDiagnosticSnapshot | null; private readonly getObservationMetrics: DaemonSessionOptions["getObservationMetrics"]; private readonly logger: pino.Logger; @@ -92,6 +97,7 @@ export class DaemonSession { this.listProjects = options.listProjects; this.listWorkspaces = options.listWorkspaces; this.listProviderAvailability = options.listProviderAvailability; + this.getTerminalsUnavailable = options.getTerminalsUnavailable ?? (() => null); this.getWebSocketRuntimeMetrics = options.getWebSocketRuntimeMetrics ?? (() => null); this.getObservationMetrics = options.getObservationMetrics; this.logger = options.logger; @@ -173,6 +179,12 @@ export class DaemonSession { async handleGetStatusRequest( msg: Extract, ): Promise { + // Read outside the degraded-status fallback below: a status that cannot say whether + // terminal cleanup finished must fail the request, not report that nothing is left. + const terminalCleanupUnconfirmed = (await readUnconfirmedTerminalCleanup(this.paseoHome)).map( + ({ pid, role, reason, detail }) => ({ pid, role, reason, detail }), + ); + const terminalsUnavailable = this.getTerminalsUnavailable(); try { const pidInfo = await getPidLockInfo(this.paseoHome); const providers = (await this.listProviderAvailability()).map((p) => ({ @@ -192,6 +204,8 @@ export class DaemonSession { listen: this.daemonRuntimeConfig?.listen ?? null, relay: this.daemonRuntimeConfig?.getRelayConfig() ?? null, providers, + terminalCleanupUnconfirmed, + ...(terminalsUnavailable ? { terminalsUnavailable } : {}), }, }); } catch (error) { @@ -208,6 +222,8 @@ export class DaemonSession { listen: null, relay: null, providers: [], + terminalCleanupUnconfirmed, + ...(terminalsUnavailable ? { terminalsUnavailable } : {}), }, }); } diff --git a/packages/server/src/server/test-utils/paseo-daemon.test.ts b/packages/server/src/server/test-utils/paseo-daemon.test.ts index 4ee417c221..3fbb0b5232 100644 --- a/packages/server/src/server/test-utils/paseo-daemon.test.ts +++ b/packages/server/src/server/test-utils/paseo-daemon.test.ts @@ -45,7 +45,15 @@ describe("createTestPaseoDaemon", () => { createTestPaseoDaemon({ listenPort: takenPort, cleanup: false, mcpEnabled: false }), ).rejects.toMatchObject({ code: "EADDRINUSE" }); - expect(await readdir(scratchTmp)).toEqual([]); + // A home under this scratch dir is too long for a unix socket path, so the terminal worker + // listens in the per-user `paseo-` directory of the temp root. Like tmux's, that + // owner-only directory is shared by every worker and outlives them; it must be empty. + const sharedSocketDir = `paseo-${process.getuid?.() ?? 0}`; + const leftovers = await readdir(scratchTmp); + expect(leftovers.filter((name) => name !== sharedSocketDir)).toEqual([]); + if (leftovers.includes(sharedSocketDir)) { + expect(await readdir(path.join(scratchTmp, sharedSocketDir))).toEqual([]); + } }, 60_000); }); diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index dfa324315c..6705c1cdd9 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -2054,6 +2054,8 @@ export class VoiceAssistantWebSocketServer { daemonDiagnostics: true, // COMPAT(daemonSelfUpdate): added in v0.1.93, remove gate after 2026-12-13. daemonSelfUpdate: this.daemonRuntimeConfig?.desktopManaged !== true, + // COMPAT(terminalsSurviveRestart): added in v0.12.0, remove gate after 2027-04-21. + terminalsSurviveRestart: true, // COMPAT(agentForkContext): added in v0.1.102, remove gate after 2026-12-28. agentForkContext: true, // COMPAT(agentForkContextCursor): added in v0.1.108, remove gate after 2027-01-14. diff --git a/packages/server/src/server/workspace-git-service.test.ts b/packages/server/src/server/workspace-git-service.test.ts index b3fab7b89a..f3571f9932 100644 --- a/packages/server/src/server/workspace-git-service.test.ts +++ b/packages/server/src/server/workspace-git-service.test.ts @@ -796,17 +796,21 @@ describe("WorkspaceGitServiceImpl", () => { const service = createService({ getCheckoutStatus, subscribe, runGitCommand }); cwds.forEach((cwd) => service.registerWorkspace({ cwd }, vi.fn())); - await vi.waitFor(() => { - // Admission precedes async path resolution; disposal must race pending subscriptions. - expect(subscribe).toHaveBeenCalledTimes(WORKSPACE_GIT_OBSERVATION_SETUP_CONCURRENCY); - expect(service.getMetrics()).toMatchObject({ - workspaceRefreshAdmissionActiveCount: WORKSPACE_GIT_REFRESH_CONCURRENCY, - workspaceRefreshAdmissionPendingCount: 1, - workspaceObservationSetupAdmissionActiveCount: WORKSPACE_GIT_OBSERVATION_SETUP_CONCURRENCY, - workspaceObservationSetupAdmissionPendingCount: - cwds.length - WORKSPACE_GIT_OBSERVATION_SETUP_CONCURRENCY, - }); - }); + await vi.waitFor( + () => { + expect(service.getMetrics()).toMatchObject({ + workspaceRefreshAdmissionActiveCount: WORKSPACE_GIT_REFRESH_CONCURRENCY, + workspaceRefreshAdmissionPendingCount: 1, + workspaceObservationSetupAdmissionActiveCount: + WORKSPACE_GIT_OBSERVATION_SETUP_CONCURRENCY, + workspaceObservationSetupAdmissionPendingCount: + cwds.length - WORKSPACE_GIT_OBSERVATION_SETUP_CONCURRENCY, + }); + // Admission precedes asynchronous path setup; dispose only once watchers are pending. + expect(subscribe).toHaveBeenCalledTimes(WORKSPACE_GIT_OBSERVATION_SETUP_CONCURRENCY); + }, + { timeout: 10_000 }, + ); const queuedRefreshSettlement = service.getSnapshot(cwds.at(-1)!).then( () => ({ status: "fulfilled" as const, error: null }), (error: unknown) => ({ status: "rejected" as const, error }), diff --git a/packages/server/src/terminal/paseo-cli-path.ts b/packages/server/src/terminal/paseo-cli-path.ts new file mode 100644 index 0000000000..18dff56f1b --- /dev/null +++ b/packages/server/src/terminal/paseo-cli-path.ts @@ -0,0 +1,83 @@ +import { existsSync } from "node:fs"; +import { createRequire } from "node:module"; +import { dirname, join, resolve as resolvePath } from "node:path"; + +// Where the Paseo CLI lives, for the PATH entry and PASEO_HOOK_CLI every terminal gets. Kept +// apart from terminal.ts so the daemon can resolve it without loading node-pty. + +const require = createRequire(import.meta.url); +const PASEO_CLI_BIN_ENTRY = "@getpaseo/cli/bin/paseo"; + +export function resolveExternalProcessPath(filePath: string): string { + return filePath.replace(/\.asar(?=[/\\]|$)/, ".asar.unpacked"); +} + +export function resolvePaseoCliBinDir(): string | null { + const cliExecutable = resolvePaseoCliExecutablePath(); + return cliExecutable ? dirname(cliExecutable) : null; +} + +export function resolvePaseoCliExecutablePath(): string | null { + const configuredCli = process.env.PASEO_CLI?.trim(); + if (configuredCli) { + return resolvePath(configuredCli); + } + + const cliEntrypoint = resolvePaseoCliBinEntrypoint(); + if (!cliEntrypoint) { + return null; + } + + const externalCliEntrypoint = resolveExternalProcessPath(cliEntrypoint); + const npmBinDir = findNpmBinDir(dirname(externalCliEntrypoint)); + if (npmBinDir) { + const shim = resolvePaseoCliShim(npmBinDir); + if (shim) { + return shim; + } + } + + return externalCliEntrypoint; +} + +function resolvePaseoCliBinEntrypoint(): string | null { + try { + return require.resolve(PASEO_CLI_BIN_ENTRY); + } catch { + return null; + } +} + +function findNpmBinDir(startPath: string): string | null { + let current = startPath; + while (true) { + const candidate = join(current, "node_modules", ".bin"); + if (hasPaseoCliShim(candidate)) { + return candidate; + } + + const parent = dirname(current); + if (parent === current) { + return null; + } + current = parent; + } +} + +function hasPaseoCliShim(binDir: string): boolean { + return resolvePaseoCliShim(binDir) !== null; +} + +function resolvePaseoCliShim(binDir: string): string | null { + for (const name of paseoCliShimNames()) { + const candidate = join(binDir, name); + if (existsSync(candidate)) { + return candidate; + } + } + return null; +} + +function paseoCliShimNames(): string[] { + return process.platform === "win32" ? ["paseo.cmd", "paseo.exe", "paseo"] : ["paseo"]; +} diff --git a/packages/server/src/terminal/terminal-manager-factory.ts b/packages/server/src/terminal/terminal-manager-factory.ts deleted file mode 100644 index 88dcc2d692..0000000000 --- a/packages/server/src/terminal/terminal-manager-factory.ts +++ /dev/null @@ -1,12 +0,0 @@ -import type { TerminalManager } from "./terminal-manager.js"; -import { createWorkerTerminalManager } from "./worker-terminal-manager.js"; - -export interface ConfiguredTerminalManagerOptions { - getTerminalActivityUrl?: () => string | null; -} - -export function createConfiguredTerminalManager( - options: ConfiguredTerminalManagerOptions = {}, -): TerminalManager { - return createWorkerTerminalManager(options); -} diff --git a/packages/server/src/terminal/terminal-manager.ts b/packages/server/src/terminal/terminal-manager.ts index df4ced389b..86b4cdb662 100644 --- a/packages/server/src/terminal/terminal-manager.ts +++ b/packages/server/src/terminal/terminal-manager.ts @@ -1,5 +1,6 @@ import { createTerminal, + type LocalTerminalSession, type TerminalActivityTransition, type TerminalSession, type TerminalStateSnapshot, @@ -50,6 +51,11 @@ export type TerminalWorkspaceContributionChangedListener = ( ) => void; export interface TerminalManager { + /** + * Set when this host's terminals cannot be served at all (a worker from a newer Paseo holds + * them). Callers that answer terminal requests report it instead of an empty list. + */ + getUnavailableError?(): Error | null; getTerminals(cwd: string, options?: { workspaceId?: string }): Promise; createTerminal(options: { id?: string; @@ -64,6 +70,8 @@ export interface TerminalManager { cols?: number; activityToken?: string; activityUrl?: string | null; + baseEnv?: Record; + paseoCliPath?: string | null; }): Promise; registerCwdEnv(options: { cwd: string; env: Record }): void; validateTerminalActivityToken(terminalId: string, token: string): "valid" | "unknown" | "invalid"; @@ -97,15 +105,21 @@ export interface TerminalManagerOptions { getTerminalActivityUrl?: () => string | null; } +/** The in-process manager the terminal worker runs: it owns each PTY, so it knows its pid. */ +export interface LocalTerminalManager extends TerminalManager { + getTerminalProcessId(id: string): number | null; + getTerminalActivityToken(id: string): string | null; +} + function createActivityToken(): string { return randomBytes(32).toString("base64url"); } export function createTerminalManager( managerOptions: TerminalManagerOptions = {}, -): TerminalManager { +): LocalTerminalManager { const terminalsByCwd = new Map(); - const terminalsById = new Map(); + const terminalsById = new Map(); const terminalExitUnsubscribeById = new Map void>(); const terminalTitleUnsubscribeById = new Map void>(); const terminalActivityUnsubscribeById = new Map void>(); @@ -186,7 +200,7 @@ export function createTerminalManager( return bestMatchRoot ? defaultEnvByRootCwd.get(bestMatchRoot) : undefined; } - function registerSession(session: TerminalSession): TerminalSession { + function registerSession(session: LocalTerminalSession): LocalTerminalSession { terminalsById.set(session.id, session); const unsubscribeExit = session.onExit(() => { removeSessionById(session.id, { kill: false }); @@ -321,6 +335,8 @@ export function createTerminalManager( cols?: number; activityToken?: string; activityUrl?: string | null; + baseEnv?: Record; + paseoCliPath?: string | null; }): Promise { assertAbsolutePath(options.cwd); @@ -355,6 +371,8 @@ export function createTerminalManager( ...(options.rows !== undefined ? { rows: options.rows } : {}), ...(options.cols !== undefined ? { cols: options.cols } : {}), ...(mergedEnv ? { env: mergedEnv } : {}), + ...(options.baseEnv ? { baseEnv: options.baseEnv } : {}), + ...(options.paseoCliPath === undefined ? {} : { paseoCliPath: options.paseoCliPath }), activityEnv, }), ); @@ -390,6 +408,14 @@ export function createTerminalManager( return terminalsById.get(id); }, + getTerminalProcessId(id: string): number | null { + return terminalsById.get(id)?.pid ?? null; + }, + + getTerminalActivityToken(id: string): string | null { + return terminalActivityTokenById.get(id) ?? null; + }, + async getTerminalState( id: string, options?: TerminalStateSnapshotOptions, diff --git a/packages/server/src/terminal/terminal-session-controller.test.ts b/packages/server/src/terminal/terminal-session-controller.test.ts index b438af8381..27f69cb44d 100644 --- a/packages/server/src/terminal/terminal-session-controller.test.ts +++ b/packages/server/src/terminal/terminal-session-controller.test.ts @@ -17,6 +17,7 @@ import { SessionDelivery } from "../server/session/owned-subscriptions/index.js" import type { TerminalManager, TerminalsChangedEvent } from "./terminal-manager.js"; import { isSameOrDescendantPath } from "../server/path-utils.js"; import { PluginSessionSocket } from "../server/plugins/session-socket.js"; +import { TerminalWorkerIncompatibleError } from "./terminal-worker-pool.js"; function createController( options: TerminalSessionControllerOptions & { emitBinary(frame: Uint8Array): void }, @@ -347,6 +348,76 @@ describe("terminal-session-controller legacy terminal creation", () => { }); }); +describe("terminal-session-controller with terminals held by a newer Paseo", () => { + const incompatible = new TerminalWorkerIncompatibleError({ + paseoVersion: "9.9.9", + protocolVersion: 7, + }); + + function incompatibleManager(): TerminalManager { + return { + getUnavailableError: () => incompatible, + getTerminals: vi.fn(async () => Promise.reject(incompatible)), + createTerminal: vi.fn(async () => Promise.reject(incompatible)), + registerCwdEnv: vi.fn(), + validateTerminalActivityToken: vi.fn(() => "unknown"), + getTerminal: vi.fn(), + getTerminalState: vi.fn(), + setTerminalTitle: vi.fn(), + setTerminalActivity: vi.fn(), + clearTerminalAttention: vi.fn(), + killTerminal: vi.fn(), + killTerminalAndWait: vi.fn(), + captureTerminal: vi.fn(), + // The incompatible worker's terminals are not in this daemon's view. + listDirectories: vi.fn(() => []), + killAll: vi.fn(), + subscribeTerminalsChanged: vi.fn(() => vi.fn()), + subscribeTerminalActivity: vi.fn(() => vi.fn()), + subscribeTerminalWorkspaceContributionChanged: vi.fn(() => vi.fn()), + }; + } + + test.each([ + { type: "list_terminals_request" as const, requestId: "req-list" }, + { type: "list_terminals_request" as const, requestId: "req-list-cwd", cwd: "/work/repo" }, + { + type: "create_terminal_request" as const, + requestId: "req-create", + cwd: "/work/repo", + workspaceId: "ws-1", + }, + ])( + "answers $type ($requestId) with a terminal_worker_incompatible rpc_error", + async (request) => { + const outboundMessages: SessionOutboundMessage[] = []; + const controller = createController({ + terminalManager: incompatibleManager(), + emit: (message) => outboundMessages.push(message), + emitBinary: vi.fn(), + hasBinaryChannel: () => true, + isPathWithinRoot: isSameOrDescendantPath, + sessionLogger: createLogger(), + listTerminalWorkspaceRefs: async () => [{ workspaceId: "ws-1", cwd: "/work/repo" }], + }); + + await controller.dispatch(request); + + expect(outboundMessages).toEqual([ + { + type: "rpc_error", + payload: { + requestId: request.requestId, + requestType: request.type, + error: incompatible.message, + code: "terminal_worker_incompatible", + }, + }, + ]); + }, + ); +}); + async function flushMicrotasks(): Promise { await new Promise((resolve) => setTimeout(resolve, 0)); } diff --git a/packages/server/src/terminal/terminal-session-controller.ts b/packages/server/src/terminal/terminal-session-controller.ts index 4f46dbc88c..e41186fdb1 100644 --- a/packages/server/src/terminal/terminal-session-controller.ts +++ b/packages/server/src/terminal/terminal-session-controller.ts @@ -38,6 +38,7 @@ import { import type { TerminalSession } from "./terminal.js"; import type { TerminalManager, TerminalsChangedEvent } from "./terminal-manager.js"; import { applyTerminalSize } from "./terminal-size-ownership.js"; +import { TerminalWorkerIncompatibleError } from "./terminal-worker-pool.js"; import type { TerminalActivity } from "@getpaseo/protocol/terminal-activity"; import { terminalSubscriptionKey } from "@getpaseo/protocol/terminal-subscription-key"; @@ -427,6 +428,8 @@ export class TerminalSessionController { } try { + const unavailable = this.terminalManager.getUnavailableError?.(); + if (unavailable) throw unavailable; let terminals: TerminalSession[]; if (msg.workspaceId !== undefined) { terminals = (await this.getAllTerminalSessions()).filter( @@ -449,6 +452,10 @@ export class TerminalSessionController { }); } catch (error) { this.sessionLogger.error({ err: error, cwd: msg.cwd }, "Failed to list terminals"); + if (error instanceof TerminalWorkerIncompatibleError) { + this.emitIncompatible(msg, error); + return; + } this.emit({ type: "list_terminals_response", payload: { @@ -460,6 +467,23 @@ export class TerminalSessionController { } } + // An empty list or a generic failure would read as "no terminals"; the client needs to know + // they exist and why it cannot reach them. + private emitIncompatible( + msg: { type: string; requestId: string }, + error: TerminalWorkerIncompatibleError, + ): void { + this.emit({ + type: "rpc_error", + payload: { + requestId: msg.requestId, + requestType: msg.type, + error: error.message, + code: error.code, + }, + }); + } + private async getAllTerminalSessions(): Promise { if (!this.terminalManager) { return []; @@ -597,6 +621,10 @@ export class TerminalSessionController { }); } catch (error) { this.sessionLogger.error({ err: error, cwd: msg.cwd }, "Failed to create terminal"); + if (error instanceof TerminalWorkerIncompatibleError) { + this.emitIncompatible(msg, error); + return; + } this.emit({ type: "create_terminal_response", payload: { diff --git a/packages/server/src/terminal/terminal-worker-client.test.ts b/packages/server/src/terminal/terminal-worker-client.test.ts new file mode 100644 index 0000000000..a363ab500d --- /dev/null +++ b/packages/server/src/terminal/terminal-worker-client.test.ts @@ -0,0 +1,74 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { existsSync, mkdtempSync, rmSync, statSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { isPlatform } from "../test-utils/platform.js"; +import { encodeFrame, FrameDecoder, TerminalWorkerFrameError } from "./terminal-worker-frames.js"; +import { connectTerminalWorker, TerminalWorkerConnectError } from "./terminal-worker-client.js"; +import { launchTerminalWorker } from "./terminal-worker-launch.js"; +import { terminalWorkerRecordPath } from "./terminal-worker-record.js"; + +const temporaryDirs: string[] = []; + +afterEach(() => { + for (const dir of temporaryDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }); + } +}); + +async function waitUntil(predicate: () => boolean, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + while (!predicate()) { + if (Date.now() > deadline) throw new Error("condition not met in time"); + await new Promise((resolve) => setTimeout(resolve, 50)); + } +} + +describe("terminal worker frames", () => { + it("decodes frames split across chunks and several frames in one chunk", () => { + const first = encodeFrame({ type: "a", data: new Map([["k", 1]]) }); + const second = encodeFrame({ type: "b" }); + const joined = Buffer.concat([first, second]); + const decoder = new FrameDecoder(); + expect(decoder.push(joined.subarray(0, 3))).toEqual([]); + expect(decoder.push(joined.subarray(3, first.length + 2))).toEqual([ + { type: "a", data: new Map([["k", 1]]) }, + ]); + expect(decoder.push(joined.subarray(first.length + 2))).toEqual([{ type: "b" }]); + }); + + it("rejects a frame header announcing an oversized payload", () => { + const header = Buffer.alloc(4); + header.writeUInt32BE(0xffffffff, 0); + expect(() => new FrameDecoder().push(header)).toThrow(TerminalWorkerFrameError); + }); +}); + +describe("terminal worker authentication", () => { + it("closes a connection with the wrong secret and accepts the record's secret", async () => { + const paseoHome = mkdtempSync(path.join(tmpdir(), "tw-auth-")); + temporaryDirs.push(paseoHome); + const { record, recordPath } = await launchTerminalWorker({ + paseoHome, + paseoVersion: "0.0.0-test", + }); + expect(recordPath).toBe(terminalWorkerRecordPath(paseoHome, record.pid)); + if (!isPlatform("win32")) { + expect(statSync(recordPath).mode & 0o777).toBe(0o600); + expect(statSync(record.socketPath).mode & 0o777).toBe(0o600); + } + + await expect( + connectTerminalWorker({ socketPath: record.socketPath, secret: "wrong-secret" }), + ).rejects.toBeInstanceOf(TerminalWorkerConnectError); + + const connection = await connectTerminalWorker({ + socketPath: record.socketPath, + secret: record.secret, + }); + expect(connection.workerPid).toBe(record.pid); + // With no terminals, a worker whose controller leaves exits and removes its record. + connection.disconnect(); + await waitUntil(() => !existsSync(recordPath), 10_000); + }); +}); diff --git a/packages/server/src/terminal/terminal-worker-client.ts b/packages/server/src/terminal/terminal-worker-client.ts new file mode 100644 index 0000000000..532f3075c6 --- /dev/null +++ b/packages/server/src/terminal/terminal-worker-client.ts @@ -0,0 +1,141 @@ +import { EventEmitter } from "node:events"; +import { createConnection } from "node:net"; +import { attachFramedSocket, type FramedSocket } from "./terminal-worker-frames.js"; +import { + TERMINAL_WORKER_PROTOCOL_VERSION, + type TerminalWorkerHello, + type TerminalWorkerHelloAck, + type TerminalWorkerRequest, + type TerminalWorkerToParentMessage, +} from "./terminal-worker-protocol.js"; + +const DEFAULT_CONNECT_TIMEOUT_MS = 5_000; + +/** The daemon's (or cleanup's) authenticated connection to one terminal worker. */ +export interface TerminalWorkerConnection { + readonly connected: boolean; + readonly workerPid: number; + send(message: TerminalWorkerRequest, callback: (error: Error | null) => void): boolean; + /** Closes the socket. The worker keeps running and keeps its terminals. */ + disconnect(): void; + on(event: "message", listener: (message: TerminalWorkerToParentMessage) => void): this; + on(event: "close", listener: () => void): this; +} + +export class TerminalWorkerConnectError extends Error { + constructor(message: string, options?: { cause?: unknown }) { + super(message, options); + this.name = "TerminalWorkerConnectError"; + } +} + +class SocketTerminalWorkerConnection extends EventEmitter implements TerminalWorkerConnection { + readonly workerPid: number; + private readonly framed: FramedSocket; + private closed = false; + + constructor(framed: FramedSocket, workerPid: number) { + super(); + this.framed = framed; + this.workerPid = workerPid; + framed.socket.once("close", () => { + this.closed = true; + this.emit("close"); + }); + } + + get connected(): boolean { + return !this.closed && !this.framed.socket.destroyed; + } + + send(message: TerminalWorkerRequest, callback: (error: Error | null) => void): boolean { + return this.framed.send(message, callback); + } + + disconnect(): void { + this.framed.close(); + } + + deliver(message: TerminalWorkerToParentMessage): void { + this.emit("message", message); + } +} + +function isHelloAck(message: unknown): message is TerminalWorkerHelloAck { + return ( + typeof message === "object" && + message !== null && + (message as { type?: unknown }).type === "helloAck" && + typeof (message as { pid?: unknown }).pid === "number" + ); +} + +/** + * Connects to a worker's socket and authenticates with the record's secret. The worker closes + * the socket on a wrong secret, so a rejected hello surfaces as a connect error. + */ +export function connectTerminalWorker(input: { + socketPath: string; + secret: string; + timeoutMs?: number; +}): Promise { + const timeoutMs = input.timeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS; + return new Promise((resolve, reject) => { + const socket = createConnection(input.socketPath); + let connection: SocketTerminalWorkerConnection | null = null; + let settled = false; + const fail = (error: Error) => { + if (settled) return; + settled = true; + clearTimeout(timer); + socket.destroy(); + reject(error); + }; + const timer = setTimeout(() => { + fail( + new TerminalWorkerConnectError( + `Terminal worker at ${input.socketPath} did not answer within ${timeoutMs}ms`, + ), + ); + }, timeoutMs); + const framed = attachFramedSocket(socket, (message) => { + if (connection) { + connection.deliver(message as TerminalWorkerToParentMessage); + return; + } + if (!isHelloAck(message)) { + fail(new TerminalWorkerConnectError("Terminal worker sent an unexpected first frame")); + return; + } + settled = true; + clearTimeout(timer); + connection = new SocketTerminalWorkerConnection(framed, message.pid); + resolve(connection); + }); + // `on`, not `once`: an error after authentication must not go unhandled. `close` + // follows every socket error and ends the connection. + socket.on("error", (error) => { + fail( + new TerminalWorkerConnectError( + `Cannot connect to terminal worker at ${input.socketPath}: ${error.message}`, + { cause: error }, + ), + ); + }); + socket.once("close", () => { + fail( + new TerminalWorkerConnectError( + `Terminal worker at ${input.socketPath} closed the connection during authentication`, + ), + ); + }); + socket.once("connect", () => { + const hello: TerminalWorkerHello = { + type: "hello", + secret: input.secret, + protocolVersion: TERMINAL_WORKER_PROTOCOL_VERSION, + }; + framed.send(hello); + }); + }); +} diff --git a/packages/server/src/terminal/terminal-worker-frames.ts b/packages/server/src/terminal/terminal-worker-frames.ts new file mode 100644 index 0000000000..f0b990e239 --- /dev/null +++ b/packages/server/src/terminal/terminal-worker-frames.ts @@ -0,0 +1,98 @@ +import type { Socket } from "node:net"; +import { deserialize, serialize } from "node:v8"; + +// Frames on the terminal worker socket: a 4-byte big-endian length, then a v8-serialized +// payload (the same structured-clone semantics the old fork IPC used with +// `serialization: "advanced"`). The frame format is part of the frozen envelope: every worker +// version must keep it. +const FRAME_HEADER_BYTES = 4; +const MAX_FRAME_BYTES = 256 * 1024 * 1024; + +export class TerminalWorkerFrameError extends Error { + constructor(message: string) { + super(message); + this.name = "TerminalWorkerFrameError"; + } +} + +export function encodeFrame(value: unknown): Buffer { + const payload = serialize(value); + if (payload.length > MAX_FRAME_BYTES) { + throw new TerminalWorkerFrameError( + `Terminal worker frame of ${payload.length} bytes is too large`, + ); + } + const header = Buffer.allocUnsafe(FRAME_HEADER_BYTES); + header.writeUInt32BE(payload.length, 0); + return Buffer.concat([header, payload]); +} + +export class FrameDecoder { + private buffered: Buffer = Buffer.alloc(0); + + push(chunk: Buffer): unknown[] { + this.buffered = this.buffered.length === 0 ? chunk : Buffer.concat([this.buffered, chunk]); + const messages: unknown[] = []; + while (this.buffered.length >= FRAME_HEADER_BYTES) { + const length = this.buffered.readUInt32BE(0); + if (length > MAX_FRAME_BYTES) { + throw new TerminalWorkerFrameError(`Terminal worker frame of ${length} bytes is too large`); + } + const end = FRAME_HEADER_BYTES + length; + if (this.buffered.length < end) break; + messages.push(deserialize(this.buffered.subarray(FRAME_HEADER_BYTES, end))); + this.buffered = this.buffered.subarray(end); + } + return messages; + } +} + +export interface FramedSocket { + readonly socket: Socket; + send(message: unknown, callback?: (error: Error | null) => void): boolean; + close(): void; +} + +/** + * Wraps a connected socket in length-prefixed frames. A malformed frame destroys the socket: + * the peer is either not a terminal worker endpoint or is corrupt, and both end the session. + */ +export function attachFramedSocket( + socket: Socket, + onMessage: (message: unknown) => void, +): FramedSocket { + const decoder = new FrameDecoder(); + socket.on("data", (chunk: Buffer) => { + let messages: unknown[]; + try { + messages = decoder.push(chunk); + } catch (error) { + socket.destroy(error instanceof Error ? error : new Error(String(error))); + return; + } + for (const message of messages) { + if (socket.destroyed) return; + onMessage(message); + } + }); + return { + socket, + send(message, callback) { + if (socket.destroyed || !socket.writable) { + callback?.(new Error("Terminal worker socket is closed")); + return false; + } + let frame: Buffer; + try { + frame = encodeFrame(message); + } catch (error) { + callback?.(error instanceof Error ? error : new Error(String(error))); + return false; + } + return socket.write(frame, (error) => callback?.(error ?? null)); + }, + close() { + socket.end(() => socket.destroy()); + }, + }; +} diff --git a/packages/server/src/terminal/terminal-worker-launch.ts b/packages/server/src/terminal/terminal-worker-launch.ts new file mode 100644 index 0000000000..573abe3bd8 --- /dev/null +++ b/packages/server/src/terminal/terminal-worker-launch.ts @@ -0,0 +1,93 @@ +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { buildSelfNodeCommand } from "../server/paseo-env.js"; +import { ensurePrivateDirectory } from "../server/private-files.js"; +import { + readTerminalWorkerRecord, + terminalWorkerDirectory, + terminalWorkerRecordPath, + type TerminalWorkerRecord, +} from "./terminal-worker-record.js"; +import { resolveTerminalWorkerEntry, type TerminalWorkerEntry } from "./terminal-worker-runtime.js"; + +const LAUNCH_TIMEOUT_MS = 20_000; + +export class TerminalWorkerLaunchError extends Error { + constructor(message: string, options?: { cause?: unknown }) { + super(message, options); + this.name = "TerminalWorkerLaunchError"; + } +} + +/** + * Starts a terminal worker outside this process's tree: the launcher spawns the worker + * detached and exits once it listens, so the worker is re-parented and survives this daemon. + * Resolves with the record the worker wrote; a failed start leaves no worker behind. + */ +export async function launchTerminalWorker(input: { + paseoHome: string; + paseoVersion: string; + timeoutMs?: number; + /** Defaults to the copy of this installation's bundle (source in dev). */ + entry?: TerminalWorkerEntry; +}): Promise<{ record: TerminalWorkerRecord; recordPath: string }> { + const recordId = randomUUID(); + ensurePrivateDirectory(terminalWorkerDirectory(input.paseoHome)); + const entry = input.entry ?? resolveTerminalWorkerEntry(input); + const launch = buildSelfNodeCommand([ + ...entry.execArgv, + entry.scriptPath, + "--launch", + "--record-id", + recordId, + "--paseo-home", + input.paseoHome, + "--paseo-version", + input.paseoVersion, + ...(entry.runtime ? ["--runtime", entry.runtime] : []), + ]); + const stdout = await new Promise((resolve, reject) => { + execFile( + launch.command, + launch.args, + { + env: launch.env, + timeout: input.timeoutMs ?? LAUNCH_TIMEOUT_MS, + windowsHide: true, + encoding: "utf8", + }, + (error, out, err) => { + if (error) { + const detail = err.trim() || error.message; + reject( + new TerminalWorkerLaunchError(`Terminal worker failed to start: ${detail}`, { + cause: error, + }), + ); + return; + } + resolve(out); + }, + ); + }); + + let pid: number; + try { + const parsed = JSON.parse(stdout.trim()) as { pid?: unknown }; + if (typeof parsed.pid !== "number") throw new Error("missing pid"); + pid = parsed.pid; + } catch (error) { + throw new TerminalWorkerLaunchError( + `Terminal worker launcher printed an unexpected result: ${stdout.trim()}`, + { cause: error }, + ); + } + const recordPath = terminalWorkerRecordPath(input.paseoHome, pid); + const record = await readTerminalWorkerRecord(recordPath); + if (record.recordId !== recordId) { + throw new TerminalWorkerLaunchError( + `Terminal worker record ${recordPath} belongs to another launch`, + ); + } + return { record, recordPath }; +} diff --git a/packages/server/src/terminal/terminal-worker-ownership.test.ts b/packages/server/src/terminal/terminal-worker-ownership.test.ts new file mode 100644 index 0000000000..7abbb9aa62 --- /dev/null +++ b/packages/server/src/terminal/terminal-worker-ownership.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from "vitest"; +import type { ManagedProcessInspection } from "../server/managed-processes/managed-processes.js"; +import { + classifyTerminalProgramProcess, + classifyTerminalWorkerProcess, +} from "./terminal-worker-ownership.js"; + +const BIRTH = "Wed Oct 21 18:12:49 2026"; +const OTHER_BIRTH = "Wed Oct 21 18:13:02 2026"; + +function alive(startedAt: string | null, commandLine: string | null): ManagedProcessInspection { + return { status: "alive", snapshot: { pid: 4242, startedAt, commandLine } }; +} + +const workerRecord = { + recordId: "record-1", + identity: { + startedAt: BIRTH, + commandLine: "node terminal-worker-process.js --paseo-terminal-worker record-1", + }, +}; + +describe("terminal program ownership", () => { + it("owns a program that ran exec: same pid and birth time, new command line", () => { + expect( + classifyTerminalProgramProcess({ startedAt: BIRTH }, alive(BIRTH, "sleep 100000")), + ).toEqual({ status: "owned" }); + }); + + it("treats a reused pid with the same command line but another birth time as unrelated", () => { + expect( + classifyTerminalProgramProcess({ startedAt: BIRTH }, alive(OTHER_BIRTH, "/bin/zsh -l")), + ).toMatchObject({ status: "unrelated" }); + }); + + it("never owns a process without a birth time on either side, whatever its command", () => { + expect( + classifyTerminalProgramProcess({ startedAt: BIRTH }, alive(null, "/bin/zsh -l")), + ).toMatchObject({ status: "unknown" }); + expect( + classifyTerminalProgramProcess({ startedAt: null }, alive(BIRTH, "/bin/zsh -l")), + ).toMatchObject({ status: "unknown" }); + }); + + it("reports an inspection error as unknown, not gone", () => { + expect( + classifyTerminalProgramProcess( + { startedAt: BIRTH }, + { status: "error", error: new Error("ps failed") }, + ), + ).toMatchObject({ status: "unknown", reason: expect.stringContaining("ps failed") }); + }); + + it("reports a missing process as gone", () => { + expect(classifyTerminalProgramProcess({ startedAt: BIRTH }, { status: "not-found" })).toEqual({ + status: "gone", + }); + }); +}); + +describe("terminal worker ownership", () => { + it("owns the worker whose birth time matches and whose command line carries its record id", () => { + expect( + classifyTerminalWorkerProcess(workerRecord, alive(BIRTH, workerRecord.identity.commandLine)), + ).toEqual({ status: "owned" }); + }); + + it("treats another worker reusing the pid in the same second as unrelated", () => { + expect( + classifyTerminalWorkerProcess( + workerRecord, + alive(BIRTH, "node terminal-worker-process.js --paseo-terminal-worker record-2"), + ), + ).toMatchObject({ status: "unrelated" }); + }); + + it("treats a worker pid reused with another birth time as unrelated", () => { + expect( + classifyTerminalWorkerProcess( + workerRecord, + alive(OTHER_BIRTH, workerRecord.identity.commandLine), + ), + ).toMatchObject({ status: "unrelated" }); + }); + + it("does not own a worker whose command line cannot be read", () => { + expect(classifyTerminalWorkerProcess(workerRecord, alive(BIRTH, null))).toMatchObject({ + status: "unknown", + }); + }); +}); diff --git a/packages/server/src/terminal/terminal-worker-ownership.ts b/packages/server/src/terminal/terminal-worker-ownership.ts new file mode 100644 index 0000000000..9bd02b01cd --- /dev/null +++ b/packages/server/src/terminal/terminal-worker-ownership.ts @@ -0,0 +1,69 @@ +import type { ManagedProcessInspection } from "../server/managed-processes/managed-processes.js"; +import { TERMINAL_WORKER_MARKER_ARG, type TerminalWorkerRecord } from "./terminal-worker-record.js"; + +// Who a pid named in a terminal worker record belongs to, before anything signals it. +// +// This is deliberately not `processIdentityMatches` from managed-processes.ts: that matcher +// falls back to command text when a birth time is missing, and requires the original argv when +// it is present. A terminal program is often a shell that ran `exec`, which keeps its pid and +// birth time but changes its command line; and a reused pid can run the same command line. +// Here birth time alone identifies a terminal program, and nothing without one is ever owned. + +export type ProcessOwnership = + | { status: "gone" } + | { status: "unrelated"; reason: string } + | { status: "owned" } + | { status: "unknown"; reason: string }; + +export function classifyTerminalWorkerProcess( + record: Pick, + inspection: ManagedProcessInspection, +): ProcessOwnership { + const birth = classifyBirth(record.identity.startedAt, inspection); + if (birth.status !== "owned" || inspection.status !== "alive") return birth; + const commandLine = inspection.snapshot.commandLine; + if (!commandLine) { + return { status: "unknown", reason: "command line unavailable" }; + } + if (!hasWorkerMarker(commandLine, record.recordId)) { + return { status: "unrelated", reason: "command line lacks the worker marker" }; + } + return { status: "owned" }; +} + +export function classifyTerminalProgramProcess( + entry: { startedAt: string | null }, + inspection: ManagedProcessInspection, +): ProcessOwnership { + return classifyBirth(entry.startedAt, inspection); +} + +function classifyBirth( + recordedStartedAt: string | null, + inspection: ManagedProcessInspection, +): ProcessOwnership { + if (inspection.status === "not-found") return { status: "gone" }; + if (inspection.status === "error") { + const message = + inspection.error instanceof Error ? inspection.error.message : String(inspection.error); + return { status: "unknown", reason: `inspection failed: ${message}` }; + } + const startedAt = inspection.snapshot.startedAt; + if (!recordedStartedAt || !startedAt) { + return { status: "unknown", reason: "birth time unavailable" }; + } + if (startedAt !== recordedStartedAt) { + return { status: "unrelated", reason: "pid reused by a process with another birth time" }; + } + return { status: "owned" }; +} + +function hasWorkerMarker(commandLine: string, recordId: string): boolean { + const tokens = commandLine.split(/\s+/); + for (let index = 0; index < tokens.length - 1; index += 1) { + if (tokens[index] === TERMINAL_WORKER_MARKER_ARG && tokens[index + 1] === recordId) { + return true; + } + } + return false; +} diff --git a/packages/server/src/terminal/terminal-worker-pool.ts b/packages/server/src/terminal/terminal-worker-pool.ts new file mode 100644 index 0000000000..f15ad5b46d --- /dev/null +++ b/packages/server/src/terminal/terminal-worker-pool.ts @@ -0,0 +1,288 @@ +import type { TerminalActivityState } from "@getpaseo/protocol/terminal-activity"; +import type { CaptureTerminalLinesResult } from "./terminal-capture.js"; +import type { + TerminalActivityListener, + TerminalsChangedListener, + TerminalWorkspaceContributionChangedListener, +} from "./terminal-manager.js"; +import type { TerminalSession, TerminalStateSnapshot } from "./terminal.js"; +import type { WorkerCreateTerminalOptions } from "./terminal-worker-protocol.js"; +import type { DaemonTerminalManager } from "./worker-terminal-manager.js"; + +// The daemon's terminal service over up to two terminal workers: the current one, which takes +// every new terminal, and one draining a previous worker protocol, which keeps serving the +// terminals it already runs and is let go once its last one exits. With no current worker (two +// were already running), the first terminal created after one exits starts one. A worker from a +// newer Paseo +// is not driven at all: the service then refuses terminal requests with +// TerminalWorkerIncompatibleError instead of serving a partial view. + +export class TerminalWorkerIncompatibleError extends Error { + readonly code = "terminal_worker_incompatible"; + readonly paseoVersion: string; + readonly protocolVersion: unknown; + + constructor(input: { paseoVersion: string; protocolVersion: unknown }) { + super( + `Terminals are held by a newer Paseo (v${input.paseoVersion}, worker protocol ${String(input.protocolVersion)}). Update Paseo to v${input.paseoVersion} or later to use them, or run \`paseo daemon stop\` to close them.`, + ); + this.name = "TerminalWorkerIncompatibleError"; + this.paseoVersion = input.paseoVersion; + this.protocolVersion = input.protocolVersion; + } +} + +export class NoTerminalWorkerAvailableError extends Error { + constructor(message: string) { + super(message); + this.name = "NoTerminalWorkerAvailableError"; + } +} + +export interface TerminalWorkerPoolMember { + manager: DaemonTerminalManager; + workerPid: number; + /** Resolves only after confirmed worker death and its scoped cleanup. */ + exited: Promise; + /** Waits for any reconnect; false once the member is retired or stopping. */ + ready(): Promise; + stopRecovery(): void; +} + +export interface TerminalWorkerPoolOptions { + current: TerminalWorkerPoolMember | null; + draining: TerminalWorkerPoolMember[]; + /** Set when a worker from a newer Paseo holds this home's terminals. */ + incompatible: TerminalWorkerIncompatibleError | null; + /** + * Starts a current worker when there is none. Throws NoTerminalWorkerAvailableError, saying + * why, while no worker may start. + */ + startCurrent?: () => Promise; + /** Closes every terminal of every worker in the home and verifies they ended. */ + closeAll: () => Promise; + /** A draining worker's last terminal exited and the pool let it go. */ + onDrained?: (member: TerminalWorkerPoolMember) => void; +} + +export function createTerminalWorkerPool( + options: TerminalWorkerPoolOptions, +): DaemonTerminalManager { + let current = options.current; + const draining = new Set(options.draining); + const terminalsChangedListeners = new Set(); + const terminalActivityListeners = new Set(); + const contributionListeners = new Set(); + let closePromise: Promise | null = null; + let detached = false; + // One start at a time, so concurrent creates never launch two workers. + let starting: Promise | null = null; + // Registrations belong to the service and are replayed whenever a current worker starts. + const cwdEnv = new Map>(); + + function members(): TerminalWorkerPoolMember[] { + return [...(current ? [current] : []), ...draining]; + } + + function assertAvailable(): void { + if (options.incompatible) throw options.incompatible; + } + + function ownerOf(terminalId: string): DaemonTerminalManager | null { + for (const member of members()) { + if (member.manager.getTerminal(terminalId)) return member.manager; + } + return null; + } + + function routed(terminalId: string): DaemonTerminalManager { + assertAvailable(); + const owner = ownerOf(terminalId) ?? current?.manager; + if (!owner) { + throw new NoTerminalWorkerAvailableError(`Terminal ${terminalId} is not running`); + } + return owner; + } + + function releaseIfDrained(member: TerminalWorkerPoolMember): void { + if (!draining.has(member) || member.manager.listDirectories().length > 0) return; + draining.delete(member); + // With no controller and no terminals the worker exits on its own. The request that closed + // the last terminal is still waiting for its answer. + void member.manager + .detach({ afterPendingRequests: true }) + .then(() => options.onDrained?.(member)); + } + + function forwardEvents(member: TerminalWorkerPoolMember): void { + void member.exited.then(() => { + if (current === member) current = null; + return draining.delete(member); + }); + member.manager.subscribeTerminalsChanged((event) => { + for (const listener of Array.from(terminalsChangedListeners)) listener(event); + releaseIfDrained(member); + }); + member.manager.subscribeTerminalActivity((event) => { + for (const listener of Array.from(terminalActivityListeners)) listener(event); + }); + member.manager.subscribeTerminalWorkspaceContributionChanged((event) => { + for (const listener of Array.from(contributionListeners)) listener(event); + }); + } + + for (const member of members()) forwardEvents(member); + for (const member of Array.from(draining)) releaseIfDrained(member); + + async function currentManager(): Promise { + if (closePromise || detached) + throw new NoTerminalWorkerAvailableError("No terminal worker can take new terminals"); + const candidate = current; + if (candidate && (await candidate.ready())) return candidate.manager; + const startCurrent = options.startCurrent; + if (!startCurrent || closePromise || detached) { + return Promise.reject( + new NoTerminalWorkerAvailableError("No terminal worker can take new terminals"), + ); + } + starting ??= startCurrent() + .then(async (member) => { + if (closePromise || detached) { + // The service closed or let go while the worker started: it takes no terminals. + member.stopRecovery(); + await member.manager.detach(); + throw new NoTerminalWorkerAvailableError("No terminal worker can take new terminals"); + } + forwardEvents(member); + for (const [cwd, env] of cwdEnv) member.manager.registerCwdEnv({ cwd, env }); + current = member; + return member; + }) + .finally(() => { + starting = null; + }); + return starting.then((member) => member.manager); + } + + function closeTerminals(): Promise { + closePromise ??= (async () => { + for (const member of members()) member.stopRecovery(); + await starting?.catch(() => {}); + await Promise.all(members().map((member) => member.manager.detach())); + draining.clear(); + await options.closeAll(); + })(); + return closePromise; + } + + return { + getUnavailableError(): Error | null { + return options.incompatible; + }, + + async getTerminals(cwd, filter) { + assertAvailable(); + const lists = await Promise.all( + members().map((member) => member.manager.getTerminals(cwd, filter)), + ); + return lists.flat(); + }, + + async createTerminal( + createOptions: WorkerCreateTerminalOptions & { workspaceId: string }, + ): Promise { + assertAvailable(); + const manager = await currentManager(); + return manager.createTerminal(createOptions); + }, + + registerCwdEnv(registration) { + cwdEnv.set(registration.cwd, registration.env); + if (current) current.manager.registerCwdEnv(registration); + }, + + validateTerminalActivityToken(terminalId, token) { + for (const member of members()) { + const result = member.manager.validateTerminalActivityToken(terminalId, token); + if (result !== "unknown") return result; + } + return "unknown"; + }, + + getTerminal(id) { + for (const member of members()) { + const session = member.manager.getTerminal(id); + if (session) return session; + } + return undefined; + }, + + async getTerminalState(id, stateOptions): Promise { + return routed(id).getTerminalState(id, stateOptions); + }, + + setTerminalTitle(id, title) { + return ownerOf(id)?.setTerminalTitle(id, title) ?? false; + }, + + async setTerminalActivity(id, state: TerminalActivityState) { + return (await ownerOf(id)?.setTerminalActivity(id, state)) ?? false; + }, + + async clearTerminalAttention(id) { + return (await ownerOf(id)?.clearTerminalAttention(id)) ?? false; + }, + + killTerminal(id) { + ownerOf(id)?.killTerminal(id); + }, + + async killTerminalAndWait(id, waitOptions) { + await routed(id).killTerminalAndWait(id, waitOptions); + }, + + async captureTerminal(id, captureOptions): Promise { + return routed(id).captureTerminal(id, captureOptions); + }, + + listDirectories() { + return [...new Set(members().flatMap((member) => member.manager.listDirectories()))]; + }, + + killAll() { + void closeTerminals().catch(() => { + // The synchronous interface cannot report it; closeTerminals() callers see the error. + }); + }, + + closeTerminals, + + async detach() { + detached = true; + for (const member of members()) member.stopRecovery(); + await starting?.catch(() => {}); + await Promise.all(members().map((member) => member.manager.detach())); + }, + + async adoptTerminals() { + const counts = await Promise.all(members().map((member) => member.manager.adoptTerminals())); + for (const member of Array.from(draining)) releaseIfDrained(member); + return counts.reduce((sum, count) => sum + count, 0); + }, + + subscribeTerminalsChanged(listener) { + terminalsChangedListeners.add(listener); + return () => terminalsChangedListeners.delete(listener); + }, + + subscribeTerminalActivity(listener) { + terminalActivityListeners.add(listener); + return () => terminalActivityListeners.delete(listener); + }, + + subscribeTerminalWorkspaceContributionChanged(listener) { + contributionListeners.add(listener); + return () => contributionListeners.delete(listener); + }, + }; +} diff --git a/packages/server/src/terminal/terminal-worker-process.ts b/packages/server/src/terminal/terminal-worker-process.ts index a70d37c5b9..6c13ba04d5 100644 --- a/packages/server/src/terminal/terminal-worker-process.ts +++ b/packages/server/src/terminal/terminal-worker-process.ts @@ -1,352 +1,777 @@ +import { spawn } from "node:child_process"; +import { randomBytes, timingSafeEqual } from "node:crypto"; +import { chmodSync, closeSync, existsSync, openSync, rmSync, writeSync } from "node:fs"; +import { createServer, type Server, type Socket } from "node:net"; +import { fileURLToPath } from "node:url"; +import { currentBootId } from "../server/pid-lock.js"; +import { PRIVATE_FILE_MODE } from "../server/private-files.js"; import { createTerminalManager } from "./terminal-manager.js"; import { captureTerminalLines } from "./terminal-capture.js"; import { TerminalOutputCoalescer } from "./terminal-output-coalescer.js"; -import type { TerminalSession, TerminalStateSnapshotOptions } from "./terminal.js"; import type { - TerminalWorkerRequest, - TerminalWorkerStateResult, - TerminalWorkerToParentMessage, - WorkerTerminalInfo, + TerminalExitInfo, + TerminalSession, + TerminalStateSnapshotOptions, +} from "./terminal.js"; +import { attachFramedSocket, type FramedSocket } from "./terminal-worker-frames.js"; +import { createTerminalWorkerProcessInspector } from "./terminal-worker-processes.js"; +import { + TERMINAL_WORKER_MARKER_ARG, + terminalWorkerLogPath, + terminalWorkerRecordPath, + writeTerminalWorkerRecord, + type TerminalWorkerRecord, +} from "./terminal-worker-record.js"; +import { resolveTerminalWorkerSocketPath } from "./terminal-worker-socket-path.js"; +import { + TERMINAL_WORKER_PROTOCOL_OVERRIDE_ENV, + TERMINAL_WORKER_PROTOCOL_VERSION, + type TerminalWorkerHello, + type TerminalWorkerHelloAck, + type TerminalWorkerListResult, + type TerminalWorkerRequest, + type TerminalWorkerStateResult, + type TerminalWorkerToParentMessage, + type WorkerTerminalInfo, } from "./terminal-worker-protocol.js"; +// The terminal worker owns every PTY. It runs detached from the daemon (re-parented by the +// launcher below) so a daemon restart, crash or forced kill of the daemon's process tree does +// not take the terminals down, and it talks to whichever daemon controls it over an +// owner-only socket authenticated by the secret in its record. +// +// `--launch` runs the launcher: it starts the real worker detached, waits for it to listen, +// prints `{ pid, socketPath }` and exits, which leaves the worker parented to init. + +const LAUNCH_ARG = "--launch"; +const READY_TIMEOUT_MS = 15_000; +const HELLO_TIMEOUT_MS = 5_000; +// A freshly launched worker waits this long for its first controller before giving up. +const FIRST_CONTROLLER_TIMEOUT_MS = 30_000; +// While nobody controls it, the worker checks that its record still exists. A deleted record +// (a removed $PASEO_HOME) means no daemon can ever find these terminals again. +const ORPHAN_CHECK_INTERVAL_MS = 10_000; +const READY_FD = 3; + +interface WorkerArgs { + recordId: string; + paseoHome: string; + paseoVersion: string; + // The copy under terminal-worker/ this worker runs from; absent when it runs from source. + runtime: string | null; +} + +function readArg(argv: string[], name: string): string { + const index = argv.indexOf(name); + const value = index === -1 ? undefined : argv[index + 1]; + if (!value) { + throw new Error(`Terminal worker requires ${name}`); + } + return value; +} + +function parseWorkerArgs(argv: string[], recordIdArg: string): WorkerArgs { + return { + recordId: readArg(argv, recordIdArg), + paseoHome: readArg(argv, "--paseo-home"), + paseoVersion: readArg(argv, "--paseo-version"), + runtime: argv.includes("--runtime") ? readArg(argv, "--runtime") : null, + }; +} + +// The protocol version this worker reports in its record and hello. Tests override it to stand +// up older and newer workers from one build. +function reportedProtocolVersion(): number { + const override = process.env[TERMINAL_WORKER_PROTOCOL_OVERRIDE_ENV]; + if (override === undefined || override === "") return TERMINAL_WORKER_PROTOCOL_VERSION; + const parsed = Number(override); + if (!Number.isSafeInteger(parsed)) { + throw new Error(`${TERMINAL_WORKER_PROTOCOL_OVERRIDE_ENV} is not an integer: ${override}`); + } + return parsed; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +// --------------------------------------------------------------------------------------------- +// Launcher +// --------------------------------------------------------------------------------------------- + +function runLauncher(argv: string[]): void { + const args = parseWorkerArgs(argv, "--record-id"); + const logFd = openSync(terminalWorkerLogPath(args.paseoHome), "a", PRIVATE_FILE_MODE); + const child = spawn( + process.execPath, + [ + ...process.execArgv, + fileURLToPath(import.meta.url), + TERMINAL_WORKER_MARKER_ARG, + args.recordId, + "--paseo-home", + args.paseoHome, + "--paseo-version", + args.paseoVersion, + ...(args.runtime ? ["--runtime", args.runtime] : []), + ], + { + detached: true, + stdio: ["ignore", "ignore", logFd, "pipe"], + env: process.env, + windowsHide: true, + }, + ); + closeSync(logFd); + + let settled = false; + let buffered = ""; + const finish = (code: number, output: string) => { + if (settled) return; + settled = true; + clearTimeout(timer); + if (code !== 0 && child.exitCode === null && child.pid) { + // A worker that never reported ready must not be left behind. + try { + process.kill(child.pid, "SIGKILL"); + } catch { + // Already gone. + } + } + child.unref(); + (child.stdio[READY_FD] as NodeJS.ReadableStream & { destroy(): void }).destroy(); + // Exit from the write callback: a pipe write can still be pending when process.exit runs. + (code === 0 ? process.stdout : process.stderr).write(output, () => process.exit(code)); + }; + const timer = setTimeout(() => { + finish(1, `Terminal worker did not report ready within ${READY_TIMEOUT_MS}ms\n`); + }, READY_TIMEOUT_MS); + + const readyStream = child.stdio[READY_FD] as NodeJS.ReadableStream; + readyStream.setEncoding("utf8"); + readyStream.on("data", (chunk: string) => { + buffered += chunk; + const newline = buffered.indexOf("\n"); + if (newline === -1) return; + const line = buffered.slice(0, newline); + if (line.startsWith("ready ")) { + finish(0, `${JSON.stringify({ pid: child.pid, socketPath: line.slice("ready ".length) })}\n`); + return; + } + finish(1, `${line.startsWith("error ") ? line.slice("error ".length) : line}\n`); + }); + child.once("error", (error) => finish(1, `Cannot start terminal worker: ${error.message}\n`)); + child.once("exit", (code, signal) => { + finish(1, `Terminal worker exited before it was ready (${signal ?? `code ${code}`})\n`); + }); +} + +// --------------------------------------------------------------------------------------------- +// Worker +// --------------------------------------------------------------------------------------------- + type TerminalCreateRequest = Extract; -const manager = createTerminalManager(); -const unsubscribeByTerminalId = new Map void>>(); -const outputCoalescerByTerminalId = new Map(); -let ipcClosing = false; +function runWorker(argv: string[]): void { + const args = parseWorkerArgs(argv, TERMINAL_WORKER_MARKER_ARG); + const protocolVersion = reportedProtocolVersion(); + const manager = createTerminalManager(); + const inspector = createTerminalWorkerProcessInspector(); + const sessionsById = new Map(); + const unsubscribeByTerminalId = new Map void>>(); + const outputCoalescerByTerminalId = new Map(); + // A terminal is published (terminalCreated) only once its program is recorded. An exit that + // arrives before then is held here and sent after the creation, so the controller never + // receives an exit for a terminal it does not know yet. + const exitsHeldDuringCreation = new Map(); + const secret = randomBytes(32).toString("base64url"); + let record: TerminalWorkerRecord | null = null; + let server: Server | null = null; + let controller: FramedSocket | null = null; + let everControlled = false; + let exiting = false; -interface InFlightTerminalCreateRequest { - requestId: string; - errorReported: boolean; -} - -let inFlightTerminalCreateRequest: InFlightTerminalCreateRequest | null = null; - -// The conpty failure signal is process-scoped, not request-scoped. Serializing -// creates keeps an async spawn failure attributable to exactly one request. -let createTerminalQueue: Promise = Promise.resolve(); - -// node-pty completes its Windows conpty spawn asynchronously on a separate -// conout worker thread. When that spawn fails (bad cwd, missing command, etc.) -// it throws an exception there that cannot be caught at the call site and would -// otherwise crash this worker process and sever every existing terminal. -process.on("uncaughtException", (error) => { - console.error("Terminal worker uncaught exception (kept alive):", error); - reportInFlightTerminalCreateFailure(error); -}); - -function sendToParent(message: TerminalWorkerToParentMessage): void { - if (ipcClosing || !process.connected || !process.send) { - return; + interface InFlightTerminalCreateRequest { + requestId: string; + errorReported: boolean; } - try { - process.send(message, (error) => { - if (error) { - ipcClosing = true; - } - }); - } catch { - ipcClosing = true; - } -} -function buildTerminalStateResult( - session: TerminalSession | undefined, - options?: TerminalStateSnapshotOptions, -): TerminalWorkerStateResult { - if (!session) { - return null; - } - return { ...session.getStateSnapshot(options), replayPreamble: session.getReplayPreamble() }; -} + let inFlightTerminalCreateRequest: InFlightTerminalCreateRequest | null = null; -function toTerminalInfo(session: TerminalSession): WorkerTerminalInfo { - return { - id: session.id, - name: session.name, - cwd: session.cwd, - workspaceId: session.workspaceId, - ...(session.getTitle() ? { title: session.getTitle() } : {}), - activity: session.getActivity(), - }; -} + // The conpty failure signal is process-scoped, not request-scoped. Serializing + // creates keeps an async spawn failure attributable to exactly one request. + let createTerminalQueue: Promise = Promise.resolve(); -function terminalWorkerErrorMessage(error: unknown): string { - return error instanceof Error ? error.message : "Terminal worker request failed"; -} - -function reportInFlightTerminalCreateFailure(error: unknown): void { - if (!inFlightTerminalCreateRequest || inFlightTerminalCreateRequest.errorReported) { - return; - } - inFlightTerminalCreateRequest.errorReported = true; - sendToParent({ - type: "response", - requestId: inFlightTerminalCreateRequest.requestId, - ok: false, - error: terminalWorkerErrorMessage(error), - }); -} - -function clearTerminalSubscriptions(terminalId: string): void { - const subscriptions = unsubscribeByTerminalId.get(terminalId); - if (subscriptions) { - for (const unsubscribe of subscriptions) { - try { - unsubscribe(); - } catch { - // no-op - } - } - } - unsubscribeByTerminalId.delete(terminalId); - const coalescer = outputCoalescerByTerminalId.get(terminalId); - if (coalescer) { - coalescer.dispose(); - outputCoalescerByTerminalId.delete(terminalId); - } -} - -function watchTerminal(session: TerminalSession): void { - clearTerminalSubscriptions(session.id); - - // Coalesce pty output chunks into a single IPC message per ~5ms window so a - // burst of small chunks no longer costs one process.send each. The batch - // carries the LAST chunk's revision (the highest) so downstream snapshot - // replay dedup stays correct. - let pendingOutputRevision: number | undefined; - const outputCoalescer = new TerminalOutputCoalescer({ - timers: { setTimeout, clearTimeout }, - onFlush: ({ payload }) => { - const revision = pendingOutputRevision; - pendingOutputRevision = undefined; - sendToParent({ - type: "terminalMessage", - terminalId: session.id, - message: { type: "output", data: payload.toString("utf8"), revision }, - replayPreamble: session.getReplayPreamble(), - }); - }, - }); - outputCoalescerByTerminalId.set(session.id, outputCoalescer); - - const unsubscribeMessage = session.subscribe( - (message) => { - if (message.type === "output") { - pendingOutputRevision = message.revision; - outputCoalescer.handle(message.data); - return; - } - // Non-output messages (snapshot/snapshotReady/titleChange) must not jump - // ahead of buffered output: flush the coalescer first, then forward. - outputCoalescer.flush(); - sendToParent({ - type: "terminalMessage", - terminalId: session.id, - message, - replayPreamble: session.getReplayPreamble(), - }); - }, - // Creation already sends an authoritative state in terminalCreated. A second - // asynchronous state snapshot can arrive after the create response and - // overwrite a resize issued immediately by the caller. - { initialSnapshot: "ready" }, - ); - const unsubscribeExit = session.onExit((info) => { - outputCoalescer.flush(); - clearTerminalSubscriptions(session.id); - sendToParent({ - type: "terminalExit", - terminalId: session.id, - info, - }); - }); - const unsubscribeTitle = session.onTitleChange((title) => { - outputCoalescer.flush(); - sendToParent({ - type: "terminalTitleChange", - terminalId: session.id, - title, - }); - }); - const unsubscribeCommandFinished = session.onCommandFinished((info) => { - outputCoalescer.flush(); - sendToParent({ - type: "terminalCommandFinished", - terminalId: session.id, - info, - }); - }); - const unsubscribeActivity = session.onActivityChange((transition) => { - sendToParent({ - type: "terminalActivityChange", - terminalId: session.id, - activity: transition.activity, - previous: transition.previous, - }); - }); - - unsubscribeByTerminalId.set(session.id, [ - unsubscribeMessage, - unsubscribeExit, - unsubscribeTitle, - unsubscribeCommandFinished, - unsubscribeActivity, - ]); -} - -function enqueueCreateTerminalRequest(message: TerminalCreateRequest): Promise { - const nextRequest = createTerminalQueue.then(() => handleCreateTerminalRequest(message)); - createTerminalQueue = nextRequest.catch(() => {}); - return nextRequest; -} - -async function handleCreateTerminalRequest(message: TerminalCreateRequest): Promise { - const request: InFlightTerminalCreateRequest = { - requestId: message.requestId, - errorReported: false, - }; - inFlightTerminalCreateRequest = request; - try { - const { workspaceId } = message.options; - if (!workspaceId) { - throw new Error("workspaceId is required"); - } - const session = await manager.createTerminal({ ...message.options, workspaceId }); - if (request.errorReported) { - session.kill(); - return; - } - watchTerminal(session); - const initialSnapshot = session.getStateSnapshot(); - sendToParent({ - type: "terminalCreated", - terminal: toTerminalInfo(session), - state: initialSnapshot.state, - }); - sendToParent({ - type: "response", - requestId: message.requestId, - ok: true, - result: { - terminal: toTerminalInfo(session), - state: initialSnapshot.state, - }, - }); - } catch (error) { + // node-pty completes its Windows conpty spawn asynchronously on a separate + // conout worker thread. When that spawn fails (bad cwd, missing command, etc.) + // it throws an exception there that cannot be caught at the call site and would + // otherwise crash this worker process and sever every existing terminal. + process.on("uncaughtException", (error) => { + console.error("Terminal worker uncaught exception (kept alive):", error); reportInFlightTerminalCreateFailure(error); - } finally { - if (inFlightTerminalCreateRequest === request) { - inFlightTerminalCreateRequest = null; + }); + + function sendToController(message: TerminalWorkerToParentMessage): void { + // With no controller the event is dropped; the headless terminal keeps the screen, and the + // next daemon reads it through listTerminals. + controller?.send(message); + } + + function persistRecord(): void { + if (!record || exiting) return; + record = { + ...record, + terminals: record.terminals.filter((entry) => sessionsById.has(entry.id)), + }; + writeTerminalWorkerRecord(args.paseoHome, record); + } + + function removeOwnFiles(): void { + rmSync(terminalWorkerRecordPath(args.paseoHome, process.pid), { force: true }); + if (record && process.platform !== "win32") { + rmSync(record.socketPath, { force: true }); } } -} -async function handleRequest(message: TerminalWorkerRequest): Promise { - switch (message.type) { - case "createTerminal": { - await enqueueCreateTerminalRequest(message); - return; - } + function exitWorker(code: number): void { + if (exiting) return; + exiting = true; + removeOwnFiles(); + server?.close(); + process.exit(code); + } - case "registerCwdEnv": { - manager.registerCwdEnv({ cwd: message.cwd, env: message.env }); - sendToParent({ type: "response", requestId: message.requestId, ok: true }); - return; - } + async function killEveryTerminalAndExit(): Promise { + const sessions = Array.from(sessionsById.values()); + await Promise.all( + sessions.map((session) => + session.killAndWait().catch((error: unknown) => { + console.error(`Terminal worker could not kill terminal ${session.id}:`, error); + }), + ), + ); + exitWorker(0); + } - case "setActivity": { - await manager.setTerminalActivity(message.terminalId, message.state); - sendToParent({ type: "response", requestId: message.requestId, ok: true }); - return; - } - - case "clearAttention": { - await manager.clearTerminalAttention(message.terminalId); - sendToParent({ type: "response", requestId: message.requestId, ok: true }); - return; - } - - case "killTerminal": { - manager.killTerminal(message.terminalId); - // Removal is owned by session.onExit -> terminalExit; the parent mirror - // clears contribution and emits terminalsChanged from that single path. - clearTerminalSubscriptions(message.terminalId); - sendToParent({ type: "response", requestId: message.requestId, ok: true }); - return; - } - - case "killTerminalAndWait": { - await manager.killTerminalAndWait(message.terminalId, message.options); - clearTerminalSubscriptions(message.terminalId); - sendToParent({ type: "response", requestId: message.requestId, ok: true }); - return; - } - - case "getTerminalState": { - // Flush buffered output before snapshotting: the headless state already includes it, - // so if the coalescer emitted it afterward (in a batch carrying a revision past the - // snapshot's) the controller's revision dedup wouldn't drop it and the client would - // see the bytes twice. Flushing first sends them with a revision <= the snapshot's. - outputCoalescerByTerminalId.get(message.terminalId)?.flush(); - sendToParent({ - type: "response", - requestId: message.requestId, - ok: true, - result: buildTerminalStateResult(manager.getTerminal(message.terminalId), message.options), - }); - return; - } - - case "captureTerminal": { - const session = manager.getTerminal(message.terminalId); - const result = session - ? captureTerminalLines(session, { - start: message.start, - end: message.end, - stripAnsi: message.stripAnsi, - }) - : { lines: [], totalLines: 0 }; - sendToParent({ - type: "response", - requestId: message.requestId, - ok: true, - result, - }); - return; - } - - case "killAll": { - manager.killAll(); - for (const terminalId of Array.from(unsubscribeByTerminalId.keys())) { - clearTerminalSubscriptions(terminalId); - } - sendToParent({ type: "response", requestId: message.requestId, ok: true }); - return; - } - - case "send": { - const session = manager.getTerminal(message.terminalId); - session?.send(message.message); - sendToParent({ type: "response", requestId: message.requestId, ok: true }); - return; + function exitIfIdle(): void { + if (!controller && everControlled && sessionsById.size === 0) { + exitWorker(0); } } -} -process.on("message", (message: TerminalWorkerRequest) => { - void handleRequest(message).catch((error: unknown) => { - sendToParent({ + function buildTerminalStateResult( + session: TerminalSession | undefined, + options?: TerminalStateSnapshotOptions, + ): TerminalWorkerStateResult { + if (!session) { + return null; + } + return { ...session.getStateSnapshot(options), replayPreamble: session.getReplayPreamble() }; + } + + function toTerminalInfo(session: TerminalSession): WorkerTerminalInfo { + return { + id: session.id, + name: session.name, + cwd: session.cwd, + workspaceId: session.workspaceId, + ...(session.getTitle() ? { title: session.getTitle() } : {}), + activity: session.getActivity(), + }; + } + + function terminalWorkerErrorMessage(error: unknown): string { + return error instanceof Error ? error.message : "Terminal worker request failed"; + } + + function reportInFlightTerminalCreateFailure(error: unknown): void { + if (!inFlightTerminalCreateRequest || inFlightTerminalCreateRequest.errorReported) { + return; + } + inFlightTerminalCreateRequest.errorReported = true; + sendToController({ type: "response", - requestId: message.requestId, + requestId: inFlightTerminalCreateRequest.requestId, ok: false, error: terminalWorkerErrorMessage(error), }); - }); -}); + } -process.once("disconnect", () => { - ipcClosing = true; - manager.killAll(); -}); + function clearTerminalSubscriptions(terminalId: string): void { + const subscriptions = unsubscribeByTerminalId.get(terminalId); + if (subscriptions) { + for (const unsubscribe of subscriptions) { + try { + unsubscribe(); + } catch { + // no-op + } + } + } + unsubscribeByTerminalId.delete(terminalId); + const coalescer = outputCoalescerByTerminalId.get(terminalId); + if (coalescer) { + coalescer.dispose(); + outputCoalescerByTerminalId.delete(terminalId); + } + } + + function watchTerminal(session: TerminalSession): void { + clearTerminalSubscriptions(session.id); + + // Coalesce pty output chunks into a single frame per ~5ms window so a burst of small + // chunks no longer costs one socket write each. The batch carries the LAST chunk's + // revision (the highest) so downstream snapshot replay dedup stays correct. + let pendingOutputRevision: number | undefined; + const outputCoalescer = new TerminalOutputCoalescer({ + timers: { setTimeout, clearTimeout }, + onFlush: ({ payload }) => { + const revision = pendingOutputRevision; + pendingOutputRevision = undefined; + sendToController({ + type: "terminalMessage", + terminalId: session.id, + message: { type: "output", data: payload.toString("utf8"), revision }, + replayPreamble: session.getReplayPreamble(), + }); + }, + }); + outputCoalescerByTerminalId.set(session.id, outputCoalescer); + + const unsubscribeMessage = session.subscribe( + (message) => { + if (message.type === "output") { + pendingOutputRevision = message.revision; + outputCoalescer.handle(message.data); + return; + } + // Non-output messages (snapshot/snapshotReady/titleChange) must not jump + // ahead of buffered output: flush the coalescer first, then forward. + outputCoalescer.flush(); + sendToController({ + type: "terminalMessage", + terminalId: session.id, + message, + replayPreamble: session.getReplayPreamble(), + }); + }, + // Creation already sends an authoritative state in terminalCreated. A second + // asynchronous state snapshot can arrive after the create response and + // overwrite a resize issued immediately by the caller. + { initialSnapshot: "ready" }, + ); + const unsubscribeExit = session.onExit((info) => { + outputCoalescer.flush(); + clearTerminalSubscriptions(session.id); + sessionsById.delete(session.id); + persistRecord(); + if (exitsHeldDuringCreation.has(session.id)) { + exitsHeldDuringCreation.set(session.id, info); + return; + } + sendToController({ + type: "terminalExit", + terminalId: session.id, + info, + }); + exitIfIdle(); + }); + const unsubscribeTitle = session.onTitleChange((title) => { + outputCoalescer.flush(); + sendToController({ + type: "terminalTitleChange", + terminalId: session.id, + title, + }); + }); + const unsubscribeCommandFinished = session.onCommandFinished((info) => { + outputCoalescer.flush(); + sendToController({ + type: "terminalCommandFinished", + terminalId: session.id, + info, + }); + }); + const unsubscribeActivity = session.onActivityChange((transition) => { + sendToController({ + type: "terminalActivityChange", + terminalId: session.id, + activity: transition.activity, + previous: transition.previous, + }); + }); + + unsubscribeByTerminalId.set(session.id, [ + unsubscribeMessage, + unsubscribeExit, + unsubscribeTitle, + unsubscribeCommandFinished, + unsubscribeActivity, + ]); + } + + // A terminal enters the record only with its program's birth time, so cleanup can always + // tell the program from a later process that reuses its pid. A program that already exited + // needs no entry; one whose birth time cannot be read is killed and the create fails. + async function recordTerminalProgram(session: TerminalSession): Promise { + if (session.getExitInfo()) return; + // The manager drops a session as soon as its program exits, so only a live one has a pid. + const pid = manager.getTerminalProcessId(session.id); + if (pid === null) { + throw new Error(`Terminal ${session.id} has no program pid`); + } + const inspection = await inspector.inspect(pid); + if (inspection.status === "not-found" || session.getExitInfo()) { + return; + } + if (inspection.status === "error" || !inspection.snapshot.startedAt) { + const reason = + inspection.status === "error" ? errorMessage(inspection.error) : "no birth time reported"; + throw new Error(`Cannot identify terminal program ${pid}: ${reason}`); + } + if (!record) { + throw new Error("Terminal worker record is not written yet"); + } + record = { + ...record, + terminals: [ + ...record.terminals.filter((entry) => entry.id !== session.id), + { id: session.id, pid, startedAt: inspection.snapshot.startedAt }, + ], + }; + persistRecord(); + } + + function enqueueCreateTerminalRequest(message: TerminalCreateRequest): Promise { + const nextRequest = createTerminalQueue.then(() => handleCreateTerminalRequest(message)); + createTerminalQueue = nextRequest.catch(() => {}); + return nextRequest; + } + + async function handleCreateTerminalRequest(message: TerminalCreateRequest): Promise { + const request: InFlightTerminalCreateRequest = { + requestId: message.requestId, + errorReported: false, + }; + inFlightTerminalCreateRequest = request; + try { + const { workspaceId } = message.options; + if (!workspaceId) { + throw new Error("workspaceId is required"); + } + const session = await manager.createTerminal({ ...message.options, workspaceId }); + if (request.errorReported) { + session.kill(); + return; + } + sessionsById.set(session.id, session); + exitsHeldDuringCreation.set(session.id, null); + watchTerminal(session); + try { + await recordTerminalProgram(session); + } catch (error) { + exitsHeldDuringCreation.delete(session.id); + clearTerminalSubscriptions(session.id); + sessionsById.delete(session.id); + await session.killAndWait().catch(() => undefined); + throw error; + } + const initialSnapshot = session.getStateSnapshot(); + sendToController({ + type: "terminalCreated", + terminal: toTerminalInfo(session), + state: initialSnapshot.state, + }); + sendToController({ + type: "response", + requestId: message.requestId, + ok: true, + result: { + terminal: toTerminalInfo(session), + state: initialSnapshot.state, + }, + }); + const heldExit = exitsHeldDuringCreation.get(session.id); + exitsHeldDuringCreation.delete(session.id); + if (heldExit) { + sendToController({ type: "terminalExit", terminalId: session.id, info: heldExit }); + exitIfIdle(); + } + } catch (error) { + reportInFlightTerminalCreateFailure(error); + } finally { + if (inFlightTerminalCreateRequest === request) { + inFlightTerminalCreateRequest = null; + } + } + } + + function listTerminals(): TerminalWorkerListResult { + return Array.from(sessionsById.values()).map((session) => { + // Buffered output is already in the headless state; flushing first keeps a later + // batch from carrying bytes the snapshot already shows. + outputCoalescerByTerminalId.get(session.id)?.flush(); + return { + terminal: toTerminalInfo(session), + state: session.getStateSnapshot().state, + activityToken: manager.getTerminalActivityToken(session.id), + replayPreamble: session.getReplayPreamble(), + }; + }); + } + + async function handleRequest(message: TerminalWorkerRequest): Promise { + switch (message.type) { + case "createTerminal": { + await enqueueCreateTerminalRequest(message); + return; + } + + case "registerCwdEnv": { + manager.registerCwdEnv({ cwd: message.cwd, env: message.env }); + sendToController({ type: "response", requestId: message.requestId, ok: true }); + return; + } + + case "setActivity": { + await manager.setTerminalActivity(message.terminalId, message.state); + sendToController({ type: "response", requestId: message.requestId, ok: true }); + return; + } + + case "clearAttention": { + await manager.clearTerminalAttention(message.terminalId); + sendToController({ type: "response", requestId: message.requestId, ok: true }); + return; + } + + case "killTerminal": { + manager.killTerminal(message.terminalId); + // Removal is owned by session.onExit -> terminalExit; the parent mirror + // clears contribution and emits terminalsChanged from that single path. + sendToController({ type: "response", requestId: message.requestId, ok: true }); + return; + } + + case "killTerminalAndWait": { + await manager.killTerminalAndWait(message.terminalId, message.options); + sendToController({ type: "response", requestId: message.requestId, ok: true }); + return; + } + + case "getTerminalState": { + // Flush buffered output before snapshotting: the headless state already includes it, + // so if the coalescer emitted it afterward (in a batch carrying a revision past the + // snapshot's) the controller's revision dedup wouldn't drop it and the client would + // see the bytes twice. Flushing first sends them with a revision <= the snapshot's. + outputCoalescerByTerminalId.get(message.terminalId)?.flush(); + sendToController({ + type: "response", + requestId: message.requestId, + ok: true, + result: buildTerminalStateResult( + manager.getTerminal(message.terminalId), + message.options, + ), + }); + return; + } + + case "captureTerminal": { + const session = manager.getTerminal(message.terminalId); + const result = session + ? captureTerminalLines(session, { + start: message.start, + end: message.end, + stripAnsi: message.stripAnsi, + }) + : { lines: [], totalLines: 0 }; + sendToController({ + type: "response", + requestId: message.requestId, + ok: true, + result, + }); + return; + } + + case "listTerminals": { + sendToController({ + type: "response", + requestId: message.requestId, + ok: true, + result: listTerminals(), + }); + return; + } + + case "shutdown": { + const sessions = Array.from(sessionsById.values()); + await Promise.all(sessions.map((session) => session.killAndWait().catch(() => undefined))); + // Exit once the response frame is flushed, so the requester sees the confirmation. + const requester = controller; + if (!requester) { + exitWorker(0); + return; + } + requester.send({ type: "response", requestId: message.requestId, ok: true }, () => + exitWorker(0), + ); + return; + } + + case "send": { + const session = manager.getTerminal(message.terminalId); + session?.send(message.message); + sendToController({ type: "response", requestId: message.requestId, ok: true }); + return; + } + } + } + + function isHello(message: unknown): message is TerminalWorkerHello { + return ( + typeof message === "object" && + message !== null && + (message as { type?: unknown }).type === "hello" && + typeof (message as { secret?: unknown }).secret === "string" + ); + } + + function secretMatches(candidate: string): boolean { + const expected = Buffer.from(secret, "utf8"); + const actual = Buffer.from(candidate, "utf8"); + return expected.length === actual.length && timingSafeEqual(expected, actual); + } + + function handleConnection(socket: Socket): void { + let authenticated = false; + socket.on("error", () => { + // `close` follows; a broken controller socket only means no controller. + }); + const helloTimer = setTimeout(() => socket.destroy(), HELLO_TIMEOUT_MS); + const framed = attachFramedSocket(socket, (message) => { + if (!authenticated) { + clearTimeout(helloTimer); + if (!isHello(message) || !secretMatches(message.secret)) { + socket.destroy(); + return; + } + authenticated = true; + // One controller at a time: a new authenticated connection replaces the old one. + const previous = controller; + controller = framed; + everControlled = true; + previous?.socket.destroy(); + const ack: TerminalWorkerHelloAck = { + type: "helloAck", + pid: process.pid, + protocolVersion, + }; + framed.send(ack); + return; + } + if (framed !== controller) { + return; + } + const request = message as TerminalWorkerRequest; + void handleRequest(request).catch((error: unknown) => { + sendToController({ + type: "response", + requestId: request.requestId, + ok: false, + error: terminalWorkerErrorMessage(error), + }); + }); + }); + socket.once("close", () => { + clearTimeout(helloTimer); + if (controller === framed) { + controller = null; + exitIfIdle(); + } + }); + } + + function reportReady(line: string): void { + try { + writeSync(READY_FD, `${line}\n`); + closeSync(READY_FD); + } catch { + // Started without a launcher pipe (tests that spawn the worker directly). + } + } + + async function start(): Promise { + const socketPath = resolveTerminalWorkerSocketPath({ + paseoHome: args.paseoHome, + workerPid: process.pid, + }); + if (process.platform !== "win32") { + rmSync(socketPath, { force: true }); + } + const listening = createServer(handleConnection); + await new Promise((resolve, reject) => { + listening.once("error", reject); + listening.listen(socketPath, () => { + listening.off("error", reject); + resolve(); + }); + }); + server = listening; + if (process.platform !== "win32") { + chmodSync(socketPath, PRIVATE_FILE_MODE); + } + + const self = await inspector.inspect(process.pid); + if ( + self.status !== "alive" || + !self.snapshot.startedAt || + !self.snapshot.commandLine?.includes(args.recordId) + ) { + const reason = self.status === "error" ? errorMessage(self.error) : "no usable snapshot"; + throw new Error(`Terminal worker cannot read its own process identity: ${reason}`); + } + record = { + recordId: args.recordId, + pid: process.pid, + bootId: currentBootId(), + createdAt: new Date().toISOString(), + identity: { startedAt: self.snapshot.startedAt, commandLine: self.snapshot.commandLine }, + protocolVersion, + paseoVersion: args.paseoVersion, + socketPath, + secret, + runtime: args.runtime, + terminals: [], + }; + writeTerminalWorkerRecord(args.paseoHome, record); + reportReady(`ready ${socketPath}`); + + setTimeout(() => { + if (!everControlled && sessionsById.size === 0) exitWorker(0); + }, FIRST_CONTROLLER_TIMEOUT_MS).unref(); + setInterval(() => { + if (!controller && !existsSync(terminalWorkerRecordPath(args.paseoHome, process.pid))) { + void killEveryTerminalAndExit(); + } + }, ORPHAN_CHECK_INTERVAL_MS); + } + + process.on("SIGTERM", () => { + void killEveryTerminalAndExit(); + }); + + start().catch((error: unknown) => { + const message = errorMessage(error).replace(/\n/g, " | "); + console.error(`Terminal worker failed to start: ${errorMessage(error)}`); + reportReady(`error ${message}`); + exitWorker(1); + }); +} + +if (process.argv.includes(LAUNCH_ARG)) { + runLauncher(process.argv.slice(2)); +} else { + runWorker(process.argv.slice(2)); +} diff --git a/packages/server/src/terminal/terminal-worker-processes.ts b/packages/server/src/terminal/terminal-worker-processes.ts new file mode 100644 index 0000000000..b0c2e6d0cb --- /dev/null +++ b/packages/server/src/terminal/terminal-worker-processes.ts @@ -0,0 +1,185 @@ +import { readFile } from "node:fs/promises"; +import { z } from "zod"; +import { execCommand } from "../utils/spawn.js"; +import { + createSystemManagedProcessTable, + type ManagedProcessCommandRunner, + type ManagedProcessInspection, + windowsPowerShellPath, +} from "../server/managed-processes/managed-processes.js"; + +// Process inspection for terminal worker ownership. Birth times are compared as the strings +// `ps -o lstart` and Win32_Process.CreationDate print, so every reader must print them the same +// way: `ps` follows LC_TIME, and the worker, the daemon, the supervisor and the CLI can run +// under different locales. Every `ps` here runs under LC_ALL=C. + +const PS_TIMEOUT_MS = 15_000; +const POSIX_LSTART_WIDTH = 24; + +export interface ProcessTableEntry { + pid: number; + ppid: number; + startedAt: string; + // POSIX session id where `ps` reports one (Linux). macOS `ps` has no session column. + sessionId: number | null; +} + +export interface TerminalWorkerProcessInspector { + /** True where the process table names each process's session (Linux). */ + readonly reportsSessions: boolean; + inspect(pid: number): Promise; + /** One snapshot of every process, for walking descendants. Throws when it cannot be read. */ + listProcesses(): Promise; + /** + * The environment a process was started with, as NAME=value entries; null when the process + * is gone. Throws when it cannot be read. Only called where `reportsSessions` is true. + */ + readEnvironment(pid: number): Promise; +} + +const WindowsProcessListSchema = z.array( + z.object({ + ProcessId: z.number().int(), + ParentProcessId: z.number().int().nullable().optional(), + CreationDate: z.string().nullable().optional(), + }), +); + +function createLocaleNeutralCommandRunner(): ManagedProcessCommandRunner { + return { + exec: (command, args) => + execCommand(command, args, { + envMode: "internal", + envOverlay: { LC_ALL: "C" }, + timeout: PS_TIMEOUT_MS, + maxBuffer: 64 * 1024 * 1024, + }).then(({ stdout, stderr }) => ({ stdout: String(stdout), stderr: String(stderr) })), + }; +} + +export function createTerminalWorkerProcessInspector( + options: { platform?: NodeJS.Platform; commandRunner?: ManagedProcessCommandRunner } = {}, +): TerminalWorkerProcessInspector { + const platform = options.platform ?? process.platform; + const commandRunner = options.commandRunner ?? createLocaleNeutralCommandRunner(); + const table = createSystemManagedProcessTable({ platform, commandRunner }); + return { + reportsSessions: platform === "linux", + inspect: (pid) => table.inspect(pid), + listProcesses: () => + platform === "win32" + ? listWindowsProcesses(commandRunner) + : listPosixProcesses(commandRunner, platform), + readEnvironment: (pid) => readLinuxEnvironment(platform, pid), + }; +} + +async function readLinuxEnvironment( + platform: NodeJS.Platform, + pid: number, +): Promise { + if (platform !== "linux") { + throw new Error(`reading a process environment is not supported on ${platform}`); + } + let raw: string; + try { + raw = await readFile(`/proc/${pid}/environ`, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + return raw.split("\0").filter((entry) => entry.length > 0); +} + +async function listPosixProcesses( + commandRunner: ManagedProcessCommandRunner, + platform: NodeJS.Platform, +): Promise { + const withSession = platform === "linux"; + const columns = withSession + ? ["-o", "pid=", "-o", "ppid=", "-o", "sid=", "-o", "lstart="] + : ["-o", "pid=", "-o", "ppid=", "-o", "lstart="]; + const { stdout } = await commandRunner.exec("ps", ["-A", "-ww", ...columns]); + const entries: ProcessTableEntry[] = []; + const leadingInts = withSession ? 3 : 2; + for (const line of stdout.split("\n")) { + const match = line.match( + withSession ? /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(.+?)\s*$/ : /^\s*(\d+)\s+(\d+)\s+(.+?)\s*$/, + ); + if (!match) continue; + const startedAt = match[leadingInts + 1].slice(0, POSIX_LSTART_WIDTH).trim(); + entries.push({ + pid: Number(match[1]), + ppid: Number(match[2]), + sessionId: withSession ? Number(match[3]) : null, + startedAt, + }); + } + if (entries.length === 0) { + throw new Error("ps returned no processes"); + } + return entries; +} + +async function listWindowsProcesses( + commandRunner: ManagedProcessCommandRunner, +): Promise { + const command = + "@(Get-CimInstance Win32_Process | Select-Object ProcessId,ParentProcessId,CreationDate) | ConvertTo-Json -Compress"; + const { stdout } = await commandRunner.exec(windowsPowerShellPath(), [ + "-NoProfile", + "-NonInteractive", + "-Command", + command, + ]); + const parsed: unknown = JSON.parse(stdout.trim()); + const rows = WindowsProcessListSchema.parse(Array.isArray(parsed) ? parsed : [parsed]); + return rows.flatMap((row) => + row.CreationDate + ? [ + { + pid: row.ProcessId, + ppid: row.ParentProcessId ?? 0, + startedAt: row.CreationDate, + sessionId: null, + }, + ] + : [], + ); +} + +/** + * Every process below `rootPid` in the snapshot, plus (where the platform reports sessions) + * every member of the session `rootPid` led and everything below those members. A PTY program + * is a session leader, and the kernel keeps its pid reserved as the session id while any member + * lives, so the session still names a HUP-ignoring job after the shell exits and the job is + * reparented. + */ +export function collectDescendants( + snapshot: ProcessTableEntry[], + rootPid: number, +): ProcessTableEntry[] { + const childrenByParent = new Map(); + for (const entry of snapshot) { + const siblings = childrenByParent.get(entry.ppid) ?? []; + siblings.push(entry); + childrenByParent.set(entry.ppid, siblings); + } + const found = new Map(); + const queue = [rootPid]; + for (const entry of snapshot) { + if (entry.sessionId === rootPid && entry.pid !== rootPid) { + found.set(entry.pid, entry); + queue.push(entry.pid); + } + } + while (queue.length > 0) { + const parent = queue.shift()!; + for (const child of childrenByParent.get(parent) ?? []) { + if (child.pid === rootPid || found.has(child.pid)) continue; + found.set(child.pid, child); + queue.push(child.pid); + } + } + return Array.from(found.values()); +} diff --git a/packages/server/src/terminal/terminal-worker-protocol.test.ts b/packages/server/src/terminal/terminal-worker-protocol.test.ts new file mode 100644 index 0000000000..c3810bf9cf --- /dev/null +++ b/packages/server/src/terminal/terminal-worker-protocol.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from "vitest"; +import { classifyTerminalWorkerProtocol } from "./terminal-worker-protocol.js"; + +describe("classifyTerminalWorkerProtocol", () => { + it.each([ + [5, "current"], + [4, "previous"], + [3, "obsolete"], + [-1, "obsolete"], + [6, "newer"], + // A version this daemon cannot read comes from a Paseo it does not know. + ["6", "newer"], + [4.5, "newer"], + [Number.NaN, "newer"], + [undefined, "newer"], + ])("a worker reporting %s to a daemon at 5 is %s", (reported, fit) => { + expect(classifyTerminalWorkerProtocol(reported, 5)).toBe(fit); + }); +}); diff --git a/packages/server/src/terminal/terminal-worker-protocol.ts b/packages/server/src/terminal/terminal-worker-protocol.ts index 5e7d1a30c0..86fc4a7772 100644 --- a/packages/server/src/terminal/terminal-worker-protocol.ts +++ b/packages/server/src/terminal/terminal-worker-protocol.ts @@ -31,6 +31,55 @@ export interface WorkerCreateTerminalOptions { cols?: number; activityToken?: string; activityUrl?: string | null; + // The worker outlives the daemon that started it, so the controlling daemon + // sends its own environment and Paseo CLI path with every create instead of + // letting the worker reuse whatever it inherited at launch. + baseEnv?: Record; + paseoCliPath?: string | null; +} + +// Bumped when a request, response or event changes shape. The frozen envelope never changes +// across versions: the frame format, `hello`/`helloAck`, the `listTerminals` info fields, and +// `shutdown`. A daemon drives a worker of its own version, drains one a version older (no new +// terminals; it exits after its last terminal), shuts down anything older, and leaves a newer +// worker running untouched. See docs/terminal-worker.md. +export const TERMINAL_WORKER_PROTOCOL_VERSION = 1; + +// Test-only: the protocol version a worker reports, so tests can stand up older and newer +// workers from one build. +export const TERMINAL_WORKER_PROTOCOL_OVERRIDE_ENV = "PASEO_TERMINAL_WORKER_PROTOCOL_OVERRIDE"; + +/** How a daemon at `current` treats a worker reporting `reported`. */ +export type TerminalWorkerProtocolFit = "current" | "previous" | "obsolete" | "newer"; + +export function classifyTerminalWorkerProtocol( + reported: unknown, + current: number = TERMINAL_WORKER_PROTOCOL_VERSION, +): TerminalWorkerProtocolFit { + // A version this daemon cannot read comes from a Paseo it does not know: treat it as newer. + if (typeof reported !== "number" || !Number.isSafeInteger(reported)) return "newer"; + if (reported === current) return "current"; + if (reported === current - 1) return "previous"; + return reported < current ? "obsolete" : "newer"; +} + +export interface TerminalWorkerHello { + type: "hello"; + secret: string; + protocolVersion: number; +} + +export interface TerminalWorkerHelloAck { + type: "helloAck"; + pid: number; + protocolVersion: number; +} + +export interface WorkerListedTerminal { + terminal: WorkerTerminalInfo; + state: TerminalState; + activityToken: string | null; + replayPreamble: string; } export interface WorkerKillAndWaitOptions { @@ -87,7 +136,12 @@ export type TerminalWorkerRequest = stripAnsi?: boolean; } | { - type: "killAll"; + type: "listTerminals"; + requestId: string; + } + | { + // Kill every terminal (waiting for each program to exit) and exit. + type: "shutdown"; requestId: string; } | { @@ -149,6 +203,8 @@ export type TerminalWorkerEvent = export type TerminalWorkerToParentMessage = TerminalWorkerResponse | TerminalWorkerEvent; +export type TerminalWorkerListResult = WorkerListedTerminal[]; + export type TerminalWorkerCaptureResult = CaptureTerminalLinesResult; // The worker fills TerminalStateSnapshot.replayPreamble on getTerminalState so // the parent can cache the input-mode preamble instead of re-deriving it. diff --git a/packages/server/src/terminal/terminal-worker-reaper.test.ts b/packages/server/src/terminal/terminal-worker-reaper.test.ts new file mode 100644 index 0000000000..a872325d31 --- /dev/null +++ b/packages/server/src/terminal/terminal-worker-reaper.test.ts @@ -0,0 +1,640 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { execFile, spawn, type ChildProcess } from "node:child_process"; +import { chmodSync, existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { Worker } from "node:worker_threads"; +import { isPlatform } from "../test-utils/platform.js"; +import { currentBootId } from "../server/pid-lock.js"; +import { + createTerminalWorkerProcessInspector, + type TerminalWorkerProcessInspector, +} from "./terminal-worker-processes.js"; +import { + readUnconfirmedTerminalCleanup, + reapTerminalWorkers, + retryUnconfirmedTerminalCleanup, + TerminalCleanupNotConfirmedError, +} from "./terminal-worker-reaper.js"; +import { + readTerminalWorkerRecord, + terminalCleanupRecordPath, + terminalWorkerDirectory, + writeTerminalWorkerRecord, + type TerminalWorkerRecord, +} from "./terminal-worker-record.js"; + +// Real throwaway processes stand in for workers and terminal programs, so every signal the +// reaper sends lands on a process this test started. Nothing here uses invented pids. + +const inspector = createTerminalWorkerProcessInspector(); +const children: ChildProcess[] = []; +const extraPids: number[] = []; +const temporaryDirs: string[] = []; + +function spawnIdle( + script = "setInterval(() => {}, 1e9)", + env: NodeJS.ProcessEnv = process.env, +): ChildProcess { + const child = spawn(process.execPath, ["-e", script], { + detached: true, + stdio: ["ignore", "pipe", "ignore"], + env, + }); + children.push(child); + return child; +} + +// A program with one child of its own in a separate process group, so only the descendant walk +// (not the program's group kill) can reach it. +async function spawnProgramWithChild(): Promise<{ program: ChildProcess; childPid: number }> { + const program = spawnIdle(` + const child = require("node:child_process").spawn(process.execPath, ["-e", "setInterval(() => {}, 1e9)"], { stdio: "ignore", detached: true }); + process.stdout.write(String(child.pid) + "\\n"); + setInterval(() => {}, 1e9); + `); + const childPid = await new Promise((resolve) => { + program.stdout!.once("data", (chunk: Buffer) => resolve(Number(chunk.toString().trim()))); + }); + extraPids.push(childPid); + return { program, childPid }; +} + +async function birthOf(pid: number): Promise { + const inspection = await inspector.inspect(pid); + if (inspection.status !== "alive" || !inspection.snapshot.startedAt) { + throw new Error(`cannot read birth time of ${pid}`); + } + return inspection.snapshot.startedAt; +} + +async function goneProcessPid(): Promise { + const child = spawn(process.execPath, ["-e", ""], { stdio: "ignore" }); + await new Promise((resolve) => child.once("exit", resolve)); + return child.pid!; +} + +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +async function waitUntilGone(pid: number): Promise { + const deadline = Date.now() + 5_000; + while (isAlive(pid) && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 50)); + } +} + +// A member of the terminal "t1" carries its id, as every program a terminal starts does. +function terminalEnv(terminalId: string): NodeJS.ProcessEnv { + return { ...process.env, PASEO_TERMINAL_ID: terminalId }; +} + +// A process's start environment: /proc where the real inspector reads it (Linux), `ps -E` on +// macOS, whose real inspector has no session walk and so never reads one. +function readEnvironment(pid: number): Promise { + if (process.platform === "linux") return inspector.readEnvironment(pid); + return new Promise((resolve) => { + execFile("ps", ["-E", "-ww", "-o", "command=", "-p", String(pid)], (error, stdout) => { + resolve(error ? null : stdout.trim().split(/\s+/)); + }); + }); +} + +// The real process table as a Linux `ps` would print it for one session: `memberPid` belongs to +// the session `sessionId` led, and every other process to none this test knows. macOS `ps` has +// no session column, so this is the only way to run the Linux branch here; the member is a real +// process the test started. `reusedRoot` adds a process that took the session id's pid. +function withSessionMember(input: { + memberPid: number; + sessionId: number; + reusedRoot?: boolean; + environmentUnreadable?: boolean; +}): TerminalWorkerProcessInspector { + return { + reportsSessions: true, + inspect: (pid) => inspector.inspect(pid), + listProcesses: async () => { + // A fresh snapshot each call, so it is rewritten in place. + const snapshot = await inspector.listProcesses(); + for (const entry of snapshot) { + entry.sessionId = entry.pid === input.memberPid ? input.sessionId : null; + } + if (input.reusedRoot) { + snapshot.push({ + pid: input.sessionId, + ppid: 1, + startedAt: "Fri Jan 1 00:00:00 2100", + sessionId: input.sessionId, + }); + } + return snapshot; + }, + readEnvironment: (pid) => + input.environmentUnreadable + ? Promise.reject(new Error("permission denied")) + : readEnvironment(pid), + }; +} + +// A record this process cannot open until another thread restores it, the way a worker's +// rename over its record ends a Windows sharing violation. The thread waits for `release()`, then +// restores the record while this thread is still inside the (synchronous) record read. +async function unreadableUntilReleased( + recordPath: string, +): Promise<{ release: () => void; restored: Promise }> { + chmodSync(recordPath, 0); + const signal = new Int32Array(new SharedArrayBuffer(8)); + const thread = new Worker( + ` + const { parentPort, workerData } = require("node:worker_threads"); + const signal = new Int32Array(workerData.signal); + parentPort.postMessage("ready"); + Atomics.wait(signal, 0, 0); + Atomics.wait(signal, 1, 0, 30); + require("node:fs").chmodSync(workerData.recordPath, 0o600); + `, + { eval: true, workerData: { recordPath, signal: signal.buffer } }, + ); + await new Promise((resolve) => thread.once("message", resolve)); + const restored = new Promise((resolve, reject) => { + thread.once("exit", () => resolve()); + thread.once("error", reject); + }); + return { + release: () => { + Atomics.store(signal, 0, 1); + Atomics.notify(signal, 0); + }, + restored, + }; +} + +function makeHome(): string { + const home = mkdtempSync(path.join(tmpdir(), "tw-reaper-")); + temporaryDirs.push(home); + return home; +} + +function writeRecord( + home: string, + input: Partial & Pick, +): string { + const record: TerminalWorkerRecord = { + recordId: "record-under-test", + bootId: currentBootId(), + createdAt: new Date().toISOString(), + identity: { + startedAt: "Thu Jan 1 00:00:00 1970", + commandLine: "node terminal-worker --paseo-terminal-worker record-under-test", + }, + protocolVersion: 1, + paseoVersion: "0.0.0-test", + socketPath: path.join(home, "missing.sock"), + secret: "not-a-real-secret", + terminals: [], + ...input, + }; + return writeTerminalWorkerRecord(home, record); +} + +afterEach(async () => { + for (const child of children.splice(0)) { + if (child.pid && isAlive(child.pid)) { + try { + process.kill(-child.pid, "SIGKILL"); + } catch { + child.kill("SIGKILL"); + } + } + } + for (const pid of extraPids.splice(0)) { + if (isAlive(pid)) process.kill(pid, "SIGKILL"); + } + for (const dir of temporaryDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }); + } +}); + +describe.skipIf(isPlatform("win32"))("terminal worker reaper", () => { + it("ends the programs a dead worker left, and their children, then retires the record", async () => { + const home = makeHome(); + const { program, childPid } = await spawnProgramWithChild(); + const recordPath = writeRecord(home, { + pid: await goneProcessPid(), + terminals: [{ id: "t1", pid: program.pid!, startedAt: await birthOf(program.pid!) }], + }); + + const result = await reapTerminalWorkers({ paseoHome: home, inspector }); + + await waitUntilGone(program.pid!); + await waitUntilGone(childPid); + expect(isAlive(program.pid!)).toBe(false); + expect(isAlive(childPid)).toBe(false); + expect(result.signaled).toEqual(expect.arrayContaining([program.pid!, childPid])); + expect(existsSync(recordPath)).toBe(false); + }); + + it("never signals a live process that reuses the worker pid, but still ends its programs", async () => { + const home = makeHome(); + const reusedWorkerPid = spawnIdle().pid!; + const program = spawnIdle(); + writeRecord(home, { + pid: reusedWorkerPid, + terminals: [{ id: "t1", pid: program.pid!, startedAt: await birthOf(program.pid!) }], + }); + + const result = await reapTerminalWorkers({ paseoHome: home, inspector }); + + await waitUntilGone(program.pid!); + expect(isAlive(program.pid!)).toBe(false); + expect(isAlive(reusedWorkerPid)).toBe(true); + expect(result.signaled).not.toContain(reusedWorkerPid); + }); + + it("leaves a program alive when its pid now belongs to a process with another birth time", async () => { + const home = makeHome(); + const unrelated = spawnIdle(); + const recordPath = writeRecord(home, { + pid: await goneProcessPid(), + terminals: [{ id: "t1", pid: unrelated.pid!, startedAt: "Thu Jan 1 00:00:00 1970" }], + }); + + await reapTerminalWorkers({ paseoHome: home, inspector, gracefulMs: 0 }); + + expect(isAlive(unrelated.pid!)).toBe(true); + expect(existsSync(recordPath)).toBe(false); + }); + + it("retires a record from an earlier boot without signaling anything it names", async () => { + const home = makeHome(); + const program = spawnIdle(); + const recordPath = writeRecord(home, { + pid: spawnIdle().pid!, + bootId: null, + createdAt: "2000-01-01T00:00:00.000Z", + terminals: [{ id: "t1", pid: program.pid!, startedAt: await birthOf(program.pid!) }], + }); + + await reapTerminalWorkers({ paseoHome: home, inspector }); + + expect(isAlive(program.pid!)).toBe(true); + expect(existsSync(recordPath)).toBe(false); + }); + + it("keeps a record of what it could not inspect and reports cleanup unconfirmed", async () => { + const home = makeHome(); + const uninspectable = spawnIdle(); + const killable = spawnIdle(); + const failing: TerminalWorkerProcessInspector = { + reportsSessions: inspector.reportsSessions, + inspect: (pid) => + pid === uninspectable.pid + ? Promise.resolve({ status: "error", error: new Error("ps unavailable") }) + : inspector.inspect(pid), + listProcesses: () => inspector.listProcesses(), + readEnvironment: (pid) => inspector.readEnvironment(pid), + }; + const workerPid = await goneProcessPid(); + const recordPath = writeRecord(home, { + pid: workerPid, + terminals: [ + { id: "t1", pid: uninspectable.pid!, startedAt: await birthOf(uninspectable.pid!) }, + { id: "t2", pid: killable.pid!, startedAt: await birthOf(killable.pid!) }, + ], + }); + + let caught: unknown; + try { + await reapTerminalWorkers({ paseoHome: home, inspector: failing, confirmMs: 500 }); + } catch (error) { + caught = error; + } + + expect(caught).toBeInstanceOf(TerminalCleanupNotConfirmedError); + expect((caught as TerminalCleanupNotConfirmedError).unresolved).toEqual([ + expect.objectContaining({ pid: uninspectable.pid, reason: "inspection failed" }), + ]); + await waitUntilGone(killable.pid!); + expect(isAlive(killable.pid!)).toBe(false); + expect(isAlive(uninspectable.pid!)).toBe(true); + // The closing state goes to the cleanup record; the worker's own record is left as it was. + const kept = await readTerminalWorkerRecord( + terminalCleanupRecordPath(home, "record-under-test"), + ); + expect(kept.pid).toBe(workerPid); + expect(kept.closing?.unresolved.length).toBeGreaterThan(0); + expect(kept.terminals.map((entry) => entry.pid)).toEqual([uninspectable.pid]); + expect((await readTerminalWorkerRecord(recordPath)).closing).toBeUndefined(); + }); + + it("ends a session member its gone terminal program left behind, where sessions are reported", async () => { + const home = makeHome(); + const member = spawnIdle(undefined, terminalEnv("t1")); + const goneProgramPid = await goneProcessPid(); + const recordPath = writeRecord(home, { + pid: await goneProcessPid(), + terminals: [{ id: "t1", pid: goneProgramPid, startedAt: await birthOf(process.pid) }], + }); + + const result = await reapTerminalWorkers({ + paseoHome: home, + inspector: withSessionMember({ memberPid: member.pid!, sessionId: goneProgramPid }), + }); + + await waitUntilGone(member.pid!); + expect(isAlive(member.pid!)).toBe(false); + expect(result.signaled).toContain(member.pid); + expect(existsSync(recordPath)).toBe(false); + }); + + it("leaves alive a later session that reuses a gone terminal program's session id", async () => { + const home = makeHome(); + // Born after the recorded program, as any member of a later session is; only the missing + // terminal id tells it apart. + const member = spawnIdle(undefined, terminalEnv("another-terminal")); + const goneProgramPid = await goneProcessPid(); + const recordPath = writeRecord(home, { + pid: await goneProcessPid(), + terminals: [{ id: "t1", pid: goneProgramPid, startedAt: "Thu Jan 1 00:00:00 1970" }], + }); + + const result = await reapTerminalWorkers({ + paseoHome: home, + inspector: withSessionMember({ memberPid: member.pid!, sessionId: goneProgramPid }), + }); + + expect(isAlive(member.pid!)).toBe(true); + expect(result.signaled).toEqual([]); + expect(existsSync(recordPath)).toBe(false); + }); + + it("signals no session member it cannot identify, and keeps the cleanup unconfirmed", async () => { + const home = makeHome(); + const member = spawnIdle(undefined, terminalEnv("t1")); + const goneProgramPid = await goneProcessPid(); + writeRecord(home, { + pid: await goneProcessPid(), + terminals: [{ id: "t1", pid: goneProgramPid, startedAt: "Thu Jan 1 00:00:00 1970" }], + }); + + await expect( + reapTerminalWorkers({ + paseoHome: home, + inspector: withSessionMember({ + memberPid: member.pid!, + sessionId: goneProgramPid, + environmentUnreadable: true, + }), + }), + ).rejects.toBeInstanceOf(TerminalCleanupNotConfirmedError); + + expect(isAlive(member.pid!)).toBe(true); + expect(await readUnconfirmedTerminalCleanup(home)).toEqual([ + expect.objectContaining({ pid: goneProgramPid, reason: "inspection failed" }), + ]); + }); + + it("signals no session member whose environment carries no terminal id, and keeps the cleanup unconfirmed", async () => { + const home = makeHome(); + // A job started with a scrubbed environment (`env -i`) carries no terminal id at all: it + // may still be this terminal's, so it is neither signaled nor dismissed as unrelated. + const member = spawnIdle(undefined, {}); + const goneProgramPid = await goneProcessPid(); + writeRecord(home, { + pid: await goneProcessPid(), + terminals: [{ id: "t1", pid: goneProgramPid, startedAt: "Thu Jan 1 00:00:00 1970" }], + }); + const scrubbedMember = withSessionMember({ memberPid: member.pid!, sessionId: goneProgramPid }); + + for (let attempt = 0; attempt < 2; attempt += 1) { + await expect( + reapTerminalWorkers({ paseoHome: home, inspector: scrubbedMember }), + ).rejects.toBeInstanceOf(TerminalCleanupNotConfirmedError); + expect(isAlive(member.pid!)).toBe(true); + expect(await readUnconfirmedTerminalCleanup(home)).toEqual([ + expect.objectContaining({ + pid: goneProgramPid, + reason: "inspection failed", + detail: expect.stringContaining(`session member ${member.pid}`), + }), + ]); + } + expect(existsSync(terminalCleanupRecordPath(home, "record-under-test"))).toBe(true); + }); + + it("does not walk a gone terminal program's pid that another process took before the table was read", async () => { + const home = makeHome(); + const member = spawnIdle(undefined, terminalEnv("t1")); + const goneProgramPid = await goneProcessPid(); + writeRecord(home, { + pid: await goneProcessPid(), + terminals: [{ id: "t1", pid: goneProgramPid, startedAt: "Thu Jan 1 00:00:00 1970" }], + }); + + const result = await reapTerminalWorkers({ + paseoHome: home, + inspector: withSessionMember({ + memberPid: member.pid!, + sessionId: goneProgramPid, + reusedRoot: true, + }), + }); + + expect(isAlive(member.pid!)).toBe(true); + expect(result.signaled).toEqual([]); + }); + + // A kill cleanup that meets an unreadable worker record cannot check what it names and + // leaves nothing else behind, so status and retry must keep reporting it after a respawn. + it.each([ + ["cleanup", (home: string) => terminalCleanupRecordPath(home, "record-under-test")], + ["worker", (home: string) => path.join(terminalWorkerDirectory(home), "4242.json")], + ])("never reports an unreadable %s record as finished", async (_kind, recordPathIn) => { + const home = makeHome(); + mkdirSync(terminalWorkerDirectory(home), { recursive: true }); + const unreadablePath = recordPathIn(home); + writeFileSync(unreadablePath, "{ not json"); + + for (const attempt of [ + async () => await readUnconfirmedTerminalCleanup(home), + () => retryUnconfirmedTerminalCleanup(home), + () => reapTerminalWorkers({ paseoHome: home, inspector }), + ]) { + let caught: unknown; + try { + await attempt(); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(TerminalCleanupNotConfirmedError); + expect((caught as TerminalCleanupNotConfirmedError).unreadableRecords).toEqual([ + expect.objectContaining({ path: unreadablePath }), + ]); + } + expect(existsSync(unreadablePath)).toBe(true); + }); + + it("signals nothing while the process table cannot be read, so a retry still ends the children", async () => { + const home = makeHome(); + const { program, childPid } = await spawnProgramWithChild(); + const recordPath = writeRecord(home, { + pid: await goneProcessPid(), + terminals: [{ id: "t1", pid: program.pid!, startedAt: await birthOf(program.pid!) }], + }); + const tableUnavailable: TerminalWorkerProcessInspector = { + reportsSessions: inspector.reportsSessions, + inspect: (pid) => inspector.inspect(pid), + listProcesses: () => Promise.reject(new Error("ps timed out")), + readEnvironment: (pid) => inspector.readEnvironment(pid), + }; + + await expect( + reapTerminalWorkers({ paseoHome: home, inspector: tableUnavailable }), + ).rejects.toBeInstanceOf(TerminalCleanupNotConfirmedError); + + expect(isAlive(program.pid!)).toBe(true); + expect(isAlive(childPid)).toBe(true); + const kept = await readTerminalWorkerRecord( + terminalCleanupRecordPath(home, "record-under-test"), + ); + expect(kept.closing?.unresolved.length).toBeGreaterThan(0); + expect(kept.terminals.map((entry) => entry.pid)).toEqual([program.pid]); + + const retry = await reapTerminalWorkers({ paseoHome: home, inspector, gracefulMs: 0 }); + + await waitUntilGone(program.pid!); + await waitUntilGone(childPid); + expect(isAlive(program.pid!)).toBe(false); + expect(isAlive(childPid)).toBe(false); + expect(retry.signaled).toEqual(expect.arrayContaining([program.pid!, childPid])); + expect(existsSync(recordPath)).toBe(false); + expect(existsSync(terminalCleanupRecordPath(home, "record-under-test"))).toBe(false); + }); + + it("escalates to SIGKILL a program that ignores SIGHUP and SIGTERM", async () => { + const home = makeHome(); + const stubborn = spawnIdle(` + process.on("SIGHUP", () => {}); + process.on("SIGTERM", () => {}); + setInterval(() => {}, 1e9); + `); + writeRecord(home, { + pid: await goneProcessPid(), + terminals: [{ id: "t1", pid: stubborn.pid!, startedAt: await birthOf(stubborn.pid!) }], + }); + + await reapTerminalWorkers({ paseoHome: home, inspector }); + + await waitUntilGone(stubborn.pid!); + expect(isAlive(stubborn.pid!)).toBe(false); + }); + + it.skipIf(process.getuid?.() === 0)( + "keeps the daemon event loop responsive while status retries an unreadable record", + async () => { + const home = makeHome(); + const recordPath = writeRecord(home, { pid: process.pid }); + chmodSync(recordPath, 0); + let serviced = false; + setImmediate(() => { + serviced = true; + }); + try { + await expect( + Promise.resolve().then(() => readUnconfirmedTerminalCleanup(home)), + ).rejects.toBeInstanceOf(TerminalCleanupNotConfirmedError); + expect(serviced).toBe(true); + } finally { + chmodSync(recordPath, 0o600); + } + }, + ); + + // chmod cannot make a file unreadable to root. + describe.skipIf(process.getuid?.() === 0)("a record that cannot be opened for a moment", () => { + it("still ends a live worker it names, then retires the record", async () => { + const home = makeHome(); + const worker = spawn( + process.execPath, + ["-e", "setInterval(() => {}, 1e9)", "--", "--paseo-terminal-worker", "record-under-test"], + { detached: true, stdio: "ignore" }, + ); + children.push(worker); + const recordPath = writeRecord(home, { + pid: worker.pid!, + identity: { + startedAt: await birthOf(worker.pid!), + commandLine: "node -e --paseo-terminal-worker record-under-test", + }, + }); + const record = await unreadableUntilReleased(recordPath); + + record.release(); + const result = await reapTerminalWorkers({ paseoHome: home, inspector, gracefulMs: 0 }); + await record.restored; + + await waitUntilGone(worker.pid!); + expect(isAlive(worker.pid!)).toBe(false); + expect(result.signaled).toContain(worker.pid!); + expect(existsSync(recordPath)).toBe(false); + }); + + it("confirms the cleanup of a worker that already exited, then retires the record", async () => { + const home = makeHome(); + const program = spawnIdle(); + const recordPath = writeRecord(home, { + pid: await goneProcessPid(), + terminals: [{ id: "t1", pid: program.pid!, startedAt: await birthOf(program.pid!) }], + }); + const record = await unreadableUntilReleased(recordPath); + + record.release(); + const result = await reapTerminalWorkers({ paseoHome: home, inspector, gracefulMs: 0 }); + await record.restored; + + await waitUntilGone(program.pid!); + expect(isAlive(program.pid!)).toBe(false); + expect(result.retired).toBe(1); + expect(existsSync(recordPath)).toBe(false); + }); + + it("still reports a record that stays unreadable, and signals nothing it names", async () => { + const home = makeHome(); + const worker = spawn( + process.execPath, + ["-e", "setInterval(() => {}, 1e9)", "--", "--paseo-terminal-worker", "record-under-test"], + { detached: true, stdio: "ignore" }, + ); + children.push(worker); + const recordPath = writeRecord(home, { + pid: worker.pid!, + identity: { + startedAt: await birthOf(worker.pid!), + commandLine: "node -e --paseo-terminal-worker record-under-test", + }, + }); + chmodSync(recordPath, 0); + + let caught: unknown; + try { + await reapTerminalWorkers({ paseoHome: home, inspector, gracefulMs: 0 }); + } catch (error) { + caught = error; + } + + expect(caught).toBeInstanceOf(TerminalCleanupNotConfirmedError); + expect((caught as TerminalCleanupNotConfirmedError).unreadableRecords).toEqual([ + expect.objectContaining({ path: recordPath, error: expect.stringContaining("EACCES") }), + ]); + expect(isAlive(worker.pid!)).toBe(true); + chmodSync(recordPath, 0o600); + expect(existsSync(recordPath)).toBe(true); + }); + }); +}); diff --git a/packages/server/src/terminal/terminal-worker-reaper.ts b/packages/server/src/terminal/terminal-worker-reaper.ts new file mode 100644 index 0000000000..2647d4b25e --- /dev/null +++ b/packages/server/src/terminal/terminal-worker-reaper.ts @@ -0,0 +1,642 @@ +import { randomUUID } from "node:crypto"; +import { setTimeout as delay } from "node:timers/promises"; +import treeKill from "tree-kill"; +import { writtenDuringThisBoot } from "../server/pid-lock.js"; +import { connectTerminalWorker } from "./terminal-worker-client.js"; +import { pruneTerminalWorkerRuntimes } from "./terminal-worker-runtime.js"; +import { + classifyTerminalProgramProcess, + classifyTerminalWorkerProcess, + type ProcessOwnership, +} from "./terminal-worker-ownership.js"; +import { + collectDescendants, + createTerminalWorkerProcessInspector, + type ProcessTableEntry, + type TerminalWorkerProcessInspector, +} from "./terminal-worker-processes.js"; +import { + listTerminalWorkerRecords, + retireTerminalWorkerRecord, + terminalCleanupRecordPath, + writeTerminalCleanupRecord, + type TerminalCleanupUnresolved, + type TerminalWorkerRecord, + type TerminalWorkerRecordFile, + type TerminalWorkerTerminalEntry, +} from "./terminal-worker-record.js"; + +// The one kill path for terminal workers outside a connected daemon, and the one the daemon +// uses for its own kill stop. Every kill intent (stop, quit, --force, a kill restart whose +// shutdown hung, a dead or stopped worker, a program that ignores HUP) ends here: +// +// 1. Classify the worker and every terminal program of each record on its own, by birth time. +// A record from an earlier boot names only processes the reboot ended: retire it unsignaled. +// Descendants of owned programs (and, on Linux, members of their sessions) join the +// candidates from one process-table snapshot taken before anything is signaled. Once a +// program is gone, its session id can belong to a later session, so a member of a gone +// program's session joins only when its environment carries that terminal's id. When the +// table cannot be read, nothing in the record is signaled: ending a program would reparent +// the children only the table can find, and a retry could never reach them again. +// 2. An owned worker gets an authenticated `shutdown`: it kills each terminal and waits for it. +// 3. After `gracefulMs`, SIGKILL whatever owned process is left, re-inspecting right before +// each signal. +// 4. Succeed only when every candidate is gone or proven unrelated. Otherwise keep a record of +// the unresolved entries, marked closing, in a cleanup record no worker rewrites, and throw +// TerminalCleanupNotConfirmedError. An unreadable record is never reported confirmed. + +const DEFAULT_GRACEFUL_MS = 5_000; +const DEFAULT_CONFIRM_MS = 3_000; +const POLL_INTERVAL_MS = 100; + +export interface UnresolvedTerminalProcess extends TerminalCleanupUnresolved { + recordPath: string; +} + +// What a cleanup could not confirm, as a caller reports it: read here, or from the daemon +// status of a host the caller cannot read itself. +export interface UnconfirmedTerminalProcess { + pid: number; + role: string; + reason: string; + detail: string; +} + +export interface UnreadableTerminalWorkerRecord { + path: string; + error: string; +} + +export class TerminalCleanupNotConfirmedError extends Error { + readonly code = "TERMINAL_CLEANUP_NOT_CONFIRMED"; + readonly unresolved: UnconfirmedTerminalProcess[]; + readonly unreadableRecords: UnreadableTerminalWorkerRecord[]; + + constructor( + unresolved: UnconfirmedTerminalProcess[], + unreadableRecords: UnreadableTerminalWorkerRecord[] = [], + ) { + super( + [ + "Terminal cleanup was not confirmed.", + ...(unresolved.length > 0 + ? [ + "These processes may still be running:", + ...unresolved.map( + (entry) => ` pid ${entry.pid} (${entry.role}): ${entry.reason} (${entry.detail})`, + ), + "Their record is kept; the next `paseo daemon stop` retries the cleanup.", + ] + : []), + ...(unreadableRecords.length > 0 + ? [ + "These records cannot be read, so the processes they name cannot be checked:", + ...unreadableRecords.map((entry) => ` ${entry.path}: ${entry.error}`), + "Repair or remove each once you have checked its processes are gone.", + ] + : []), + ].join("\n"), + ); + this.name = "TerminalCleanupNotConfirmedError"; + this.unresolved = unresolved; + this.unreadableRecords = unreadableRecords; + } +} + +/** + * Retries the cleanup an earlier kill could not confirm (records marked closing), leaving any + * live worker a daemon controls alone. Throws TerminalCleanupNotConfirmedError when processes + * still survive, or when a record cannot be read, so a kill restart can report them. + */ +export async function retryUnconfirmedTerminalCleanup(paseoHome: string): Promise { + const files = await listTerminalWorkerRecords(paseoHome); + const recordPaths = files.flatMap((file) => + ("record" in file && file.record.closing) || "error" in file ? [file.path] : [], + ); + if (recordPaths.length === 0) return; + await reapTerminalWorkers({ paseoHome, gracefulMs: 0, recordPaths }); +} + +/** + * What the last cleanup of this home could not confirm gone, as its closing records name it. + * Read without retrying, so a daemon can report it (daemon status) to clients that cannot read + * this home themselves. Throws TerminalCleanupNotConfirmedError when any record cannot be read, + * worker records included: a kill cleanup that met it could not check what it names, and a + * failed cleanup leaves nothing else behind, so the cleanup cannot be reported finished. + */ +export async function readUnconfirmedTerminalCleanup( + paseoHome: string, +): Promise { + const files = await listTerminalWorkerRecords(paseoHome); + const unresolved = files.flatMap((file) => + "record" in file && file.record.closing + ? file.record.closing.unresolved.map((entry) => ({ ...entry, recordPath: file.path })) + : [], + ); + const unreadable = files.flatMap((file) => + "error" in file ? [{ path: file.path, error: file.error.message }] : [], + ); + if (unreadable.length > 0) { + throw new TerminalCleanupNotConfirmedError(unresolved, unreadable); + } + return unresolved; +} + +export interface TerminalWorkerReapResult { + records: number; + retired: number; + signaled: number[]; +} + +export interface ReapTerminalWorkersOptions { + paseoHome: string; + /** Time the worker gets to close its terminals before SIGKILL. 0 skips the shutdown request. */ + gracefulMs?: number; + confirmMs?: number; + /** Limit cleanup to these record files. Defaults to every record in the home. */ + recordPaths?: string[]; + inspector?: TerminalWorkerProcessInspector; + log?: (message: string, fields?: Record) => void; +} + +interface Candidate { + pid: number; + startedAt: string; + role: UnresolvedTerminalProcess["role"]; + terminalId: string; + state: ProcessOwnership; + // Set when this program's children could not be looked up: its cleanup is never reported + // confirmed, even once the program itself is gone, and its entry stays for the retry. + descendantsUnconfirmed: string | null; +} + +interface CandidateDiscovery { + candidates: Candidate[]; + // The process table could not be read, so no process of the record may be signaled yet. + tableError: string | null; +} + +export async function reapTerminalWorkers( + options: ReapTerminalWorkersOptions, +): Promise { + const inspector = options.inspector ?? createTerminalWorkerProcessInspector(); + const gracefulMs = options.gracefulMs ?? DEFAULT_GRACEFUL_MS; + const confirmMs = options.confirmMs ?? DEFAULT_CONFIRM_MS; + const log = options.log ?? (() => {}); + const wanted = options.recordPaths ? new Set(options.recordPaths) : null; + const result: TerminalWorkerReapResult = { records: 0, retired: 0, signaled: [] }; + const unresolved: UnresolvedTerminalProcess[] = []; + const unreadable: UnreadableTerminalWorkerRecord[] = []; + + for (const file of await listTerminalWorkerRecords(options.paseoHome)) { + if (wanted && !wanted.has(file.path)) continue; + result.records += 1; + if ("error" in file) { + // Nothing in an unreadable record can identify a process, so nothing may be signaled, + // and nothing it names can be confirmed gone. + log("Skipping unreadable terminal worker record", { + recordPath: file.path, + error: file.error.message, + }); + unreadable.push({ path: file.path, error: file.error.message }); + continue; + } + const outcome = await reapRecord({ + paseoHome: options.paseoHome, + file, + inspector, + gracefulMs, + confirmMs, + log, + }); + result.signaled.push(...outcome.signaled); + if (outcome.unresolved.length === 0) { + result.retired += 1; + } else { + unresolved.push(...outcome.unresolved); + } + } + + // Copies of worker files no remaining record names belong to no worker now. + const pruned = await pruneTerminalWorkerRuntimes(options.paseoHome); + for (const failure of pruned.failed) { + log("Cannot remove a terminal worker copy; the next cleanup retries", failure); + } + + if (unresolved.length > 0 || unreadable.length > 0) { + throw new TerminalCleanupNotConfirmedError(unresolved, unreadable); + } + return result; +} + +type ReadableRecordFile = Extract; + +interface ReapRecordInput { + paseoHome: string; + file: ReadableRecordFile; + inspector: TerminalWorkerProcessInspector; + gracefulMs: number; + confirmMs: number; + log: NonNullable; +} + +async function reapRecord( + input: ReapRecordInput, +): Promise<{ signaled: number[]; unresolved: UnresolvedTerminalProcess[] }> { + const { file, inspector, log } = input; + const { record } = file; + const recordPath = file.path; + if (!writtenDuringThisBoot({ startedAt: record.createdAt, bootId: record.bootId })) { + await retireTerminalWorkerRecord(file); + log("Retired terminal worker record from an earlier boot", { recordPath }); + return { signaled: [], unresolved: [] }; + } + + const worker = classifyTerminalWorkerProcess(record, await inspector.inspect(record.pid)); + const discovery = await collectCandidates(record, inspector, log); + const tracked: TrackedProcesses = { + record, + inspector, + worker, + candidates: discovery.candidates, + }; + + if (discovery.tableError !== null) { + return await keepUnresolved(input, tracked, [], `not signaled: ${discovery.tableError}`); + } + + if (tracked.worker.status === "owned" && input.gracefulMs > 0) { + await requestWorkerShutdown(record, input.gracefulMs, log); + } + await waitUntilSettled( + tracked, + Date.now() + (tracked.worker.status === "owned" ? input.gracefulMs : 0), + ); + + let signaled: number[] = []; + if (!isSettled(tracked)) { + signaled = await forceKillOwned(tracked, recordPath, log); + await waitUntilSettled(tracked, Date.now() + input.confirmMs); + } + + return await keepUnresolved(input, tracked, signaled, null); +} + +// Retires the record when every process it names is settled; otherwise writes the cleanup +// record, marked closing, with only the unresolved entries. A live worker keeps rewriting its +// own record, so the closing state never goes there. `notSignaled` explains open processes that +// were deliberately left alone. +async function keepUnresolved( + input: Pick, + tracked: TrackedProcesses, + signaled: number[], + notSignaled: string | null, +): Promise<{ signaled: number[]; unresolved: UnresolvedTerminalProcess[] }> { + const { record } = tracked; + const { file, log } = input; + const recordPath = terminalCleanupRecordPath(input.paseoHome, record.recordId); + const { unresolved, remaining } = collectUnresolved(tracked, recordPath, notSignaled); + if (unresolved.length === 0) { + await retireTerminalWorkerRecord(file); + log("Terminal worker record cleaned up", { recordPath: file.path, signaled }); + return { signaled, unresolved }; + } + await writeTerminalCleanupRecord(input.paseoHome, { + ...record, + terminals: remaining, + closing: { + unresolved: unresolved.map(({ pid, role, reason, detail }) => ({ + pid, + role, + reason, + detail, + })), + }, + }); + log("Terminal worker cleanup not confirmed; record kept", { + recordPath, + unresolved: unresolved.map((entry) => ({ pid: entry.pid, reason: entry.reason })), + }); + return { signaled, unresolved }; +} + +interface TrackedProcesses { + record: TerminalWorkerRecord; + inspector: TerminalWorkerProcessInspector; + worker: ProcessOwnership; + candidates: Candidate[]; +} + +function isOpen(state: ProcessOwnership): boolean { + return state.status === "owned" || state.status === "unknown"; +} + +function isSettled(tracked: TrackedProcesses): boolean { + return !isOpen(tracked.worker) && tracked.candidates.every((c) => !isOpen(c.state)); +} + +async function refreshTracked(tracked: TrackedProcesses): Promise { + if (isOpen(tracked.worker)) { + tracked.worker = classifyTerminalWorkerProcess( + tracked.record, + await tracked.inspector.inspect(tracked.record.pid), + ); + } + for (const candidate of tracked.candidates) { + if (isOpen(candidate.state)) { + candidate.state = classifyTerminalProgramProcess( + candidate, + await tracked.inspector.inspect(candidate.pid), + ); + } + } +} + +async function waitUntilSettled(tracked: TrackedProcesses, deadline: number): Promise { + await refreshTracked(tracked); + while (!isSettled(tracked) && Date.now() < deadline) { + await delay(POLL_INTERVAL_MS); + await refreshTracked(tracked); + } +} + +// SIGKILL every process still owned, re-inspecting each right before its signal. +async function forceKillOwned( + tracked: TrackedProcesses, + recordPath: string, + log: NonNullable, +): Promise { + const { record, inspector } = tracked; + const signaled: number[] = []; + if (tracked.worker.status === "owned") { + tracked.worker = classifyTerminalWorkerProcess(record, await inspector.inspect(record.pid)); + if (tracked.worker.status === "owned") { + await forceKillWorker(record.pid); + signaled.push(record.pid); + log("Force-killed terminal worker", { pid: record.pid, recordPath }); + } + } + for (const candidate of tracked.candidates) { + if (candidate.state.status !== "owned") continue; + candidate.state = classifyTerminalProgramProcess( + candidate, + await inspector.inspect(candidate.pid), + ); + if (candidate.state.status !== "owned") continue; + forceKillProgram(candidate.pid); + signaled.push(candidate.pid); + log("Force-killed terminal program", { + pid: candidate.pid, + terminalId: candidate.terminalId, + recordPath, + }); + } + return signaled; +} + +function collectUnresolved( + tracked: TrackedProcesses, + recordPath: string, + notSignaled: string | null, +): { unresolved: UnresolvedTerminalProcess[]; remaining: TerminalWorkerTerminalEntry[] } { + const unresolved: UnresolvedTerminalProcess[] = []; + if (isOpen(tracked.worker)) { + unresolved.push( + describeUnresolved({ + pid: tracked.record.pid, + role: "worker", + state: tracked.worker, + recordPath, + notSignaled, + }), + ); + } + const remaining: TerminalWorkerTerminalEntry[] = []; + for (const candidate of tracked.candidates) { + const stillOpen = isOpen(candidate.state); + if (!stillOpen && candidate.descendantsUnconfirmed === null) continue; + unresolved.push( + stillOpen + ? describeUnresolved({ + pid: candidate.pid, + role: candidate.role, + state: candidate.state, + recordPath, + notSignaled, + }) + : { + pid: candidate.pid, + role: candidate.role, + reason: "inspection failed", + detail: candidate.descendantsUnconfirmed!, + recordPath, + }, + ); + remaining.push({ + id: candidate.terminalId, + pid: candidate.pid, + startedAt: candidate.startedAt, + }); + } + return { unresolved, remaining }; +} + +async function collectCandidates( + record: TerminalWorkerRecord, + inspector: TerminalWorkerProcessInspector, + log: NonNullable, +): Promise { + const candidates = new Map(); + for (const entry of record.terminals) { + candidates.set(entry.pid, { + pid: entry.pid, + startedAt: entry.startedAt, + role: "terminal program", + terminalId: entry.id, + state: classifyTerminalProgramProcess(entry, await inspector.inspect(entry.pid)), + descendantsUnconfirmed: null, + }); + } + // Owned programs are walked through the table. Where sessions are reported, so are programs + // already gone: the kernel keeps a session id reserved while a member lives, so a job the + // shell left behind is still found. A pid reused by another process (unrelated) leads someone + // else's session and is never walked. + const roots = Array.from(candidates.values()).filter( + (c) => c.state.status === "owned" || (c.state.status === "gone" && inspector.reportsSessions), + ); + if (roots.length === 0) return { candidates: Array.from(candidates.values()), tableError: null }; + + let snapshot: ProcessTableEntry[]; + try { + snapshot = await inspector.listProcesses(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + log("Cannot list processes for terminal cleanup", { error: message }); + for (const root of roots) { + root.descendantsUnconfirmed = `its child processes could not be listed: ${message}`; + } + return { + candidates: Array.from(candidates.values()), + tableError: `process table unavailable: ${message}`, + }; + } + for (const root of roots) { + const inTable = snapshot.find((entry) => entry.pid === root.pid); + if (inTable && inTable.startedAt !== root.startedAt) { + // The program ended and its pid was reused before the snapshot: what is below that pid + // now belongs to someone else, and the old session ended with every member of it. + log("Not walking a terminal program pid reused before the process table was read", { + pid: root.pid, + terminalId: root.terminalId, + }); + continue; + } + if (!inTable && !inspector.reportsSessions) continue; + // Below a live program its parent links prove ownership. Once it is gone, its session id + // may have been reused by a later session, so each member must carry the terminal's id. + const rootGone = !inTable; + for (const descendant of collectDescendants(snapshot, root.pid)) { + if (candidates.has(descendant.pid)) continue; + if (rootGone) { + const proof = await proveTerminalMember(inspector, descendant.pid, root.terminalId); + if (proof.status === "gone") continue; + if (proof.status === "unknown") { + root.descendantsUnconfirmed = `session member ${descendant.pid} cannot be identified: ${proof.reason}`; + continue; + } + if (proof.status === "unrelated") { + log("Ignoring a session member that does not carry its terminal's id", { + pid: descendant.pid, + terminalId: root.terminalId, + }); + continue; + } + } + candidates.set(descendant.pid, { + pid: descendant.pid, + startedAt: descendant.startedAt, + role: "terminal program", + terminalId: root.terminalId, + state: { status: "owned" }, + descendantsUnconfirmed: null, + }); + } + } + return { candidates: Array.from(candidates.values()), tableError: null }; +} + +// A terminal's programs inherit PASEO_TERMINAL_ID (terminal-manager.ts) from its PTY program. +// A process in a later session that happens to reuse the session id does not carry it. +async function proveTerminalMember( + inspector: TerminalWorkerProcessInspector, + pid: number, + terminalId: string, +): Promise { + let environment: string[] | null; + try { + environment = await inspector.readEnvironment(pid); + } catch (error) { + return { + status: "unknown", + reason: `environment unreadable: ${error instanceof Error ? error.message : String(error)}`, + }; + } + if (environment === null) return { status: "gone" }; + if (environment.includes(`PASEO_TERMINAL_ID=${terminalId}`)) return { status: "owned" }; + // Another terminal's id proves the process belongs elsewhere. No id at all proves nothing: a + // job started with a scrubbed environment (`env -i`) can still be this terminal's. + return environment.some((entry) => entry.startsWith("PASEO_TERMINAL_ID=")) + ? { status: "unrelated", reason: "environment carries another terminal's id" } + : { status: "unknown", reason: "its environment carries no terminal id" }; +} + +async function requestWorkerShutdown( + record: TerminalWorkerRecord, + gracefulMs: number, + log: NonNullable, +): Promise { + const deadline = Date.now() + gracefulMs; + let connection; + try { + connection = await connectTerminalWorker({ + socketPath: record.socketPath, + secret: record.secret, + timeoutMs: Math.min(gracefulMs, 2_000), + }); + } catch (error) { + // A worker that cannot be reached is escalated by birth time below; a failed connection + // alone never authorizes a signal. + log("Terminal worker did not accept a shutdown request", { + pid: record.pid, + error: error instanceof Error ? error.message : String(error), + }); + return; + } + const responded = new Promise((resolve) => { + connection.on("message", (message) => { + if (message.type === "response") resolve(); + }); + }); + const closed = new Promise((resolve) => connection.on("close", () => resolve())); + const sendFailed = new Promise((resolve) => { + connection.send({ type: "shutdown", requestId: randomUUID() }, (error) => { + if (error) resolve(); + }); + }); + const timeout = new AbortController(); + await Promise.race([ + responded, + closed, + sendFailed, + delay(Math.max(0, deadline - Date.now()), undefined, { signal: timeout.signal }).catch( + () => undefined, + ), + ]); + timeout.abort(); + connection.disconnect(); +} + +function forceKillWorker(pid: number): Promise { + if (process.platform === "win32") { + return new Promise((resolve) => { + treeKill(pid, "SIGKILL", () => resolve()); + }); + } + signalIgnoringMissing(pid, "SIGKILL"); + return Promise.resolve(); +} + +function forceKillProgram(pid: number): void { + if (process.platform !== "win32") { + // A PTY program leads its own session and process group. The group cannot outlive its + // leader's pid while any member is alive, so it holds nothing but this program's job. + signalIgnoringMissing(-pid, "SIGKILL"); + } + signalIgnoringMissing(pid, "SIGKILL"); +} + +function signalIgnoringMissing(pid: number, signal: NodeJS.Signals): void { + try { + process.kill(pid, signal); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== "ESRCH" && code !== "EPERM") throw error; + } +} + +function describeUnresolved(input: { + pid: number; + role: UnresolvedTerminalProcess["role"]; + state: ProcessOwnership; + recordPath: string; + notSignaled: string | null; +}): UnresolvedTerminalProcess { + const { pid, role, state, recordPath, notSignaled } = input; + if (state.status === "unknown") { + return { pid, role, reason: "inspection failed", detail: state.reason, recordPath }; + } + if (notSignaled !== null) { + return { pid, role, reason: "inspection failed", detail: notSignaled, recordPath }; + } + return { pid, role, reason: "still alive", detail: "survived SIGKILL", recordPath }; +} diff --git a/packages/server/src/terminal/terminal-worker-record.ts b/packages/server/src/terminal/terminal-worker-record.ts new file mode 100644 index 0000000000..3176d2ee8e --- /dev/null +++ b/packages/server/src/terminal/terminal-worker-record.ts @@ -0,0 +1,230 @@ +import { readdir, readFile, rm } from "node:fs/promises"; +import { setTimeout as delay } from "node:timers/promises"; +import path from "node:path"; +import { z } from "zod"; +import { writePrivateFileAtomicSync, writePrivateFileAtomic } from "../server/private-files.js"; + +// Owner-only metadata a detached terminal worker leaves in $PASEO_HOME so the next daemon can +// find, authenticate and adopt it, and so cleanup can identify every process it is allowed to +// signal. Process identity is the birth time `ps`/Win32_Process report, never the pid alone. + +const TerminalWorkerTerminalEntrySchema = z.object({ + id: z.string().min(1), + pid: z.number().int().positive(), + startedAt: z.string().min(1), +}); + +// One process a cleanup could not confirm gone, kept so a later caller (a retry, or a remote +// client reading daemon status) can name it. +const TerminalCleanupUnresolvedSchema = z.object({ + pid: z.number().int().positive(), + role: z.enum(["worker", "terminal program"]), + reason: z.enum(["still alive", "inspection failed"]), + detail: z.string(), +}); + +const TerminalWorkerRecordSchema = z.object({ + recordId: z.string().min(1), + pid: z.number().int().positive(), + bootId: z.string().nullable(), + createdAt: z.string().min(1), + identity: z.object({ + startedAt: z.string().min(1), + commandLine: z.string().min(1), + }), + // Read loosely: a version this daemon cannot interpret marks a worker from a newer Paseo + // (classifyTerminalWorkerProtocol), not an unreadable record. + protocolVersion: z.union([z.number(), z.string()]), + paseoVersion: z.string(), + socketPath: z.string().min(1), + secret: z.string().min(1), + // The copy of the worker's files it runs from (a directory name under terminal-worker/), or + // null when it runs from source. A copy named by any record is never deleted. + runtime: z.string().nullable().optional(), + terminals: z.array(TerminalWorkerTerminalEntrySchema), + // Set only in a cleanup record (below), with what a cleanup could not confirm gone. A record + // marked closing is never adopted: the next daemon or stop retries the cleanup instead. + closing: z.object({ unresolved: z.array(TerminalCleanupUnresolvedSchema).min(1) }).optional(), +}); + +export type TerminalWorkerTerminalEntry = z.infer; +export type TerminalCleanupUnresolved = z.infer; +export type TerminalWorkerRecord = z.infer; + +// A worker record (`.json`) belongs to the worker, which rewrites it from memory whenever +// a terminal starts or exits. A cleanup that cannot confirm every process gone therefore keeps +// its state in a cleanup record of its own (`closing-.json`) that no worker writes, so +// the kill intent and what it could not confirm survive any later worker rewrite. +type TerminalWorkerRecordKind = "worker" | "cleanup"; + +export type TerminalWorkerRecordFile = + | { + // The file holding this record's state: the cleanup record when there is one. + path: string; + record: TerminalWorkerRecord; + // The worker's own record file, when it still exists. + workerRecordPath: string | null; + } + | { path: string; error: Error }; + +// The worker's command line carries this marker followed by its record id, so a reused pid +// running some other process (even another worker) never matches the record. +export const TERMINAL_WORKER_MARKER_ARG = "--paseo-terminal-worker"; + +export function terminalWorkerDirectory(paseoHome: string): string { + return path.join(paseoHome, "terminal-worker"); +} + +export function terminalWorkerRecordPath(paseoHome: string, workerPid: number): string { + return path.join(terminalWorkerDirectory(paseoHome), `${workerPid}.json`); +} + +export function terminalCleanupRecordPath(paseoHome: string, recordId: string): string { + return path.join(terminalWorkerDirectory(paseoHome), `closing-${recordId}.json`); +} + +export function terminalWorkerLogPath(paseoHome: string): string { + return path.join(paseoHome, "terminal-worker.log"); +} + +export function writeTerminalWorkerRecord(paseoHome: string, record: TerminalWorkerRecord): string { + const recordPath = terminalWorkerRecordPath(paseoHome, record.pid); + writePrivateFileAtomicSync(recordPath, `${JSON.stringify(record, null, 2)}\n`); + return recordPath; +} + +/** Writes the state of a cleanup that could not confirm every process gone. */ +export async function writeTerminalCleanupRecord( + paseoHome: string, + record: TerminalWorkerRecord & { closing: NonNullable }, +): Promise { + if (!RECORD_ID_PATTERN.test(record.recordId)) { + throw new Error(`Terminal worker record id is not a file name: ${record.recordId}`); + } + const recordPath = terminalCleanupRecordPath(paseoHome, record.recordId); + await writePrivateFileAtomic(recordPath, `${JSON.stringify(record, null, 2)}\n`); + return recordPath; +} + +export async function readTerminalWorkerRecord(recordPath: string): Promise { + return TerminalWorkerRecordSchema.parse(JSON.parse(await readRecordFile(recordPath))); +} + +// A worker rewrites its record by renaming a new file over it. On Windows an open() that races +// that rename fails with EPERM (or EBUSY) for a moment, so such an error is retried for a short +// window before the record counts as unreadable. Reads and retries yield so daemon status and +// cleanup never block unrelated requests while another process holds the record open. +const TRANSIENT_READ_ERRORS = new Set(["EPERM", "EBUSY", "EACCES"]); +const TRANSIENT_READ_WINDOW_MS = 250; +const TRANSIENT_READ_PAUSE_MS = 10; + +async function readRecordFile(recordPath: string): Promise { + const deadline = Date.now() + TRANSIENT_READ_WINDOW_MS; + for (;;) { + try { + return await readFile(recordPath, "utf8"); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (!code || !TRANSIENT_READ_ERRORS.has(code) || Date.now() >= deadline) throw error; + await delay(TRANSIENT_READ_PAUSE_MS); + } + } +} + +const RECORD_ID_PATTERN = /^[A-Za-z0-9-]+$/; +const WORKER_RECORD_NAME = /^\d+\.json$/; +const CLEANUP_RECORD_NAME = /^closing-[A-Za-z0-9-]+\.json$/; + +function recordKindOf(fileName: string): TerminalWorkerRecordKind | null { + if (WORKER_RECORD_NAME.test(fileName)) return "worker"; + if (CLEANUP_RECORD_NAME.test(fileName)) return "cleanup"; + return null; +} + +/** + * Every record in the home. A cleanup record is merged with the worker record of the same + * record id: its closing state wins, and terminals the worker recorded since join it, so the + * retry covers everything the worker still runs. + */ +export async function listTerminalWorkerRecords( + paseoHome: string, +): Promise { + const directory = terminalWorkerDirectory(paseoHome); + let names: string[]; + try { + names = await readdir(directory); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + throw error; + } + const errors: TerminalWorkerRecordFile[] = []; + const workers: Array<{ path: string; record: TerminalWorkerRecord }> = []; + const cleanups: Array<{ path: string; record: TerminalWorkerRecord }> = []; + for (const name of names.sort()) { + const kind = recordKindOf(name); + if (!kind) continue; + const recordPath = path.join(directory, name); + try { + const record = await readTerminalWorkerRecord(recordPath); + (kind === "worker" ? workers : cleanups).push({ path: recordPath, record }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") continue; + errors.push({ + path: recordPath, + error: error instanceof Error ? error : new Error(String(error)), + }); + } + } + const files: TerminalWorkerRecordFile[] = []; + const merged = new Set(); + for (const cleanup of cleanups) { + const worker = workers.find( + (entry) => + entry.record.recordId === cleanup.record.recordId && + entry.record.pid === cleanup.record.pid, + ); + if (worker) merged.add(worker.path); + const known = new Set(cleanup.record.terminals.map((entry) => `${entry.id}:${entry.pid}`)); + const added = (worker?.record.terminals ?? []).filter( + (entry) => !known.has(`${entry.id}:${entry.pid}`), + ); + files.push({ + path: cleanup.path, + record: { ...cleanup.record, terminals: [...cleanup.record.terminals, ...added] }, + workerRecordPath: worker?.path ?? null, + }); + } + for (const worker of workers) { + if (merged.has(worker.path)) continue; + files.push({ path: worker.path, record: worker.record, workerRecordPath: worker.path }); + } + return [...files, ...errors]; +} + +/** + * Deletes a record, the worker's own record file and the socket it names, unless a later worker + * that reused the pid wrote that file since. The pipe namespace on Windows needs no unlink. + */ +export async function retireTerminalWorkerRecord(file: { + path: string; + record: TerminalWorkerRecord; + workerRecordPath: string | null; +}): Promise { + if (file.path !== file.workerRecordPath) await rm(file.path, { force: true }); + if (file.workerRecordPath) { + if (await workerRecordNamesAnother(file.workerRecordPath, file.record.recordId)) return; + await rm(file.workerRecordPath, { force: true }); + } + if (process.platform !== "win32") { + await rm(file.record.socketPath, { force: true }); + } +} + +// True when the file now belongs to another worker, or cannot be read to tell. +async function workerRecordNamesAnother(recordPath: string, recordId: string): Promise { + try { + return (await readTerminalWorkerRecord(recordPath)).recordId !== recordId; + } catch (error) { + return (error as NodeJS.ErrnoException).code !== "ENOENT"; + } +} diff --git a/packages/server/src/terminal/terminal-worker-runtime.test.ts b/packages/server/src/terminal/terminal-worker-runtime.test.ts new file mode 100644 index 0000000000..feb4bf3309 --- /dev/null +++ b/packages/server/src/terminal/terminal-worker-runtime.test.ts @@ -0,0 +1,222 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { builtinModules } from "node:module"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { isPlatform } from "../test-utils/platform.js"; +import { + prepareTerminalWorkerRuntime, + pruneTerminalWorkerRuntimes, + TerminalWorkerRuntimeError, +} from "./terminal-worker-runtime.js"; +import { + terminalWorkerDirectory, + writeTerminalWorkerRecord, + type TerminalWorkerRecord, +} from "./terminal-worker-record.js"; + +const temporaryDirs: string[] = []; + +afterEach(() => { + for (const dir of temporaryDirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +function makeDir(prefix: string): string { + const dir = mkdtempSync(path.join(tmpdir(), prefix)); + temporaryDirs.push(dir); + return dir; +} + +// A built bundle directory and a node-pty package shaped like the real ones. +function makeSources(bundleSource = "export {};\n"): { + bundleDirectory: string; + nodePtyDirectory: string; +} { + const bundleDirectory = makeDir("tw-bundle-"); + writeFileSync(path.join(bundleDirectory, "terminal-worker.mjs"), bundleSource); + mkdirSync(path.join(bundleDirectory, "shell-integration", "zsh"), { recursive: true }); + writeFileSync(path.join(bundleDirectory, "shell-integration", "zsh", ".zshenv"), "# zsh\n"); + const nodePtyDirectory = makeDir("tw-node-pty-"); + writeFileSync(path.join(nodePtyDirectory, "package.json"), '{"version":"1.2.3"}\n'); + mkdirSync(path.join(nodePtyDirectory, "lib")); + writeFileSync(path.join(nodePtyDirectory, "lib", "index.js"), "module.exports = {};\n"); + const native = path.join(nodePtyDirectory, "prebuilds", `${process.platform}-${process.arch}`); + mkdirSync(native, { recursive: true }); + writeFileSync(path.join(native, "pty.node"), "native"); + writeFileSync(path.join(native, "spawn-helper"), "#!/bin/sh\n", { mode: 0o755 }); + // Another platform's build is not copied. + mkdirSync(path.join(nodePtyDirectory, "prebuilds", "other-platform")); + return { bundleDirectory, nodePtyDirectory }; +} + +function recordNaming(runtime: string | null, pid = 4242): TerminalWorkerRecord { + return { + recordId: `record-${pid}`, + pid, + bootId: null, + createdAt: new Date().toISOString(), + identity: { startedAt: "Thu Jan 1 00:00:00 1970", commandLine: "worker" }, + protocolVersion: 1, + paseoVersion: "1.0.0", + socketPath: "/tmp/does-not-exist.sock", + secret: "secret", + runtime, + terminals: [], + }; +} + +describe("terminal worker runtime copy", () => { + it("copies the bundle, its shell integration and this platform's node-pty into the home", () => { + const home = makeDir("tw-runtime-home-"); + const entry = prepareTerminalWorkerRuntime({ + paseoHome: home, + paseoVersion: "1.0.0-beta.1", + ...makeSources(), + }); + + const directory = path.join(terminalWorkerDirectory(home), entry.runtime!); + expect(entry.runtime).toMatch(/^1\.0\.0-beta\.1-[0-9a-f]{12}$/); + expect(entry.scriptPath).toBe(path.join(directory, "terminal-worker.mjs")); + expect(entry.execArgv).toEqual([]); + expect(existsSync(path.join(directory, "shell-integration", "zsh", ".zshenv"))).toBe(true); + const nodePty = path.join(directory, "node_modules", "node-pty"); + expect(existsSync(path.join(nodePty, "lib", "index.js"))).toBe(true); + expect(readdirSync(path.join(nodePty, "prebuilds"))).toEqual([ + `${process.platform}-${process.arch}`, + ]); + if (!isPlatform("win32")) { + const helper = path.join(nodePty, "prebuilds", `${process.platform}-${process.arch}`); + expect(statSync(path.join(helper, "spawn-helper")).mode & 0o111).toBe(0o111); + } + }); + + it("reuses a finished copy of the same content and makes a new one for a different build", () => { + const home = makeDir("tw-runtime-home-"); + const sources = makeSources(); + const first = prepareTerminalWorkerRuntime({ + paseoHome: home, + paseoVersion: "1.0.0", + ...sources, + }); + const marker = path.join(terminalWorkerDirectory(home), first.runtime!, "terminal-worker.mjs"); + writeFileSync(marker, "// the running copy\n"); + + const again = prepareTerminalWorkerRuntime({ + paseoHome: home, + paseoVersion: "1.0.0", + ...sources, + }); + const rebuilt = prepareTerminalWorkerRuntime({ + paseoHome: home, + paseoVersion: "1.0.0", + ...makeSources("export const rebuilt = true;\n"), + }); + + expect(again.runtime).toBe(first.runtime); + expect(statSync(marker).size).toBe("// the running copy\n".length); + expect(rebuilt.runtime).not.toBe(first.runtime); + }); + + it("replaces a copy an earlier run left unfinished", () => { + const home = makeDir("tw-runtime-home-"); + const sources = makeSources(); + const first = prepareTerminalWorkerRuntime({ + paseoHome: home, + paseoVersion: "1.0.0", + ...sources, + }); + const directory = path.join(terminalWorkerDirectory(home), first.runtime!); + rmSync(directory, { recursive: true }); + mkdirSync(directory); + + prepareTerminalWorkerRuntime({ paseoHome: home, paseoVersion: "1.0.0", ...sources }); + + expect(existsSync(path.join(directory, "terminal-worker.mjs"))).toBe(true); + }); + + it("fails before copying anything when node-pty has no native build for this platform", () => { + const home = makeDir("tw-runtime-home-"); + const sources = makeSources(); + rmSync(path.join(sources.nodePtyDirectory, "prebuilds"), { recursive: true }); + + expect(() => + prepareTerminalWorkerRuntime({ paseoHome: home, paseoVersion: "1.0.0", ...sources }), + ).toThrow(TerminalWorkerRuntimeError); + expect(readdirSync(terminalWorkerDirectory(home))).toEqual([]); + }); + + it("removes only copies no worker record names", async () => { + const home = makeDir("tw-runtime-home-"); + const used = prepareTerminalWorkerRuntime({ + paseoHome: home, + paseoVersion: "1.0.0", + ...makeSources(), + }); + const unused = prepareTerminalWorkerRuntime({ + paseoHome: home, + paseoVersion: "0.9.0", + ...makeSources(), + }); + writeTerminalWorkerRecord(home, recordNaming(used.runtime)); + const socketDirectory = path.join(terminalWorkerDirectory(home), "sock"); + mkdirSync(socketDirectory); + + const result = await pruneTerminalWorkerRuntimes(home); + + expect(result.removed).toEqual([path.join(terminalWorkerDirectory(home), unused.runtime!)]); + expect(existsSync(path.dirname(used.scriptPath))).toBe(true); + expect(existsSync(socketDirectory)).toBe(true); + }); + + it("removes nothing while a record cannot be read", async () => { + const home = makeDir("tw-runtime-home-"); + const copy = prepareTerminalWorkerRuntime({ + paseoHome: home, + paseoVersion: "1.0.0", + ...makeSources(), + }); + writeFileSync(path.join(terminalWorkerDirectory(home), "4242.json"), "{ not json"); + + expect((await pruneTerminalWorkerRuntimes(home)).removed).toEqual([]); + expect(existsSync(copy.scriptPath)).toBe(true); + }); +}); + +describe("terminal worker bundle", () => { + // The copy holds the bundle and node-pty only. A package the bundle loads at run time + // (a require hidden from esbuild behind createRequire) would be missing from it. + it("loads nothing at run time but Node built-ins and node-pty", async () => { + const outDir = makeDir("tw-bundle-build-"); + const script = fileURLToPath( + new URL("../../scripts/build-terminal-worker.mjs", import.meta.url), + ); + const { buildTerminalWorkerBundle } = (await import(script)) as { + buildTerminalWorkerBundle(input: { outDir: string }): Promise; + }; + await buildTerminalWorkerBundle({ outDir }); + + const bundle = readFileSync(path.join(outDir, "terminal-worker.mjs"), "utf8"); + const loaded = new Set(); + for (const match of bundle.matchAll( + /(?:\brequire\w*(?:\.resolve)?\(|\bfrom\s+|\bimport\s*\()\s*"([^"]+)"/g, + )) { + loaded.add(match[1]!); + } + const builtins = new Set(builtinModules.flatMap((name) => [name, `node:${name}`])); + const packages = [...loaded].filter((specifier) => !builtins.has(specifier)); + expect(packages.map((specifier) => specifier.split("/")[0])).toEqual( + packages.map(() => "node-pty"), + ); + expect(existsSync(path.join(outDir, "shell-integration", "zsh"))).toBe(true); + }); +}); diff --git a/packages/server/src/terminal/terminal-worker-runtime.ts b/packages/server/src/terminal/terminal-worker-runtime.ts new file mode 100644 index 0000000000..7ee0696478 --- /dev/null +++ b/packages/server/src/terminal/terminal-worker-runtime.ts @@ -0,0 +1,249 @@ +import { createHash, randomUUID } from "node:crypto"; +import { + chmodSync, + existsSync, + mkdirSync, + readdirSync, + readFileSync, + renameSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { createRequire } from "node:module"; +import { access, readdir, rm } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { ensurePrivateDirectory } from "../server/private-files.js"; +import { listTerminalWorkerRecords, terminalWorkerDirectory } from "./terminal-worker-record.js"; + +// A terminal worker outlives the installation that started it. An npm or desktop update +// replaces that installation's files, so a worker launched from them would fail its next +// terminal create (node-pty's spawn-helper, the zsh integration). A built daemon therefore +// copies the worker bundle, its shell integration and node-pty into +// $PASEO_HOME/terminal-worker/-/ and launches the worker from there. +// The hash keeps two builds of one version (a local rebuild) from sharing a copy. A copy is +// removed once no worker record names it: after a worker drains, after a kill cleanup, and at +// daemon start. Dev runs (.ts source) launch from source and copy nothing. + +export const TERMINAL_WORKER_BUNDLE_NAME = "terminal-worker.mjs"; +// Written last into a finished copy; a directory without it is an interrupted copy. +const RUNTIME_MARKER = "paseo-terminal-worker-runtime.json"; +const TEMP_PREFIX = ".tmp-"; + +export class TerminalWorkerRuntimeError extends Error { + constructor(message: string, options?: { cause?: unknown }) { + super(message, options); + this.name = "TerminalWorkerRuntimeError"; + } +} + +/** What the launcher runs: the worker script, the node flags it needs, and the copy it is in. */ +export interface TerminalWorkerEntry { + scriptPath: string; + execArgv: string[]; + /** The copy's directory name under terminal-worker/, or null when run from source. */ + runtime: string | null; +} + +function sourceExecArgv(): string[] { + const loaderUrl = new URL("./terminal-ts-loader.mjs", import.meta.url).href; + const importSource = [ + 'import { register } from "node:module";', + 'import { pathToFileURL } from "node:url";', + `register(${JSON.stringify(loaderUrl)}, pathToFileURL("./"));`, + ].join(" "); + return [ + "--experimental-strip-types", + "--import", + `data:text/javascript,${encodeURIComponent(importSource)}`, + ]; +} + +/** + * The entry a daemon launches its worker from: the source file in dev, otherwise a copy of the + * built bundle in this home, prepared on demand. + */ +export function resolveTerminalWorkerEntry(input: { + paseoHome: string; + paseoVersion: string; +}): TerminalWorkerEntry { + if (import.meta.url.endsWith(".ts")) { + return { + scriptPath: fileURLToPath(new URL("./terminal-worker-process.ts", import.meta.url)), + execArgv: sourceExecArgv(), + runtime: null, + }; + } + const bundleDirectory = fileURLToPath(new URL("../terminal-worker/", import.meta.url)); + if (!existsSync(path.join(bundleDirectory, TERMINAL_WORKER_BUNDLE_NAME))) { + throw new TerminalWorkerRuntimeError( + `The terminal worker bundle is missing from ${bundleDirectory}; rebuild @getpaseo/server (npm run build).`, + ); + } + const require = createRequire(import.meta.url); + const nodePtyDirectory = path.dirname(require.resolve("node-pty/package.json")); + return prepareTerminalWorkerRuntime({ + paseoHome: input.paseoHome, + paseoVersion: input.paseoVersion, + bundleDirectory, + nodePtyDirectory, + }); +} + +function runtimeName(input: { + paseoVersion: string; + bundleDirectory: string; + nodePtyDirectory: string; +}): string { + const hash = createHash("sha256"); + hash.update(readFileSync(path.join(input.bundleDirectory, TERMINAL_WORKER_BUNDLE_NAME))); + hash.update(readFileSync(path.join(input.nodePtyDirectory, "package.json"))); + hash.update(`${process.platform}-${process.arch}`); + const version = input.paseoVersion.replace(/[^A-Za-z0-9._-]/g, "_"); + return `${version}-${hash.digest("hex").slice(0, 12)}`; +} + +// Plain fs calls on purpose: inside the desktop app the sources live in app.asar, which +// Electron's fs reads transparently but newer helpers like cpSync do not. +function copyTree(source: string, destination: string): void { + const stat = statSync(source); + if (stat.isDirectory()) { + mkdirSync(destination, { recursive: true }); + for (const name of readdirSync(source)) { + copyTree(path.join(source, name), path.join(destination, name)); + } + return; + } + writeFileSync(destination, readFileSync(source)); + // Keeps node-pty's spawn-helper executable. + chmodSync(destination, stat.mode & 0o777); +} + +function copyNodePty(source: string, destination: string): void { + mkdirSync(destination, { recursive: true }); + copyTree(path.join(source, "package.json"), path.join(destination, "package.json")); + copyTree(path.join(source, "lib"), path.join(destination, "lib")); + const native = [ + path.join("build", "Release"), + path.join("prebuilds", `${process.platform}-${process.arch}`), + ].filter((relative) => existsSync(path.join(source, relative))); + if (native.length === 0) { + throw new TerminalWorkerRuntimeError( + `node-pty at ${source} has no native build for ${process.platform}-${process.arch}`, + ); + } + for (const relative of native) { + copyTree(path.join(source, relative), path.join(destination, relative)); + } +} + +function isCompleteRuntime(directory: string): boolean { + return existsSync(path.join(directory, RUNTIME_MARKER)); +} + +/** + * Copies the worker bundle, its shell integration and node-pty into this home, unless a finished + * copy of the same content is already there. Returns the entry to launch from the copy. + */ +export function prepareTerminalWorkerRuntime(input: { + paseoHome: string; + paseoVersion: string; + bundleDirectory: string; + nodePtyDirectory: string; +}): TerminalWorkerEntry { + const root = terminalWorkerDirectory(input.paseoHome); + ensurePrivateDirectory(root); + const name = runtimeName(input); + const directory = path.join(root, name); + const entry = { + scriptPath: path.join(directory, TERMINAL_WORKER_BUNDLE_NAME), + execArgv: [], + runtime: name, + }; + if (isCompleteRuntime(directory)) return entry; + + // An interrupted copy left behind: no worker can be running from it. + rmSync(directory, { recursive: true, force: true }); + const temporary = path.join(root, `${TEMP_PREFIX}${name}-${randomUUID()}`); + try { + mkdirSync(temporary, { recursive: true }); + copyTree( + path.join(input.bundleDirectory, TERMINAL_WORKER_BUNDLE_NAME), + path.join(temporary, TERMINAL_WORKER_BUNDLE_NAME), + ); + copyTree( + path.join(input.bundleDirectory, "shell-integration"), + path.join(temporary, "shell-integration"), + ); + copyNodePty(input.nodePtyDirectory, path.join(temporary, "node_modules", "node-pty")); + writeFileSync( + path.join(temporary, RUNTIME_MARKER), + `${JSON.stringify({ paseoVersion: input.paseoVersion, copiedAt: new Date().toISOString() })}\n`, + ); + renameSync(temporary, directory); + } catch (error) { + rmSync(temporary, { recursive: true, force: true }); + if (isCompleteRuntime(directory)) return entry; + throw new TerminalWorkerRuntimeError( + `Cannot copy the terminal worker into ${directory}: ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, + ); + } + return entry; +} + +export interface PruneTerminalWorkerRuntimesResult { + removed: string[]; + /** Copies that could not be deleted (a worker that just exited can still hold files on Windows). */ + failed: Array<{ directory: string; error: string }>; +} + +/** + * Deletes every copy no worker record names. With an unreadable record it deletes nothing: that + * record could name a copy a live worker runs from. + */ +export async function pruneTerminalWorkerRuntimes( + paseoHome: string, +): Promise { + const result: PruneTerminalWorkerRuntimesResult = { removed: [], failed: [] }; + const root = terminalWorkerDirectory(paseoHome); + const files = await listTerminalWorkerRecords(paseoHome); + const inUse = new Set(); + for (const file of files) { + if ("error" in file) return result; + if (file.record.runtime) inUse.add(file.record.runtime); + } + let names: string[]; + try { + names = await readdir(root); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return result; + throw error; + } + for (const name of names) { + if (inUse.has(name)) continue; + const directory = path.join(root, name); + if ( + !name.startsWith(TEMP_PREFIX) && + !(await access(path.join(directory, RUNTIME_MARKER)).then( + () => true, + (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT" || error.code === "ENOTDIR") return false; + throw error; + }, + )) + ) + continue; + try { + await rm(directory, { recursive: true, force: true }); + result.removed.push(directory); + } catch (error) { + result.failed.push({ + directory, + error: error instanceof Error ? error.message : String(error), + }); + } + } + return result; +} diff --git a/packages/server/src/terminal/terminal-worker-socket-path.test.ts b/packages/server/src/terminal/terminal-worker-socket-path.test.ts new file mode 100644 index 0000000000..409cf60c81 --- /dev/null +++ b/packages/server/src/terminal/terminal-worker-socket-path.test.ts @@ -0,0 +1,134 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { chmodSync, mkdirSync, mkdtempSync, rmSync, symlinkSync } from "node:fs"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { isPlatform } from "../test-utils/platform.js"; +import { + resolveTerminalWorkerSocketPath, + TerminalWorkerSocketPathError, +} from "./terminal-worker-socket-path.js"; + +const temporaryDirs: string[] = []; + +function makeDir(prefix: string, root = tmpdir()): string { + const dir = mkdtempSync(path.join(root, prefix)); + temporaryDirs.push(dir); + return dir; +} + +// A home path long enough that its socket cannot fit sun_path, with multi-byte segments so the +// budget is measured in bytes, not characters. +function longHome(root: string): string { + const home = path.join(root, "worktree-ñandú-".repeat(5), "paseo-home"); + mkdirSync(home, { recursive: true }); + return home; +} + +afterEach(() => { + for (const dir of temporaryDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }); + } +}); + +describe.skipIf(isPlatform("win32"))("terminal worker socket path", () => { + it("uses the home's own socket directory when the path fits", () => { + // macOS $TMPDIR alone is ~50 bytes; a short root keeps the home's own path in budget. + const home = makeDir("tw-h-", "/tmp"); + const socketPath = resolveTerminalWorkerSocketPath({ + paseoHome: home, + workerPid: 4242, + platform: "darwin", + }); + expect(socketPath.startsWith(path.join(home, "terminal-worker", "sock"))).toBe(true); + }); + + it("falls to /tmp/paseo- when the home and TMPDIR are too long, within the byte budget", () => { + const root = makeDir("tw-sock-long-"); + const home = longHome(root); + const tmpDir = path.join(root, "tmp-ñandú-".repeat(6)); + mkdirSync(tmpDir, { recursive: true }); + const uid = process.getuid!(); + const socketPath = resolveTerminalWorkerSocketPath({ + paseoHome: home, + workerPid: 99999, + platform: "darwin", + tmpDir, + }); + expect(path.dirname(socketPath)).toBe(`/tmp/paseo-${uid}`); + expect(Buffer.byteLength(socketPath, "utf8")).toBeLessThanOrEqual(103); + }); + + it("rejects a shared-root directory that is a symlink, has another mode, or is not a directory", () => { + const root = makeDir("tw-sock-reject-"); + const home = longHome(root); + const uid = process.getuid!(); + const tmpDir = makeDir("tw-"); + const target = makeDir("tw-target-"); + chmodSync(target, 0o700); + symlinkSync(target, path.join(tmpDir, `paseo-${uid}`)); + // A budget check alone would accept the long-home fallback through the symlink; it must + // also be refused, and /tmp/paseo- takes over. + const socketPath = resolveTerminalWorkerSocketPath({ + paseoHome: home, + workerPid: 7, + platform: "darwin", + tmpDir, + }); + expect(path.dirname(socketPath)).toBe(`/tmp/paseo-${uid}`); + + const wideTmp = makeDir("tw-"); + mkdirSync(path.join(wideTmp, `paseo-${uid}`), { mode: 0o755 }); + chmodSync(path.join(wideTmp, `paseo-${uid}`), 0o755); + expect( + path.dirname( + resolveTerminalWorkerSocketPath({ + paseoHome: home, + workerPid: 7, + platform: "darwin", + tmpDir: wideTmp, + }), + ), + ).toBe(`/tmp/paseo-${uid}`); + }); + + it("names every rejected candidate with its byte length when none fits", () => { + const root = makeDir("tw-sock-none-"); + const home = longHome(root); + const tmpDir = path.join(root, "tmp-ñandú-".repeat(6)); + mkdirSync(tmpDir, { recursive: true }); + let caught: unknown; + try { + resolveTerminalWorkerSocketPath({ + paseoHome: home, + workerPid: 7, + platform: "darwin", + tmpDir, + // Another uid's /tmp/paseo- directory does not exist or is not ours; with a uid + // nobody owns, the /tmp candidate's directory is created by us but owned by our real + // uid, so it is rejected for its owner. + uid: 2_000_000_000, + }); + } catch (error) { + caught = error; + } + rmSync("/tmp/paseo-2000000000", { recursive: true, force: true }); + expect(caught).toBeInstanceOf(TerminalWorkerSocketPathError); + const rejected = (caught as TerminalWorkerSocketPathError).rejected; + expect(rejected).toHaveLength(3); + expect(rejected[0].reason).toMatch(/longer than 103 bytes/); + expect(rejected[0].byteLength).toBe(Buffer.byteLength(rejected[0].path, "utf8")); + expect(rejected[2].reason).toMatch(/owned by uid/); + }); + + it("accepts a real listen on the chosen path", async () => { + const home = makeDir("tw-l-", "/tmp"); + const socketPath = resolveTerminalWorkerSocketPath({ paseoHome: home, workerPid: 5 }); + const server = createServer(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(socketPath, () => resolve()); + }); + await new Promise((resolve) => server.close(() => resolve())); + }); +}); diff --git a/packages/server/src/terminal/terminal-worker-socket-path.ts b/packages/server/src/terminal/terminal-worker-socket-path.ts new file mode 100644 index 0000000000..99fce0e64c --- /dev/null +++ b/packages/server/src/terminal/terminal-worker-socket-path.ts @@ -0,0 +1,133 @@ +import { createHash } from "node:crypto"; +import { lstatSync, mkdirSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { ensurePrivateDirectory } from "../server/private-files.js"; +import { terminalWorkerDirectory } from "./terminal-worker-record.js"; + +// `sun_path` holds 104 bytes on macOS and 108 on Linux, NUL included. Dev homes under +// worktrees and long $TMPDIR values exceed that, so the socket goes to the first owner-only +// directory whose full path fits, measured in UTF-8 bytes. +const SOCKET_PATH_BYTE_BUDGET: Partial> = { + darwin: 103, + linux: 107, +}; +const DEFAULT_SOCKET_PATH_BYTE_BUDGET = 103; +const OWNER_ONLY_MODE = 0o700; + +export interface TerminalWorkerSocketPathInput { + paseoHome: string; + workerPid: number; + platform?: NodeJS.Platform; + tmpDir?: string; + uid?: number; +} + +export interface RejectedSocketCandidate { + path: string; + byteLength: number; + reason: string; +} + +export class TerminalWorkerSocketPathError extends Error { + readonly rejected: RejectedSocketCandidate[]; + + constructor(rejected: RejectedSocketCandidate[], budget: number) { + super( + [ + `No terminal worker socket path fits ${budget} bytes in an owner-only directory:`, + ...rejected.map( + (candidate) => ` ${candidate.path} (${candidate.byteLength} bytes): ${candidate.reason}`, + ), + ].join("\n"), + ); + this.name = "TerminalWorkerSocketPathError"; + this.rejected = rejected; + } +} + +/** First 16 hex chars of sha256(home) plus the worker pid: unique per home and worker. */ +export function terminalWorkerSocketId(paseoHome: string, workerPid: number): string { + const digest = createHash("sha256").update(path.resolve(paseoHome)).digest("hex"); + return `${digest.slice(0, 16)}-${workerPid}`; +} + +/** + * Picks (and creates) the socket path a worker listens on. Windows uses a named pipe; POSIX + * tries $PASEO_HOME/terminal-worker/sock, then /paseo-, then /tmp/paseo-. + * A shared root is never chmodded: an existing directory there is used only when it is a real + * directory owned by this user with mode 0700. + */ +export function resolveTerminalWorkerSocketPath(input: TerminalWorkerSocketPathInput): string { + const platform = input.platform ?? process.platform; + const id = terminalWorkerSocketId(input.paseoHome, input.workerPid); + if (platform === "win32") { + return `\\\\.\\pipe\\paseo-tw-${id}`; + } + + const uid = input.uid ?? process.getuid?.() ?? 0; + const budget = SOCKET_PATH_BYTE_BUDGET[platform] ?? DEFAULT_SOCKET_PATH_BYTE_BUDGET; + const fileName = `tw-${id}.sock`; + const homeSocketDirectory = path.join(terminalWorkerDirectory(input.paseoHome), "sock"); + const candidates: Array<{ directory: string; prepare: () => string | null }> = [ + { + directory: homeSocketDirectory, + prepare: () => { + ensurePrivateDirectory(homeSocketDirectory); + return verifyOwnerOnlyDirectory(homeSocketDirectory, uid); + }, + }, + ...[input.tmpDir ?? tmpdir(), "/tmp"].map((root) => { + const directory = path.join(root, `paseo-${uid}`); + return { + directory, + prepare: () => prepareSharedRootDirectory(directory, uid), + }; + }), + ]; + + const rejected: RejectedSocketCandidate[] = []; + const seen = new Set(); + for (const candidate of candidates) { + const socketPath = path.join(candidate.directory, fileName); + if (seen.has(socketPath)) continue; + seen.add(socketPath); + const byteLength = Buffer.byteLength(socketPath, "utf8"); + if (byteLength > budget) { + rejected.push({ path: socketPath, byteLength, reason: `longer than ${budget} bytes` }); + continue; + } + let problem: string | null; + try { + problem = candidate.prepare(); + } catch (error) { + problem = error instanceof Error ? error.message : String(error); + } + if (problem) { + rejected.push({ path: socketPath, byteLength, reason: problem }); + continue; + } + return socketPath; + } + throw new TerminalWorkerSocketPathError(rejected, budget); +} + +function prepareSharedRootDirectory(directory: string, uid: number): string | null { + try { + mkdirSync(directory, { mode: OWNER_ONLY_MODE }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + } + return verifyOwnerOnlyDirectory(directory, uid); +} + +/** Returns why the directory cannot hold the socket, or null when it is owner-only. */ +function verifyOwnerOnlyDirectory(directory: string, uid: number): string | null { + const stats = lstatSync(directory); + if (stats.isSymbolicLink()) return "is a symlink"; + if (!stats.isDirectory()) return "is not a directory"; + if (stats.uid !== uid) return `is owned by uid ${stats.uid}, not ${uid}`; + const mode = stats.mode & 0o777; + if (mode !== OWNER_ONLY_MODE) return `has mode ${mode.toString(8)}, not 700`; + return null; +} diff --git a/packages/server/src/terminal/terminal.ts b/packages/server/src/terminal/terminal.ts index 24f243b6ea..b8cb7ee8cd 100644 --- a/packages/server/src/terminal/terminal.ts +++ b/packages/server/src/terminal/terminal.ts @@ -15,6 +15,11 @@ import { basename, delimiter, dirname, extname, join, resolve as resolvePath } f import { createRequire } from "node:module"; import { fileURLToPath } from "node:url"; import { createExternalProcessEnv } from "../server/paseo-env.js"; +import { + resolveExternalProcessPath, + resolvePaseoCliBinDir, + resolvePaseoCliExecutablePath, +} from "./paseo-cli-path.js"; import { writePrivateFileAtomicSync } from "../server/private-files.js"; import { findExecutable, @@ -27,7 +32,6 @@ import type { TerminalActivity, TerminalActivityState } from "@getpaseo/protocol const { Terminal } = xterm; const require = createRequire(import.meta.url); -const PASEO_CLI_BIN_ENTRY = "@getpaseo/cli/bin/paseo"; let nodePtySpawnHelperChecked = false; const TERMINAL_TITLE_DEBOUNCE_MS = 150; const TERMINAL_EXIT_OUTPUT_LINE_LIMIT = 12; @@ -123,6 +127,11 @@ export interface TerminalSession { killAndWait(options?: { gracefulTimeoutMs?: number; forceTimeoutMs?: number }): Promise; } +/** A session that owns its PTY in this process, so it knows the program's pid. */ +export interface LocalTerminalSession extends TerminalSession { + readonly pid: number; +} + function parseCommandFinishedOsc(data: string): TerminalCommandFinishedInfo | null { // OSC 633 is terminal control traffic, but a foreground command can still // print arbitrary control bytes. Keep this boundary to the exact VS Code @@ -153,6 +162,11 @@ export interface CreateTerminalOptions { title?: string; command?: string; args?: string[]; + // Environment the terminal starts from instead of this process's own. A detached terminal + // worker outlives the daemon that launched it, so each create carries the controlling + // daemon's environment and its resolved Paseo CLI path. + baseEnv?: Record; + paseoCliPath?: string | null; } function toTerminalActivity(snapshot: { @@ -181,6 +195,7 @@ function isTerminalActivityInterruptInput(data: string): boolean { interface BuildTerminalEnvironmentInput { shell: string; env: Record; + baseEnv?: Record; zshShellIntegrationDir?: string; paseoCliBinDir?: string | null; paseoHookCliPath?: string | null; @@ -410,84 +425,12 @@ export async function resolveTerminalSpawnCommand( return { command: resolved, args }; } +export { resolvePaseoCliBinDir, resolvePaseoCliExecutablePath }; + export function resolveZshShellIntegrationDir(): string { return fileURLToPath(new URL("./shell-integration/zsh", import.meta.url)); } -function resolveExternalProcessPath(filePath: string): string { - return filePath.replace(/\.asar(?=[/\\]|$)/, ".asar.unpacked"); -} - -export function resolvePaseoCliBinDir(): string | null { - const cliExecutable = resolvePaseoCliExecutablePath(); - return cliExecutable ? dirname(cliExecutable) : null; -} - -export function resolvePaseoCliExecutablePath(): string | null { - const configuredCli = process.env.PASEO_CLI?.trim(); - if (configuredCli) { - return resolvePath(configuredCli); - } - - const cliEntrypoint = resolvePaseoCliBinEntrypoint(); - if (!cliEntrypoint) { - return null; - } - - const externalCliEntrypoint = resolveExternalProcessPath(cliEntrypoint); - const npmBinDir = findNpmBinDir(dirname(externalCliEntrypoint)); - if (npmBinDir) { - const shim = resolvePaseoCliShim(npmBinDir); - if (shim) { - return shim; - } - } - - return externalCliEntrypoint; -} - -function resolvePaseoCliBinEntrypoint(): string | null { - try { - return require.resolve(PASEO_CLI_BIN_ENTRY); - } catch { - return null; - } -} - -function findNpmBinDir(startPath: string): string | null { - let current = startPath; - while (true) { - const candidate = join(current, "node_modules", ".bin"); - if (hasPaseoCliShim(candidate)) { - return candidate; - } - - const parent = dirname(current); - if (parent === current) { - return null; - } - current = parent; - } -} - -function hasPaseoCliShim(binDir: string): boolean { - return resolvePaseoCliShim(binDir) !== null; -} - -function resolvePaseoCliShim(binDir: string): string | null { - for (const name of paseoCliShimNames()) { - const candidate = join(binDir, name); - if (existsSync(candidate)) { - return candidate; - } - } - return null; -} - -function paseoCliShimNames(): string[] { - return process.platform === "win32" ? ["paseo.cmd", "paseo.exe", "paseo"] : ["paseo"]; -} - function resolveZshShellIntegrationRuntimeDir(): string { let username = "unknown"; try { @@ -517,10 +460,14 @@ function prepareZshShellIntegrationRuntimeDir(sourceDir = resolveZshShellIntegra export function buildTerminalEnvironment( input: BuildTerminalEnvironmentInput, ): Record { - const baseEnv: Record = createExternalProcessEnv(process.env, input.env, { - TERM: "xterm-256color", - TERM_PROGRAM: "kitty", - }); + const baseEnv: Record = createExternalProcessEnv( + input.baseEnv ?? process.env, + input.env, + { + TERM: "xterm-256color", + TERM_PROGRAM: "kitty", + }, + ); const envWithAgentHooks = prependPaseoCliToPath( baseEnv, input.paseoCliBinDir === undefined ? resolvePaseoCliBinDir() : input.paseoCliBinDir, @@ -556,6 +503,23 @@ function injectPaseoHookCli( }; } +// The environment and CLI path a terminal starts from: the caller's when it sends them (the +// daemon controlling a detached worker does), this process's otherwise. +function resolveTerminalEnvironmentSource( + input: Pick, +): Pick { + const source: Pick< + BuildTerminalEnvironmentInput, + "baseEnv" | "paseoCliBinDir" | "paseoHookCliPath" + > = {}; + if (input.baseEnv) source.baseEnv = input.baseEnv; + if (input.paseoCliPath !== undefined) { + source.paseoCliBinDir = input.paseoCliPath ? dirname(input.paseoCliPath) : null; + source.paseoHookCliPath = input.paseoCliPath; + } + return source; +} + function prependPaseoCliToPath( env: Record, cliBinDir: string | null, @@ -967,7 +931,9 @@ export async function assertTerminalCommandCanStart( } } -export async function createTerminal(options: CreateTerminalOptions): Promise { +export async function createTerminal( + options: CreateTerminalOptions, +): Promise { const { cwd, workspaceId, @@ -980,8 +946,10 @@ export async function createTerminal(options: CreateTerminalOptions): Promise void>(); @@ -1033,6 +1001,7 @@ export async function createTerminal(options: CreateTerminalOptions): Promise void): boolean { @@ -100,14 +131,7 @@ class FakeTerminalWorker extends EventEmitter { disconnect(): void { this.connected = false; - this.emit("exit", 0, null); - } - - kill(): boolean { - this.killed = true; - this.connected = false; - this.emit("exit", 0, null); - return true; + this.emit("close"); } emitWorkerMessage(message: TerminalWorkerToParentMessage): void { @@ -115,8 +139,27 @@ class FakeTerminalWorker extends EventEmitter { } } -let manager: TerminalManager | null = null; +let manager: DaemonTerminalManager | null = null; const temporaryDirs: string[] = []; + +// A real detached worker in a throwaway home; afterEach closes its terminals and the worker. +async function startRealWorkerManager( + options: { getTerminalActivityUrl?: () => string | null; paseoHome?: string } = {}, +): Promise { + const { paseoHome = createTemporaryHome(), ...rest } = options; + return startWorkerTerminalManager({ + paseoHome, + paseoVersion: "0.0.0-test", + logger: pino({ level: "silent" }), + ...rest, + }); +} + +function createTemporaryHome(): string { + const paseoHome = mkdtempSync(join(tmpdir(), "worker-terminal-manager-home-")); + temporaryDirs.push(paseoHome); + return paseoHome; +} const terminalSessions: TerminalSession[] = []; function trackTerminal(session: TerminalSession): TerminalSession { @@ -150,7 +193,7 @@ afterEach(async () => { .catch(() => {}), ), ); - manager?.killAll(); + await manager?.closeTerminals(); manager = null; while (temporaryDirs.length > 0) { const dir = temporaryDirs.pop(); @@ -160,10 +203,177 @@ afterEach(async () => { } }); +it("retires a killed worker's sessions and shares one replacement across concurrent creates", async () => { + const paseoHome = createTemporaryHome(); + manager = await startRealWorkerManager({ paseoHome }); + manager.registerCwdEnv({ cwd: paseoHome, env: { QA_RECOVERY_ENV: "retained" } }); + const terminal = await manager.createTerminal({ + workspaceId: "ws-recovery", + cwd: paseoHome, + ...nodeTerminalCommand( + "process.stdin.on('data', chunk => process.stdout.write('echo:' + process.env.QA_RECOVERY_ENV + ':' + chunk)); setInterval(() => {}, 1000)", + ), + }); + const records = await listTerminalWorkerRecords(paseoHome); + const file = records.find((entry) => !("error" in entry)); + if (!file || "error" in file) throw new Error("Missing worker record"); + const changes: string[][] = []; + manager.subscribeTerminalsChanged((event) => + changes.push(event.terminals.map((item) => item.id)), + ); + const exits: unknown[] = []; + terminal.onExit((info) => exits.push(info)); + process.kill(file.record.pid, "SIGKILL"); + await waitForCondition(() => manager!.getTerminal(terminal.id) === undefined, 10000); + expect(changes).toContainEqual([]); + expect(exits).toHaveLength(1); + const sessions = await Promise.all( + Array.from({ length: 4 }, () => + manager!.createTerminal({ + workspaceId: "ws-recovery", + cwd: paseoHome, + ...nodeTerminalCommand( + "process.stdin.on('data', chunk => process.stdout.write('echo:' + process.env.QA_RECOVERY_ENV + ':' + chunk)); setInterval(() => {}, 1000)", + ), + }), + ), + ); + for (const session of sessions) { + const output: string[] = []; + session.subscribe((message) => { + if (message.type === "output") output.push(message.data); + }); + session.send({ type: "input", data: "recovered\r" }); + await waitForCondition(() => output.join("").includes("echo:retained:recovered"), 10000); + } + expect(await manager.getTerminals(paseoHome)).toHaveLength(4); + const remaining = (await listTerminalWorkerRecords(paseoHome)).filter( + (entry) => !("error" in entry), + ); + expect(remaining).toHaveLength(1); +}, 30000); + +it("reconnects a live worker without replacing terminal sessions or processes", async () => { + const paseoHome = createTemporaryHome(); + manager = await startRealWorkerManager({ paseoHome }); + const terminal = await manager.createTerminal({ + workspaceId: "ws-reconnect", + cwd: paseoHome, + ...nodeTerminalCommand( + "process.stdin.on('data', chunk => process.stdout.write('echo:' + chunk)); setInterval(() => {}, 1000)", + ), + }); + const [before] = await workerRecords(paseoHome); + const pids = await recordedTerminalPids(paseoHome); + let replayRequests = 0; + terminal.subscribe((message) => { + if (message.type === "snapshotReady") replayRequests += 1; + }); + // A second authenticated controller replaces the daemon's socket. The process stays alive. + const competitor = await connectTerminalWorker({ + socketPath: before!.socketPath, + secret: before!.secret, + }); + await waitForCondition(() => !competitor.connected, 10000); + await waitForCondition(async () => { + try { + return (await manager!.getTerminalState(terminal.id)) !== null; + } catch { + return false; + } + }, 10000); + await waitForCondition(() => replayRequests > 0, 1000); + expect(manager.getTerminal(terminal.id)).toBe(terminal); + expect(await recordedTerminalPids(paseoHome)).toEqual(pids); + expect((await workerRecords(paseoHome)).map((record) => record.pid)).toEqual([before!.pid]); + const output: string[] = []; + terminal.subscribe((message) => { + if (message.type === "output") output.push(message.data); + }); + terminal.send({ type: "input", data: "reconnected\r" }); + await waitForCondition(() => output.join("").includes("echo:reconnected"), 10000); +}, 30000); + +it.each(["detach", "closeTerminals"] as const)( + "does not replace a failed worker during %s", + async (operation) => { + const paseoHome = createTemporaryHome(); + manager = await startRealWorkerManager({ paseoHome }); + await manager.createTerminal({ + workspaceId: "ws-stop-recovery", + cwd: paseoHome, + ...nodeTerminalCommand("setInterval(() => {}, 1000)"), + }); + const [record] = await workerRecords(paseoHome); + process.kill(record!.pid, "SIGKILL"); + await manager[operation](); + await expect( + manager.createTerminal({ workspaceId: "ws-stop-recovery", cwd: paseoHome }), + ).rejects.toBeInstanceOf(NoTerminalWorkerAvailableError); + await manager.closeTerminals(); + expect(await workerRecords(paseoHome)).toEqual([]); + }, + 30000, +); + +it.each(["detach", "closeTerminals"] as const)( + "does not leak a replacement stopped during launch by %s", + async (operation) => { + const paseoHome = createTemporaryHome(); + manager = await startRealWorkerManager({ paseoHome }); + const terminal = await manager.createTerminal({ + workspaceId: "ws-launch-race", + cwd: paseoHome, + ...nodeTerminalCommand("setInterval(() => {}, 1000)"), + }); + const [record] = await workerRecords(paseoHome); + process.kill(record!.pid, "SIGKILL"); + await waitForCondition(() => !manager!.getTerminal(terminal.id), 10000); + const creating = manager.createTerminal({ workspaceId: "ws-launch-race", cwd: paseoHome }); + const rejected = expect(creating).rejects.toBeInstanceOf(NoTerminalWorkerAvailableError); + await manager[operation](); + await rejected; + await waitForCondition(async () => (await workerRecords(paseoHome)).length === 0, 10000); + expect(await manager.getTerminals(paseoHome)).toEqual([]); + }, + 30000, +); + +it.each(["current", "draining"] as const)( + "keeps the peer usable when the %s worker dies", + async (target) => { + const paseoHome = createTemporaryHome(); + const old = await leaveWorkerWithTerminal(paseoHome, 0); + manager = await startRealWorkerManager({ paseoHome }); + const added = await manager.createTerminal({ + workspaceId: "ws-peer", + cwd: old.cwd, + ...nodeTerminalCommand("setInterval(() => {}, 1000)"), + }); + const current = (await workerRecords(paseoHome)).find( + (record) => record.pid !== old.workerPid, + )!; + const deadId = target === "current" ? added.id : old.terminalId; + const healthyId = target === "current" ? old.terminalId : added.id; + const healthy = manager.getTerminal(healthyId); + process.kill(target === "current" ? current.pid : old.workerPid, "SIGKILL"); + await waitForCondition(() => !manager!.getTerminal(deadId), 10000); + expect(manager.getTerminal(healthyId)).toBe(healthy); + expect(await manager.getTerminalState(healthyId)).not.toBeNull(); + await manager.createTerminal({ + workspaceId: "ws-peer", + cwd: old.cwd, + ...nodeTerminalCommand("setInterval(() => {}, 1000)"), + }); + expect((await workerRecords(paseoHome)).length).toBeLessThanOrEqual(2); + }, + 30000, +); + it("creates a terminal through the worker and streams output", async () => { const cwd = mkdtempSync(join(tmpdir(), "worker-terminal-manager-output-")); temporaryDirs.push(cwd); - manager = createWorkerTerminalManager(); + manager = await startRealWorkerManager(); const session = trackTerminal( await manager.createTerminal({ workspaceId: "ws-test", @@ -208,7 +418,7 @@ it("delivers rapid small writes complete and in order through worker coalescing" const cwd = mkdtempSync(join(tmpdir(), "worker-terminal-manager-coalesce-")); temporaryDirs.push(cwd); const burstGatePath = join(cwd, "burst-ready"); - manager = createWorkerTerminalManager(); + manager = await startRealWorkerManager(); // The file gate starts the burst after this test subscribes without depending // on platform-specific PTY input echo/canonical-mode behavior. const session = trackTerminal( @@ -271,7 +481,7 @@ it("delivers rapid small writes complete and in order through worker coalescing" it("pulls fresh terminal state from the worker authority", async () => { const cwd = mkdtempSync(join(tmpdir(), "worker-terminal-manager-state-")); temporaryDirs.push(cwd); - manager = createWorkerTerminalManager(); + manager = await startRealWorkerManager(); const session = trackTerminal( await manager.createTerminal({ workspaceId: "ws-test", @@ -302,7 +512,7 @@ it.skipIf(isPlatform("win32"))( async () => { const cwd = mkdtempSync(join(tmpdir(), "worker-terminal-manager-preamble-")); temporaryDirs.push(cwd); - manager = createWorkerTerminalManager(); + manager = await startRealWorkerManager(); // \x1b[>1u pushes kitty keyboard flag 1, which the worker's input-mode // tracker records and reflects in its replay preamble (\x1b[=1;1u). const session = trackTerminal( @@ -328,7 +538,7 @@ it.skipIf(isPlatform("win32"))( it("refreshes cached terminal title after worker title changes", async () => { const cwd = mkdtempSync(join(tmpdir(), "worker-terminal-manager-title-")); temporaryDirs.push(cwd); - manager = createWorkerTerminalManager(); + manager = await startRealWorkerManager(); const session = trackTerminal( await manager.createTerminal({ workspaceId: "ws-test", @@ -348,7 +558,7 @@ it("refreshes cached terminal title after worker title changes", async () => { it("refreshes cached terminal size after worker resize", async () => { const cwd = mkdtempSync(join(tmpdir(), "worker-terminal-manager-resize-")); temporaryDirs.push(cwd); - manager = createWorkerTerminalManager(); + manager = await startRealWorkerManager(); const session = trackTerminal(await manager.createTerminal({ cwd, workspaceId: "ws-test" })); session.send({ type: "resize", rows: 10, cols: 40 }); @@ -363,7 +573,7 @@ it("refreshes cached terminal size after worker resize", async () => { it("captures terminal output from the worker authority", async () => { const cwd = mkdtempSync(join(tmpdir(), "worker-terminal-manager-capture-")); temporaryDirs.push(cwd); - manager = createWorkerTerminalManager(); + manager = await startRealWorkerManager(); const session = trackTerminal(await manager.createTerminal({ cwd, workspaceId: "ws-test" })); session.send({ type: "input", data: "echo hello world\r" }); @@ -382,7 +592,7 @@ it("does not surface fire-and-forget send timeouts as unhandled rejections", asy const worker = new FakeTerminalWorker(); manager = createWorkerTerminalManager({ requestTimeoutMs: 5, - forkWorker: () => worker, + worker, }); worker.emitWorkerMessage({ @@ -416,7 +626,7 @@ it("does not surface fire-and-forget send timeouts as unhandled rejections", asy }); it("keeps registered cwd env inheritance behind the worker manager interface", async () => { - manager = createWorkerTerminalManager(); + manager = await startRealWorkerManager(); const cwd = mkdtempSync(join(tmpdir(), "worker-terminal-manager-env-")); temporaryDirs.push(cwd); const markerPath = join(cwd, "env.txt"); @@ -449,7 +659,7 @@ it("injects parent-minted terminal activity env through the worker", async () => temporaryDirs.push(cwd); const envPath = join(cwd, "activity-env.json"); const activityUrl = "http://127.0.0.1:12345/api/terminal-activity"; - manager = createWorkerTerminalManager({ + manager = await startRealWorkerManager({ getTerminalActivityUrl: () => activityUrl, }); @@ -496,8 +706,177 @@ it("injects parent-minted terminal activity env through the worker", async () => expect(env.path?.split(delimiter)[0]).toBe(paseoCliBinDir); }); +it("adopts a detached worker's terminals with their activity tokens and keeps them usable", async () => { + const paseoHome = createTemporaryHome(); + const cwd = mkdtempSync(join(tmpdir(), "worker-terminal-manager-adopt-")); + temporaryDirs.push(cwd); + const envPath = join(cwd, "adopt-env.json"); + const first = await startRealWorkerManager({ paseoHome }); + const created = await first.createTerminal({ + workspaceId: "ws-adopt", + cwd, + ...nodeTerminalCommand(` + require("node:fs").writeFileSync( + ${JSON.stringify(envPath)}, + JSON.stringify({ token: process.env.PASEO_ACTIVITY_TOKEN }), + ); + process.stdin.on("data", (chunk) => { + process.stdout.write("adopted-output:" + chunk.toString()); + }); + setInterval(() => {}, 1000); + `), + }); + await waitForCondition(() => existsSync(envPath), 10000); + const { token } = JSON.parse(readFileSync(envPath, "utf8")) as { token: string }; + + await first.detach(); + manager = await startRealWorkerManager({ paseoHome }); + + const adopted = manager.getTerminal(created.id); + expect(adopted?.workspaceId).toBe("ws-adopt"); + expect(await manager.getTerminals(cwd)).toHaveLength(1); + expect(manager.validateTerminalActivityToken(created.id, token)).toBe("valid"); + const messages: string[] = []; + adopted?.subscribe((message) => { + if (message.type === "output") messages.push(message.data); + }); + adopted?.send({ type: "input", data: "after-adopt\r" }); + await waitForCondition(() => messages.join("").includes("adopted-output:after-adopt"), 10000); +}); + +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +// The terminals the worker itself recorded, read from its own record file. +async function recordedTerminalPids(paseoHome: string): Promise> { + const pids = new Map(); + const directory = terminalWorkerDirectory(paseoHome); + for (const name of readdirSync(directory).filter((entry) => /^\d+\.json$/.test(entry))) { + for (const entry of (await readTerminalWorkerRecord(join(directory, name))).terminals) { + pids.set(entry.id, entry.pid); + } + } + return pids; +} + +it("never adopts a live worker's terminals after a kill cleanup it could not finish, even once the worker rewrites its record", async () => { + const paseoHome = createTemporaryHome(); + const cwd = mkdtempSync(join(tmpdir(), "worker-terminal-manager-unfinished-kill-")); + temporaryDirs.push(cwd); + const first = await startRealWorkerManager({ paseoHome }); + manager = first; + const ending = await first.createTerminal({ + workspaceId: "ws-unfinished-kill", + cwd, + ...nodeTerminalCommand("setInterval(() => {}, 1000);"), + }); + const remaining = await first.createTerminal({ + workspaceId: "ws-unfinished-kill", + cwd, + ...nodeTerminalCommand("setInterval(() => {}, 1000);"), + }); + const pids = await recordedTerminalPids(paseoHome); + const endingPid = pids.get(ending.id)!; + const remainingPid = pids.get(remaining.id)!; + + // A kill cleanup whose process table read fails signals nothing and leaves the worker live. + const inspector = createTerminalWorkerProcessInspector(); + await expect( + reapTerminalWorkers({ + paseoHome, + inspector: { ...inspector, listProcesses: () => Promise.reject(new Error("ps timed out")) }, + }), + ).rejects.toBeInstanceOf(TerminalCleanupNotConfirmedError); + await first.detach(); + expect((await readUnconfirmedTerminalCleanup(paseoHome)).length).toBeGreaterThan(0); + + // One terminal ends on its own: the live worker rewrites its record from memory. + process.kill(endingPid, "SIGKILL"); + await waitForCondition( + async () => !(await recordedTerminalPids(paseoHome)).has(ending.id), + 10000, + ); + expect((await readUnconfirmedTerminalCleanup(paseoHome)).length).toBeGreaterThan(0); + + manager = await startRealWorkerManager({ paseoHome }); + + expect(manager.getTerminal(remaining.id)).toBeUndefined(); + expect(await manager.getTerminals(cwd)).toEqual([]); + await waitForCondition(() => !isProcessAlive(remainingPid), 10000); + expect(await readUnconfirmedTerminalCleanup(paseoHome)).toEqual([]); +}); + +// The worker's own `ps`, delayed: a program that exits while the worker reads its birth time +// lands its exit in the middle of creation. Only the worker launched while this PATH is set sees +// the shim; the PATH is restored as soon as it is up. +async function startWorkerWithSlowProcessTable( + delaySeconds: number, +): Promise { + const realPs = (process.env.PATH ?? "") + .split(delimiter) + .map((dir) => join(dir, "ps")) + .find((candidate) => existsSync(candidate)); + if (!realPs) throw new Error("ps not found on PATH"); + const shimDir = createTemporaryHome(); + writeFileSync(join(shimDir, "ps"), `#!/bin/sh\nsleep ${delaySeconds}\nexec ${realPs} "$@"\n`, { + mode: 0o755, + }); + const originalPath = process.env.PATH; + process.env.PATH = `${shimDir}${delimiter}${originalPath ?? ""}`; + try { + return await startRealWorkerManager(); + } finally { + process.env.PATH = originalPath; + } +} + +async function expectCompletedShortCommand( + terminalManager: DaemonTerminalManager, + script: string, +): Promise { + const cwd = mkdtempSync(join(tmpdir(), "worker-terminal-manager-short-")); + temporaryDirs.push(cwd); + + const session = await terminalManager.createTerminal({ + workspaceId: "ws-test", + cwd, + command: "/bin/sh", + args: ["-c", script], + }); + + await waitForCondition(() => session.getExitInfo() !== null, 10000); + expect(session.getExitInfo()?.exitCode).toBe(0); + expect(terminalManager.getTerminal(session.id)).toBeUndefined(); + expect(await terminalManager.getTerminals(cwd)).toEqual([]); + expect(await terminalManager.getTerminalState(session.id)).toBeNull(); +} + +it.skipIf(isPlatform("win32"))( + "creates a short command that finishes before creation returns, with its exit", + async () => { + manager = await startRealWorkerManager(); + for (let attempt = 0; attempt < 3; attempt += 1) { + await expectCompletedShortCommand(manager, "echo short; exit 0"); + } + }, +); + +it.skipIf(isPlatform("win32"))( + "publishes the creation before an exit that arrives while the program is being recorded", + async () => { + manager = await startWorkerWithSlowProcessTable(0.3); + await expectCompletedShortCommand(manager, "sleep 0.1; echo short; exit 0"); + }, +); + it("starts the default shell through the worker and accepts quoted commands", async () => { - manager = createWorkerTerminalManager(); + manager = await startRealWorkerManager(); const cwd = mkdtempSync(join(tmpdir(), "worker-terminal-manager-shell-")); temporaryDirs.push(cwd); const markerPath = join(cwd, "shell quoted marker.txt"); @@ -518,7 +897,7 @@ it("lists subdirectory terminals when querying the workspace root", async () => const subdirCwd = join(rootCwd, "apps", "mobile"); mkdirSync(subdirCwd, { recursive: true }); temporaryDirs.push(rootCwd); - manager = createWorkerTerminalManager(); + manager = await startRealWorkerManager(); const created = trackTerminal( await manager.createTerminal({ workspaceId: "ws-test", @@ -536,7 +915,7 @@ it("lists terminals locally without waiting on the worker", async () => { const worker = new FakeTerminalWorker(); manager = createWorkerTerminalManager({ requestTimeoutMs: 5, - forkWorker: () => worker, + worker, }); worker.emitWorkerMessage({ @@ -577,7 +956,7 @@ it("includes only stamped terminals in workspace-scoped local reads", async () = const worker = new FakeTerminalWorker(); manager = createWorkerTerminalManager({ requestTimeoutMs: 5, - forkWorker: () => worker, + worker, }); worker.emitWorkerMessage({ @@ -629,7 +1008,7 @@ it("rejects non-absolute cwd in getTerminals", async () => { const worker = new FakeTerminalWorker(); manager = createWorkerTerminalManager({ requestTimeoutMs: 5, - forkWorker: () => worker, + worker, }); await expect(manager.getTerminals("relative/path")).rejects.toThrow("cwd must be absolute path"); @@ -639,7 +1018,7 @@ it("surfaces worker activity changes via getActivity, onActivityChange, and term const worker = new FakeTerminalWorker(); manager = createWorkerTerminalManager({ requestTimeoutMs: 50, - forkWorker: () => worker, + worker, }); worker.emitWorkerMessage({ @@ -702,7 +1081,7 @@ it("sets terminal activity through a worker request", async () => { const worker = new FakeTerminalWorker(); manager = createWorkerTerminalManager({ requestTimeoutMs: 50, - forkWorker: () => worker, + worker, }); worker.emitWorkerMessage({ type: "terminalCreated", @@ -735,7 +1114,7 @@ it("clears terminal attention through a worker request", async () => { const worker = new FakeTerminalWorker(); manager = createWorkerTerminalManager({ requestTimeoutMs: 50, - forkWorker: () => worker, + worker, }); worker.emitWorkerMessage({ type: "terminalCreated", @@ -766,7 +1145,7 @@ it("clears terminal attention through a worker request", async () => { it("clears finished attention on a real terminal", async () => { const cwd = mkdtempSync(join(tmpdir(), "worker-terminal-manager-attention-")); temporaryDirs.push(cwd); - manager = createWorkerTerminalManager(); + manager = await startRealWorkerManager(); const session = trackTerminal( await manager.createTerminal({ workspaceId: "ws-test", @@ -802,7 +1181,7 @@ it("clears finished attention on a real terminal", async () => { it("removes worker terminals after killAndWait", async () => { const cwd = mkdtempSync(join(tmpdir(), "worker-terminal-manager-kill-")); temporaryDirs.push(cwd); - manager = createWorkerTerminalManager(); + manager = await startRealWorkerManager(); const session = trackTerminal( await manager.createTerminal({ workspaceId: "ws-test", @@ -827,7 +1206,7 @@ it("produces one terminals-changed snapshot per title change", async () => { const worker = new FakeTerminalWorker(); manager = createWorkerTerminalManager({ requestTimeoutMs: 50, - forkWorker: () => worker, + worker, }); worker.emitWorkerMessage({ @@ -864,7 +1243,7 @@ it("produces one terminals-changed snapshot and one contribution event per activ const worker = new FakeTerminalWorker(); manager = createWorkerTerminalManager({ requestTimeoutMs: 50, - forkWorker: () => worker, + worker, }); worker.emitWorkerMessage({ @@ -914,7 +1293,7 @@ it("removes a killed worker terminal from terminalExit without duplicate snapsho const worker = new FakeTerminalWorker(); manager = createWorkerTerminalManager({ requestTimeoutMs: 50, - forkWorker: () => worker, + worker, }); const workingActivity = { state: "working" as const, changedAt: 1000 }; @@ -973,7 +1352,7 @@ it("does not resurrect a terminal that exited before its create response was han const worker = new FakeTerminalWorker(); manager = createWorkerTerminalManager({ requestTimeoutMs: 50, - forkWorker: () => worker, + worker, }); const snapshots: Array<{ cwd: string; terminalIds: string[] }> = []; @@ -1026,7 +1405,7 @@ it("replays the exit to a listener attached after a create that lost the race", const worker = new FakeTerminalWorker(); manager = createWorkerTerminalManager({ requestTimeoutMs: 50, - forkWorker: () => worker, + worker, }); const created = manager.createTerminal({ @@ -1074,7 +1453,7 @@ it("rejects createTerminal and registers nothing when the worker cannot start th const worker = new FakeTerminalWorker(); manager = createWorkerTerminalManager({ requestTimeoutMs: 50, - forkWorker: () => worker, + worker, }); const snapshots: Array<{ cwd: string; terminalIds: string[] }> = []; @@ -1104,3 +1483,256 @@ it("rejects createTerminal and registers nothing when the worker cannot start th expect(await manager.getTerminals("/workspace")).toEqual([]); expect(snapshots).toEqual([]); }); + +// The live worker records in a home, by worker pid. +async function workerRecords(paseoHome: string): Promise { + return (await listTerminalWorkerRecords(paseoHome)).flatMap((file) => + "record" in file && !file.record.closing ? [file.record] : [], + ); +} + +// A worker launched while the override is set reports that protocol version; the variable is +// restored as soon as it is up, so later workers report the build's own. +async function startManagerWithWorkerProtocol( + paseoHome: string, + protocolVersion: number, +): Promise { + const previous = process.env[TERMINAL_WORKER_PROTOCOL_OVERRIDE_ENV]; + process.env[TERMINAL_WORKER_PROTOCOL_OVERRIDE_ENV] = String(protocolVersion); + try { + return await startRealWorkerManager({ paseoHome }); + } finally { + if (previous === undefined) delete process.env[TERMINAL_WORKER_PROTOCOL_OVERRIDE_ENV]; + else process.env[TERMINAL_WORKER_PROTOCOL_OVERRIDE_ENV] = previous; + } +} + +// A terminal running an idle program in a worker that reports `protocolVersion`, left running +// after its daemon lets go (stop --keep-terminals). +async function leaveWorkerWithTerminal( + paseoHome: string, + protocolVersion: number, +): Promise<{ terminalId: string; programPid: number; workerPid: number; cwd: string }> { + const cwd = mkdtempSync(join(tmpdir(), "worker-terminal-manager-protocol-")); + temporaryDirs.push(cwd); + const previous = await startManagerWithWorkerProtocol(paseoHome, protocolVersion); + const session = await previous.createTerminal({ + workspaceId: "ws-protocol", + cwd, + ...nodeTerminalCommand("setInterval(() => {}, 1000);"), + }); + const [record] = await workerRecords(paseoHome); + await previous.detach(); + return { + terminalId: session.id, + programPid: (await recordedTerminalPids(paseoHome)).get(session.id)!, + workerPid: record!.pid, + cwd, + }; +} + +it("drains a worker one protocol older: it keeps its terminal, new ones open in a second worker, and it exits after its last", async () => { + const paseoHome = createTemporaryHome(); + const old = await leaveWorkerWithTerminal(paseoHome, 0); + + manager = await startRealWorkerManager({ paseoHome }); + + expect(manager.getTerminal(old.terminalId)).toBeDefined(); + const workers = await workerRecords(paseoHome); + expect(workers).toHaveLength(2); + const current = workers.find((record) => record.pid !== old.workerPid)!; + expect(current.protocolVersion).toBe(1); + const added = await manager.createTerminal({ + workspaceId: "ws-protocol", + cwd: old.cwd, + ...nodeTerminalCommand("setInterval(() => {}, 1000);"), + }); + trackTerminal(added); + const currentTerminals = ( + await readTerminalWorkerRecord(join(terminalWorkerDirectory(paseoHome), `${current.pid}.json`)) + ).terminals.map((entry) => entry.id); + expect(currentTerminals).toEqual([added.id]); + expect((await manager.getTerminals(old.cwd)).map((session) => session.id).sort()).toEqual( + [old.terminalId, added.id].sort(), + ); + + await manager.killTerminalAndWait(old.terminalId); + + await waitForCondition(() => !isProcessAlive(old.workerPid), 10000); + expect((await workerRecords(paseoHome)).map((record) => record.pid)).toEqual([current.pid]); + expect(manager.getTerminal(added.id)).toBeDefined(); +}); + +it("shuts down a worker two protocol versions old, closing its terminals", async () => { + const paseoHome = createTemporaryHome(); + const old = await leaveWorkerWithTerminal(paseoHome, -1); + + manager = await startRealWorkerManager({ paseoHome }); + + await waitForCondition( + () => !isProcessAlive(old.workerPid) && !isProcessAlive(old.programPid), + 10000, + ); + expect(manager.getTerminal(old.terminalId)).toBeUndefined(); + const workers = await workerRecords(paseoHome); + expect(workers).toHaveLength(1); + expect(workers[0]!.protocolVersion).toBe(1); +}); + +it("leaves a worker from a newer Paseo running, reports it incompatible, and closes it on a kill stop", async () => { + const paseoHome = createTemporaryHome(); + const newer = await leaveWorkerWithTerminal(paseoHome, 2); + + manager = await startRealWorkerManager({ paseoHome }); + + const unavailable = manager.getUnavailableError?.(); + expect(unavailable).toBeInstanceOf(TerminalWorkerIncompatibleError); + expect(unavailable?.message).toContain("worker protocol 2"); + await expect(manager.getTerminals(newer.cwd)).rejects.toBeInstanceOf( + TerminalWorkerIncompatibleError, + ); + await expect( + manager.createTerminal({ workspaceId: "ws-protocol", cwd: newer.cwd }), + ).rejects.toBeInstanceOf(TerminalWorkerIncompatibleError); + expect(isProcessAlive(newer.programPid)).toBe(true); + expect((await workerRecords(paseoHome)).map((record) => record.pid)).toEqual([newer.workerPid]); + + await manager.closeTerminals(); + + expect(isProcessAlive(newer.workerPid)).toBe(false); + expect(isProcessAlive(newer.programPid)).toBe(false); + expect(await workerRecords(paseoHome)).toEqual([]); +}); + +// A worker launched on its own, as two daemons that each started one would leave it: running an +// idle program in a terminal, with no daemon attached. +async function launchDetachedWorkerWithTerminal( + paseoHome: string, + protocolVersion?: number, +): Promise<{ terminalId: string; workerPid: number; cwd: string }> { + const cwd = mkdtempSync(join(tmpdir(), "worker-terminal-manager-detached-")); + temporaryDirs.push(cwd); + const previous = process.env[TERMINAL_WORKER_PROTOCOL_OVERRIDE_ENV]; + if (protocolVersion !== undefined) { + process.env[TERMINAL_WORKER_PROTOCOL_OVERRIDE_ENV] = String(protocolVersion); + } + let launched: Awaited>; + try { + launched = await launchTerminalWorker({ paseoHome, paseoVersion: "0.0.0-test" }); + } finally { + if (previous === undefined) delete process.env[TERMINAL_WORKER_PROTOCOL_OVERRIDE_ENV]; + else process.env[TERMINAL_WORKER_PROTOCOL_OVERRIDE_ENV] = previous; + } + const worker = createWorkerTerminalManager({ + worker: await connectTerminalWorker({ + socketPath: launched.record.socketPath, + secret: launched.record.secret, + }), + }); + const session = await worker.createTerminal({ + workspaceId: "ws-detached", + cwd, + ...nodeTerminalCommand("setInterval(() => {}, 1000);"), + }); + await worker.detach(); + return { terminalId: session.id, workerPid: launched.record.pid, cwd }; +} + +// Worker pids alive in a home, from every record, closing ones included. +async function liveWorkerPids(paseoHome: string): Promise { + return (await listTerminalWorkerRecords(paseoHome)) + .flatMap((file) => ("record" in file ? [file.record.pid] : [])) + .filter(isProcessAlive) + .sort(); +} + +it.skipIf(isPlatform("win32"))( + "counts live workers whose cleanup could not be confirmed and never starts a third", + async () => { + const paseoHome = createTemporaryHome(); + const first = await launchDetachedWorkerWithTerminal(paseoHome); + const second = await launchDetachedWorkerWithTerminal(paseoHome); + // A kill cleanup that cannot read the process table signals nothing: both workers keep + // running with their records marked closing. + const inspector = createTerminalWorkerProcessInspector(); + await expect( + reapTerminalWorkers({ + paseoHome, + gracefulMs: 0, + inspector: { ...inspector, listProcesses: () => Promise.reject(new Error("ps timed out")) }, + }), + ).rejects.toBeInstanceOf(TerminalCleanupNotConfirmedError); + expect((await readUnconfirmedTerminalCleanup(paseoHome)).map((entry) => entry.role)).toContain( + "worker", + ); + + // The daemon's own retry fails the same way: only the full process table is unreadable. + const realPs = (process.env.PATH ?? "") + .split(delimiter) + .map((dir) => join(dir, "ps")) + .find((candidate) => existsSync(candidate)); + if (!realPs) throw new Error("ps not found on PATH"); + const shimDir = createTemporaryHome(); + writeFileSync( + join(shimDir, "ps"), + `#!/bin/sh\nif [ "$1" = "-A" ]; then echo "process table unavailable" >&2; exit 1; fi\nexec ${realPs} "$@"\n`, + { mode: 0o755 }, + ); + const originalPath = process.env.PATH; + process.env.PATH = `${shimDir}${delimiter}${originalPath ?? ""}`; + try { + manager = await startRealWorkerManager({ paseoHome }); + } finally { + process.env.PATH = originalPath; + } + + expect(await liveWorkerPids(paseoHome)).toEqual([first.workerPid, second.workerPid].sort()); + await expect( + manager.createTerminal({ workspaceId: "ws-detached", cwd: first.cwd }), + ).rejects.toThrow("2 terminal workers are already running"); + expect(await liveWorkerPids(paseoHome)).toEqual([first.workerPid, second.workerPid].sort()); + }, +); + +it("opens new terminals in a new worker once one of two draining workers exits", async () => { + const paseoHome = createTemporaryHome(); + const first = await launchDetachedWorkerWithTerminal(paseoHome, 0); + const second = await launchDetachedWorkerWithTerminal(paseoHome, 0); + + manager = await startRealWorkerManager({ paseoHome }); + + expect(await liveWorkerPids(paseoHome)).toEqual([first.workerPid, second.workerPid].sort()); + await expect( + manager.createTerminal({ workspaceId: "ws-detached", cwd: first.cwd }), + ).rejects.toBeInstanceOf(NoTerminalWorkerAvailableError); + + await manager.killTerminalAndWait(first.terminalId); + await waitForCondition(() => !isProcessAlive(first.workerPid), 10000); + + // Concurrent creates start one worker between them. + const created = await Promise.all( + [0, 1].map(() => + manager!.createTerminal({ + workspaceId: "ws-detached", + cwd: first.cwd, + ...nodeTerminalCommand("setInterval(() => {}, 1000);"), + }), + ), + ); + created.forEach(trackTerminal); + const workers = await workerRecords(paseoHome); + expect(workers.map((record) => record.pid).sort()).toEqual(await liveWorkerPids(paseoHome)); + expect(workers).toHaveLength(2); + const started = workers.find((record) => record.pid !== second.workerPid)!; + expect(started.protocolVersion).toBe(1); + expect( + ( + await readTerminalWorkerRecord( + join(terminalWorkerDirectory(paseoHome), `${started.pid}.json`), + ) + ).terminals + .map((entry) => entry.id) + .sort(), + ).toEqual(created.map((session) => session.id).sort()); + expect(manager.getTerminal(second.terminalId)).toBeDefined(); +}); diff --git a/packages/server/src/terminal/worker-terminal-manager.ts b/packages/server/src/terminal/worker-terminal-manager.ts index 0a1adb2c44..c9a6d7186e 100644 --- a/packages/server/src/terminal/worker-terminal-manager.ts +++ b/packages/server/src/terminal/worker-terminal-manager.ts @@ -1,7 +1,10 @@ -import { fileURLToPath } from "node:url"; -import { fork } from "node:child_process"; import { randomBytes, randomUUID } from "node:crypto"; +import { access } from "node:fs/promises"; +import { setTimeout as delay } from "node:timers/promises"; +import type { Logger } from "pino"; +import { createExternalProcessEnv } from "../server/paseo-env.js"; import { assertAbsolutePath, isSameOrDescendantPath } from "../server/path-utils.js"; +import { writtenDuringThisBoot } from "../server/pid-lock.js"; import type { TerminalState } from "@getpaseo/protocol/messages"; import type { TerminalActivity, TerminalActivityState } from "@getpaseo/protocol/terminal-activity"; import { deriveTerminalActivityStatusBucket } from "@getpaseo/protocol/terminal-activity"; @@ -14,7 +17,28 @@ import type { TerminalSession, TerminalStateSnapshot, } from "./terminal.js"; +import { resolvePaseoCliExecutablePath } from "./paseo-cli-path.js"; import type { CaptureTerminalLinesResult } from "./terminal-capture.js"; +import { connectTerminalWorker, type TerminalWorkerConnection } from "./terminal-worker-client.js"; +import { launchTerminalWorker } from "./terminal-worker-launch.js"; +import { classifyTerminalWorkerProcess } from "./terminal-worker-ownership.js"; +import { + createTerminalWorkerProcessInspector, + type TerminalWorkerProcessInspector, +} from "./terminal-worker-processes.js"; +import { reapTerminalWorkers } from "./terminal-worker-reaper.js"; +import { + listTerminalWorkerRecords, + terminalWorkerRecordPath, + type TerminalWorkerRecord, +} from "./terminal-worker-record.js"; +import { + createTerminalWorkerPool, + NoTerminalWorkerAvailableError, + TerminalWorkerIncompatibleError, + type TerminalWorkerPoolMember, +} from "./terminal-worker-pool.js"; +import { pruneTerminalWorkerRuntimes } from "./terminal-worker-runtime.js"; import type { TerminalActivityListener, TerminalActivityTransitionEvent, @@ -25,7 +49,12 @@ import type { TerminalsChangedEvent, TerminalsChangedListener, } from "./terminal-manager.js"; +import { + classifyTerminalWorkerProtocol, + type TerminalWorkerProtocolFit, +} from "./terminal-worker-protocol.js"; import type { + TerminalWorkerListResult, TerminalWorkerRequest, TerminalWorkerResponse, TerminalWorkerToParentMessage, @@ -78,49 +107,45 @@ interface WorkerTerminalRecord { session: TerminalSession; } -interface TerminalWorkerProcess { - connected: boolean; - killed: boolean; - send(message: TerminalWorkerRequest, callback: (error: Error | null) => void): boolean; - disconnect(): void; - kill(): boolean; - on(event: "message", listener: (message: TerminalWorkerToParentMessage) => void): this; - on(event: "exit", listener: (code: number | null, signal: NodeJS.Signals | null) => void): this; +export interface TerminalCreateEnvironment { + baseEnv: Record; + paseoCliPath: string | null; } interface WorkerTerminalManagerOptions { + worker: TerminalWorkerConnection; + /** Reconnects a surviving worker; null means it is confirmed gone or the owner stopped. */ + reconnect?: () => Promise; requestTimeoutMs?: number; - forkWorker?: () => TerminalWorkerProcess; getTerminalActivityUrl?: () => string | null; + resolveCreateEnvironment?: () => TerminalCreateEnvironment; + // The kill path. The daemon passes the reaper, which verifies every program is gone; the + // default asks the connected worker to shut down. + closeTerminals?: () => Promise; +} + +/** The daemon's terminal manager: its terminals live in a worker that can outlive it. */ +export interface DaemonTerminalManager extends TerminalManager { + /** Registers every terminal the connected worker already runs. Resolves with their count. */ + adoptTerminals(): Promise; + /** Kills every terminal and verifies their programs ended. */ + closeTerminals(): Promise; + /** + * Lets go of the worker; its terminals keep running for the next daemon. With + * `afterPendingRequests`, requests already sent get their answers first. + */ + detach(options?: { afterPendingRequests?: boolean }): Promise; } function createActivityToken(): string { return randomBytes(32).toString("base64url"); } -function resolveWorkerUrl(): URL { - const currentUrl = import.meta.url; - if (currentUrl.endsWith(".ts")) { - return new URL("./terminal-worker-process.ts", currentUrl); - } - return new URL("./terminal-worker-process.js", currentUrl); -} - -function resolveWorkerExecArgv(): string[] { - if (!import.meta.url.endsWith(".ts")) { - return []; - } - const loaderUrl = new URL("./terminal-ts-loader.mjs", import.meta.url).href; - const importSource = [ - 'import { register } from "node:module";', - 'import { pathToFileURL } from "node:url";', - `register(${JSON.stringify(loaderUrl)}, pathToFileURL("./"));`, - ].join(" "); - return [ - "--experimental-strip-types", - "--import", - `data:text/javascript,${encodeURIComponent(importSource)}`, - ]; +function resolveDaemonTerminalCreateEnvironment(): TerminalCreateEnvironment { + return { + baseEnv: createExternalProcessEnv(process.env), + paseoCliPath: resolvePaseoCliExecutablePath(), + }; } function isResponse(message: TerminalWorkerToParentMessage): message is TerminalWorkerResponse { @@ -138,20 +163,17 @@ function cloneTerminalInfo(info: RequiredWorkerTerminalInfo): RequiredWorkerTerm }; } -function forkTerminalWorker(): TerminalWorkerProcess { - return fork(fileURLToPath(resolveWorkerUrl()), [], { - execArgv: resolveWorkerExecArgv(), - serialization: "advanced", - stdio: ["ignore", "ignore", "inherit", "ipc"], - }) as TerminalWorkerProcess; -} - export function createWorkerTerminalManager( - managerOptions: WorkerTerminalManagerOptions = {}, -): TerminalManager { - const worker = managerOptions.forkWorker ? managerOptions.forkWorker() : forkTerminalWorker(); + managerOptions: WorkerTerminalManagerOptions, +): DaemonTerminalManager { + let worker = managerOptions.worker; + let detached = false; + let reconnecting: Promise | null = null; const requestTimeoutMs = managerOptions.requestTimeoutMs ?? REQUEST_TIMEOUT_MS; + const resolveCreateEnvironment = + managerOptions.resolveCreateEnvironment ?? resolveDaemonTerminalCreateEnvironment; const pendingRequests = new Map(); + const inFlight = new Set>(); const recordsById = new Map(); const terminalIdsByCwd = new Map>(); const terminalActivityTokenById = new Map(); @@ -162,8 +184,8 @@ export function createWorkerTerminalManager( const terminalActivityListeners = new Set(); const terminalWorkspaceContributionChangedListeners = new Set(); - let workerExited = false; - let workerShutdownTimer: ReturnType | null = null; + let workerDisconnected = false; + let closePromise: Promise | null = null; function emitTerminalsChanged(event: TerminalsChangedEvent): void { for (const listener of Array.from(terminalsChangedListeners)) { @@ -587,40 +609,77 @@ export function createWorkerTerminalManager( } } - worker.on("message", (message: TerminalWorkerToParentMessage) => { - if (isResponse(message)) { - const pending = pendingRequests.get(message.requestId); - if (!pending) { + function attachWorker(): void { + worker.on("message", (message: TerminalWorkerToParentMessage) => { + if (isResponse(message)) { + const pending = pendingRequests.get(message.requestId); + if (!pending) { + return; + } + clearTimeout(pending.timeout); + pendingRequests.delete(message.requestId); + if (message.ok) { + pending.resolve(message.result); + } else { + pending.reject(new Error(message.error)); + } return; } - clearTimeout(pending.timeout); - pendingRequests.delete(message.requestId); - if (message.ok) { - pending.resolve(message.result); - } else { - pending.reject(new Error(message.error)); - } - return; - } - handleWorkerEvent(message); - }); + handleWorkerEvent(message); + }); - worker.on("exit", (code, signal) => { - workerExited = true; - if (workerShutdownTimer) { - clearTimeout(workerShutdownTimer); - workerShutdownTimer = null; - } - rejectPendingRequests(new Error(`Terminal worker exited (${signal ?? code ?? "unknown"})`)); - }); + worker.on("close", () => { + workerDisconnected = true; + rejectPendingRequests(new Error("Terminal worker connection closed")); + const reconnect = managerOptions.reconnect; + if (!detached && reconnect) { + reconnecting = (async () => { + const connection = await reconnect(); + if (!connection) { + for (const terminalId of Array.from(recordsById.keys())) { + handleTerminalExitEvent({ + type: "terminalExit", + terminalId, + info: { exitCode: null, signal: null, lastOutputLines: [] }, + }); + } + return; + } + if (detached) { + connection.disconnect(); + return; + } + worker = connection; + workerDisconnected = false; + attachWorker(); + await adoptTerminals(); + // Output emitted without a controller is retained in the worker's screen. Existing + // streams must fetch that screen before applying further live output. + for (const record of recordsById.values()) { + handleTerminalMessageEvent({ + type: "terminalMessage", + terminalId: record.info.id, + message: { type: "snapshotReady" }, + replayPreamble: record.replayPreamble, + }); + } + })().finally(() => { + reconnecting = null; + }); + void reconnecting.catch(() => {}); + } + }); + } + attachWorker(); function sendRequest(input: TerminalWorkerRequestInput): Promise { - if (workerExited || !worker.connected) { + if (reconnecting && workerDisconnected) return reconnecting.then(() => sendRequest(input)); + if (workerDisconnected || !worker.connected) { return Promise.reject(new Error("Terminal worker is not running")); } const requestId = randomUUID(); const message = { ...input, requestId } as TerminalWorkerRequest; - return new Promise((resolve, reject) => { + const request = new Promise((resolve, reject) => { const timeout = setTimeout(() => { pendingRequests.delete(requestId); reject(new Error(`Terminal worker request timed out: ${input.type}`)); @@ -635,6 +694,14 @@ export function createWorkerTerminalManager( reject(error); }); }); + // Tracks settlement only; the caller handles the rejection. + const settled = request.then( + () => undefined, + () => undefined, + ); + inFlight.add(settled); + void settled.finally(() => inFlight.delete(settled)); + return request; } function sendBestEffortRequest(input: TerminalWorkerRequestInput): void { @@ -645,6 +712,71 @@ export function createWorkerTerminalManager( }); } + function clearRecords(): void { + for (const terminalId of Array.from(recordsById.keys())) { + removeRecord(terminalId); + } + } + + function disconnectWorker(): Promise { + if (workerDisconnected || !worker.connected) { + return Promise.resolve(); + } + return new Promise((resolve) => { + worker.on("close", () => resolve()); + worker.disconnect(); + }); + } + + function closeTerminals(): Promise { + detached = true; + closePromise ??= (async () => { + await reconnecting; + if (managerOptions.closeTerminals) { + await disconnectWorker(); + clearRecords(); + await managerOptions.closeTerminals(); + return; + } + // Unverified: production passes the reaper, which confirms every program ended. + sendBestEffortRequest({ type: "shutdown" }); + await disconnectWorker(); + clearRecords(); + })(); + return closePromise; + } + + async function adoptTerminals(): Promise { + const listed = (await sendRequest({ type: "listTerminals" })) as TerminalWorkerListResult; + const cwds = new Set(); + const liveIds = new Set(listed.map((item) => item.terminal.id)); + for (const terminalId of Array.from(recordsById.keys())) { + if (!liveIds.has(terminalId)) + handleTerminalExitEvent({ + type: "terminalExit", + terminalId, + info: { exitCode: null, signal: null, lastOutputLines: [] }, + }); + } + for (const item of listed) { + const info = asRequiredWorkerTerminalInfo(item.terminal); + registerRecord({ info, state: item.state }); + const record = recordsById.get(info.id); + if (record) { + record.replayPreamble = item.replayPreamble; + } + // The agent hooks inside each shell keep the token they were started with. + if (item.activityToken) { + terminalActivityTokenById.set(info.id, item.activityToken); + } + cwds.add(info.cwd); + } + for (const cwd of cwds) { + emitTerminalsChanged({ cwd, terminals: listTerminalItemsForCwd(cwd) }); + } + return listed.length; + } + return { async getTerminals( cwd: string, @@ -688,6 +820,7 @@ export function createWorkerTerminalManager( const creation = { session: null as TerminalSession | null }; pendingCreationsById.set(terminalId, creation); try { + const environment = resolveCreateEnvironment(); await sendRequest({ type: "createTerminal", options: { @@ -695,6 +828,8 @@ export function createWorkerTerminalManager( id: terminalId, activityToken, activityUrl: terminalActivityUrl, + baseEnv: environment.baseEnv, + paseoCliPath: environment.paseoCliPath, }, }); if (!creation.session) { @@ -812,25 +947,25 @@ export function createWorkerTerminalManager( }, killAll(): void { - void sendRequest({ type: "killAll" }) - .catch(() => { - // no-op - }) - .finally(() => { - if (worker.connected) { - worker.disconnect(); - } - if (!worker.killed && !workerShutdownTimer) { - workerShutdownTimer = setTimeout(() => { - worker.kill(); - }, 1000); - } - }); - for (const terminalId of Array.from(recordsById.keys())) { - removeRecord(terminalId); - } + void closeTerminals().catch(() => { + // The synchronous interface cannot report it; closeTerminals() callers see the error. + }); }, + closeTerminals, + + async detach(options?: { afterPendingRequests?: boolean }): Promise { + detached = true; + await reconnecting; + if (options?.afterPendingRequests) { + await Promise.allSettled(Array.from(inFlight)); + } + await disconnectWorker(); + clearRecords(); + }, + + adoptTerminals, + subscribeTerminalsChanged(listener: TerminalsChangedListener): () => void { terminalsChangedListeners.add(listener); return () => { @@ -856,6 +991,342 @@ export function createWorkerTerminalManager( }; } -export function terminateWorkerTerminalManager(manager: TerminalManager): void { - manager.killAll(); +export interface StartWorkerTerminalManagerOptions { + paseoHome: string; + paseoVersion: string; + logger: Logger; + getTerminalActivityUrl?: () => string | null; + requestTimeoutMs?: number; +} + +// The current worker plus one draining a previous protocol. Every live worker counts, +// incompatible, unreachable, unidentified and closing ones included: the daemon never starts a +// third. +const MAX_TERMINAL_WORKERS = 2; +const DRAINED_WORKER_EXIT_TIMEOUT_MS = 10_000; + +interface LiveWorkerRecord { + path: string; + record: TerminalWorkerRecord; + fit: TerminalWorkerProtocolFit; +} + +/** + * Connects the daemon to its terminal workers. A worker of this daemon's protocol is adopted + * (restart, crash, `stop --keep-terminals`); one a protocol older is adopted to drain; anything + * older is shut down, its terminals closed; one from a newer Paseo is left running and the + * terminal service reports it as incompatible. A worker is launched only when none is current, + * none is incompatible and fewer than two run; when two run, the first terminal created after one + * of them exits launches it. Records that cannot be adopted (a cleanup that + * never finished, a dead worker whose programs survived it) go through the reaper first, so a + * daemon never adopts terminals someone asked to close. + */ +export async function startWorkerTerminalManager( + options: StartWorkerTerminalManagerOptions, +): Promise { + const logger = options.logger.child({ module: "terminal-worker" }); + const inspector = createTerminalWorkerProcessInspector(); + const log = (message: string, fields?: Record) => logger.info(fields, message); + const live: LiveWorkerRecord[] = []; + const toReap: string[] = []; + for (const file of await listTerminalWorkerRecords(options.paseoHome)) { + if ("error" in file) { + logger.warn({ recordPath: file.path, err: file.error }, "Unreadable terminal worker record"); + continue; + } + const { record } = file; + if ( + record.closing || + !writtenDuringThisBoot({ startedAt: record.createdAt, bootId: record.bootId }) + ) { + toReap.push(file.path); + continue; + } + const worker = classifyTerminalWorkerProcess(record, await inspector.inspect(record.pid)); + if (worker.status === "owned") { + live.push({ + path: file.path, + record, + fit: classifyTerminalWorkerProtocol(record.protocolVersion), + }); + } else if (worker.status === "unknown") { + logger.warn( + { recordPath: file.path, pid: record.pid, reason: worker.reason }, + "Cannot identify terminal worker; leaving its record", + ); + } else { + // Its programs lost their PTY with the worker and cannot be adopted. + toReap.push(file.path); + } + } + + if (toReap.length > 0) { + await reapTerminalWorkers({ + paseoHome: options.paseoHome, + gracefulMs: 0, + recordPaths: toReap, + inspector, + log, + }).catch((error: unknown) => { + logger.warn({ err: error }, "Terminal cleanup of an unadoptable worker was not confirmed"); + }); + } + + const obsolete = live.filter((entry) => entry.fit === "obsolete"); + if (obsolete.length > 0) { + for (const entry of obsolete) { + logger.warn( + { + pid: entry.record.pid, + protocolVersion: entry.record.protocolVersion, + paseoVersion: entry.record.paseoVersion, + terminals: entry.record.terminals.length, + }, + "Closing a terminal worker two or more protocol versions old; its terminals close", + ); + } + await reapTerminalWorkers({ + paseoHome: options.paseoHome, + recordPaths: obsolete.map((entry) => entry.path), + inspector, + log, + }).catch((error: unknown) => { + logger.warn({ err: error }, "Closing an obsolete terminal worker was not confirmed"); + }); + } + + const closeAll = async () => { + const result = await reapTerminalWorkers({ paseoHome: options.paseoHome, inspector, log }); + logger.info(result, "Terminals closed"); + }; + const prune = async () => { + const result = await pruneTerminalWorkerRuntimes(options.paseoHome); + if (result.removed.length > 0) + log("Removed terminal worker copies", { removed: result.removed }); + for (const failure of result.failed) { + logger.warn(failure, "Cannot remove a terminal worker copy; the next start retries"); + } + }; + + const pruneAfterExit = async (workerPid: number) => { + if (await waitForWorkerRecordRemoval(options.paseoHome, workerPid)) { + await prune(); + return; + } + logger.warn({ pid: workerPid }, "Drained terminal worker has not exited yet"); + }; + + const newer = live.find((entry) => entry.fit === "newer"); + if (newer) { + const incompatible = new TerminalWorkerIncompatibleError({ + paseoVersion: newer.record.paseoVersion, + protocolVersion: newer.record.protocolVersion, + }); + logger.error({ pid: newer.record.pid }, incompatible.message); + await prune(); + return createTerminalWorkerPool({ + current: null, + draining: [], + incompatible, + closeAll, + }); + } + + const createMember = async ( + record: TerminalWorkerRecord, + connection: TerminalWorkerConnection, + ): Promise => { + let alive = true; + let resolveExit!: () => void; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + let recovery: Promise | null = null; + const recoveryState = { stopped: false }; + const reconnect = () => { + recovery ??= (async () => { + while (!recoveryState.stopped) { + const ownership = classifyTerminalWorkerProcess( + record, + await inspector.inspect(record.pid), + ); + if (ownership.status === "gone" || ownership.status === "unrelated") { + await reapTerminalWorkers({ + paseoHome: options.paseoHome, + recordPaths: [terminalWorkerRecordPath(options.paseoHome, record.pid)], + inspector, + log, + gracefulMs: 0, + }).catch((error) => + logger.warn({ err: error }, "Dead terminal worker cleanup was not confirmed"), + ); + alive = false; + resolveExit(); + return null; + } + try { + return await connectTerminalWorker({ + socketPath: record.socketPath, + secret: record.secret, + timeoutMs: 500, + }); + } catch { + await delay(100); + } + } + return null; + })().finally(() => { + recovery = null; + }); + return recovery; + }; + const manager = createWorkerTerminalManager({ + worker: connection, + reconnect, + ...(options.requestTimeoutMs === undefined + ? {} + : { requestTimeoutMs: options.requestTimeoutMs }), + ...(options.getTerminalActivityUrl + ? { getTerminalActivityUrl: options.getTerminalActivityUrl } + : {}), + closeTerminals: closeAll, + }); + const count = await manager.adoptTerminals(); + if (count > 0) { + logger.info({ pid: record.pid, terminals: count }, "Adopted terminals from a running worker"); + } + return { + manager, + workerPid: record.pid, + exited, + ready: async () => { + await recovery; + return alive && !recoveryState.stopped; + }, + stopRecovery: () => { + recoveryState.stopped = true; + }, + }; + }; + + let current: TerminalWorkerPoolMember | null = null; + const draining: TerminalWorkerPoolMember[] = []; + const adoptable = [ + ...live.filter((entry) => entry.fit === "current"), + ...live.filter((entry) => entry.fit === "previous"), + ]; + for (const entry of adoptable) { + let connection: TerminalWorkerConnection; + try { + connection = await connectTerminalWorker({ + socketPath: entry.record.socketPath, + secret: entry.record.secret, + }); + } catch (error) { + logger.warn( + { pid: entry.record.pid, err: error }, + "Cannot reach terminal worker; leaving it running", + ); + continue; + } + const member = await createMember(entry.record, connection); + if (entry.fit === "current" && !current) { + logger.info({ pid: entry.record.pid }, "Adopting running terminal worker"); + current = member; + } else { + // A previous protocol, or a second worker of this one: it keeps its terminals and takes + // no new ones. + logger.info( + { pid: entry.record.pid, protocolVersion: entry.record.protocolVersion }, + "Draining terminal worker; it exits after its last terminal", + ); + draining.push(member); + } + } + + // Counted from the records rather than from what startup drove: a worker a cleanup could not + // confirm gone keeps running, and a draining one exits on its own. + const startCurrent = async (): Promise => { + const running = await countRunningTerminalWorkers(options.paseoHome, inspector); + if (running >= MAX_TERMINAL_WORKERS) { + throw new NoTerminalWorkerAvailableError( + `${running} terminal workers are already running in ${options.paseoHome}; new terminals can open once one of them exits`, + ); + } + const launched = await launchTerminalWorker({ + paseoHome: options.paseoHome, + paseoVersion: options.paseoVersion, + }); + const connection = await connectTerminalWorker({ + socketPath: launched.record.socketPath, + secret: launched.record.secret, + }); + logger.info( + { pid: launched.record.pid, runtime: launched.record.runtime ?? null }, + "Launched terminal worker", + ); + return createMember(launched.record, connection); + }; + + await prune(); + if (!current) { + try { + current = await startCurrent(); + } catch (error) { + if (!(error instanceof NoTerminalWorkerAvailableError)) throw error; + logger.warn(error.message); + } + } + + return createTerminalWorkerPool({ + current, + draining, + incompatible: null, + startCurrent, + closeAll, + onDrained: (member) => { + log("Released drained terminal worker", { pid: member.workerPid }); + void pruneAfterExit(member.workerPid); + }, + }); +} + +// Every worker that may still run in this home, one per record: a worker and its cleanup record +// are listed as one. Unreadable records and workers that cannot be identified count; a worker +// confirmed gone, or a pid reused by another process, does not. +async function countRunningTerminalWorkers( + paseoHome: string, + inspector: TerminalWorkerProcessInspector, +): Promise { + let running = 0; + for (const file of await listTerminalWorkerRecords(paseoHome)) { + if ("error" in file) { + running += 1; + continue; + } + const { record } = file; + if (!writtenDuringThisBoot({ startedAt: record.createdAt, bootId: record.bootId })) continue; + const worker = classifyTerminalWorkerProcess(record, await inspector.inspect(record.pid)); + if (worker.status === "owned" || worker.status === "unknown") running += 1; + } + return running; +} + +// A worker deletes its own record as it exits. +async function waitForWorkerRecordRemoval(paseoHome: string, workerPid: number): Promise { + const recordPath = terminalWorkerRecordPath(paseoHome, workerPid); + const deadline = Date.now() + DRAINED_WORKER_EXIT_TIMEOUT_MS; + while ( + await access(recordPath).then( + () => true, + (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return false; + throw error; + }, + ) + ) { + if (Date.now() >= deadline) return false; + await delay(100); + } + return true; } diff --git a/packages/server/tsconfig.scripts.json b/packages/server/tsconfig.scripts.json index cc8e24efd4..6a75825b00 100644 --- a/packages/server/tsconfig.scripts.json +++ b/packages/server/tsconfig.scripts.json @@ -19,6 +19,11 @@ "declaration": false, "sourceMap": true }, - "include": ["scripts/supervisor-entrypoint.ts", "scripts/dev-runner.ts", "scripts/supervisor.ts"], + "include": [ + "scripts/supervisor-entrypoint.ts", + "scripts/dev-runner.ts", + "scripts/supervisor.ts", + "src/executable-resolution/which.d.ts" + ], "exclude": ["node_modules", "dist"] } diff --git a/packages/server/tsconfig.server.json b/packages/server/tsconfig.server.json index 9c868b42df..1c7a5c3579 100644 --- a/packages/server/tsconfig.server.json +++ b/packages/server/tsconfig.server.json @@ -22,7 +22,12 @@ "declarationMap": true, "sourceMap": true }, - "include": ["src/server/**/*", "src/services/**/*", "src/terminal/**/*"], + "include": [ + "src/server/**/*", + "src/services/**/*", + "src/terminal/**/*", + "src/executable-resolution/which.d.ts" + ], "exclude": [ "node_modules", "dist",