From 293fbf825d3dc24d809fa796be0d15f2cc0d1e54 Mon Sep 17 00:00:00 2001 From: Josh Bendavid Date: Mon, 16 Nov 2026 04:56:04 +0100 Subject: [PATCH] fix(server): fall back to the next socket directory when the bind is refused (#9214) A PASEO_HOME on AFS passes the socket directory checks, then refuses the Unix socket bind with EPERM, and the terminal worker failed to start. The worker now binds each eligible candidate in turn and moves on when the filesystem refuses the socket (EPERM, EACCES, EOPNOTSUPP, ENOTSUP). Other listen errors still fail the start. Fixes #9213 Co-authored-by: Claude Opus 5.5 (cherry picked from commit ade3817a45f05325fd5929ba939de220bd90c236) --- docs/terminal-worker.md | 1 + .../src/terminal/terminal-worker-process.ts | 30 +++--- .../terminal-worker-socket-path.test.ts | 94 ++++++++++++++++++- .../terminal/terminal-worker-socket-path.ts | 84 +++++++++++++++-- 4 files changed, 184 insertions(+), 25 deletions(-) diff --git a/docs/terminal-worker.md b/docs/terminal-worker.md index d83dea9957..42e1f8cbc3 100644 --- a/docs/terminal-worker.md +++ b/docs/terminal-worker.md @@ -29,6 +29,7 @@ The worker writes `$PASEO_HOME/terminal-worker/.json` (owner-only) with its - Workers never query their own birth time or command line. Readers accept legacy identity fields but ignore them. Daemons through 0.15.1 report new records unreadable and cannot adopt them. - Windows terminal programs have no recorded birth time and terminal lifecycle operations never launch PowerShell. Linux and macOS retain program birth-time checks through `ps` with `LC_ALL=C`; Linux also walks sessions to find surviving jobs. - Unix socket paths have a byte budget (103 on macOS, 107 on Linux). Long dev homes use an owner-only fallback under `$TMPDIR/paseo-/` or `/tmp/paseo-/`. An existing directory must be a real `0700` directory owned by you; a shared root is never chmodded. +- A home on a filesystem that refuses Unix sockets (AFS, some network mounts) passes those checks and then fails the bind with `EPERM`, `EACCES`, `EOPNOTSUPP` or `ENOTSUP`. The worker moves on to the next directory on those codes only; any other listen error fails the start. The record carries the path the worker bound, so nothing else recomputes it. - A kill cleanup that cannot confirm every process gone writes `closing-.json` beside the worker record. The worker rewrites its own record whenever a terminal starts or exits, so the kill intent has to live in a file it never writes. A closing record is never adopted; the next stop or start retries it. - An unreadable record of either kind makes `daemon status` and a kill restart fail with its path: a kill cleanup that met it could not check what it names, and leaves nothing else behind. A read that fails with `EPERM`, `EBUSY` or `EACCES` is retried for 250ms first: on Windows, opening a record while the worker renames its new copy over it fails that way for a moment. diff --git a/packages/server/src/terminal/terminal-worker-process.ts b/packages/server/src/terminal/terminal-worker-process.ts index c61ead0628..7e548552b0 100644 --- a/packages/server/src/terminal/terminal-worker-process.ts +++ b/packages/server/src/terminal/terminal-worker-process.ts @@ -19,7 +19,7 @@ import { type TerminalWorkerRecord, } from "./terminal-worker-record.js"; import { createTerminalWorkerRecordPublisher } from "./terminal-worker-record-publisher.js"; -import { resolveTerminalWorkerSocketPath } from "./terminal-worker-socket-path.js"; +import { listenOnTerminalWorkerSocket } from "./terminal-worker-socket-path.js"; import { TERMINAL_WORKER_PROTOCOL_OVERRIDE_ENV, TERMINAL_WORKER_PROTOCOL_VERSION, @@ -700,23 +700,25 @@ function runWorker(argv: string[]): void { } async function start(): Promise { - const socketPath = resolveTerminalWorkerSocketPath({ + const { socketPath, listener } = await listenOnTerminalWorkerSocket({ paseoHome: args.paseoHome, workerPid: process.pid, recordId: args.recordId, + listen: (candidatePath) => { + if (process.platform !== "win32") { + rmSync(candidatePath, { force: true }); + } + const listening = createServer(handleConnection); + return new Promise((resolve, reject) => { + listening.once("error", reject); + listening.listen(candidatePath, () => { + listening.off("error", reject); + resolve(listening); + }); + }); + }, }); - if (process.platform !== "win32") { - rmSync(socketPath, { force: true }); - } - const listening = createServer(handleConnection); - await new Promise((resolve, reject) => { - listening.once("error", reject); - listening.listen(socketPath, () => { - listening.off("error", reject); - resolve(); - }); - }); - server = listening; + server = listener; if (process.platform !== "win32") { chmodSync(socketPath, PRIVATE_FILE_MODE); } diff --git a/packages/server/src/terminal/terminal-worker-socket-path.test.ts b/packages/server/src/terminal/terminal-worker-socket-path.test.ts index 6017177fcf..786f095886 100644 --- a/packages/server/src/terminal/terminal-worker-socket-path.test.ts +++ b/packages/server/src/terminal/terminal-worker-socket-path.test.ts @@ -1,10 +1,11 @@ import { afterEach, describe, expect, it } from "vitest"; import { chmodSync, mkdirSync, mkdtempSync, rmSync, symlinkSync } from "node:fs"; -import { createServer } from "node:net"; +import { createServer, type Server } from "node:net"; import { tmpdir } from "node:os"; import path from "node:path"; import { isPlatform } from "../test-utils/platform.js"; import { + listenOnTerminalWorkerSocket, resolveTerminalWorkerSocketPath, TerminalWorkerSocketPathError, } from "./terminal-worker-socket-path.js"; @@ -25,7 +26,36 @@ function longHome(root: string): string { return home; } -afterEach(() => { +const openServers: Server[] = []; + +function listenFor(socketPath: string): Promise { + const server = createServer(); + return new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(socketPath, () => { + openServers.push(server); + resolve(server); + }); + }); +} + +// Stands in for a filesystem that passes the directory checks but refuses AF_UNIX bind, as AFS +// does with EPERM. +function refuseBindUnder(directory: string, code: string) { + return (socketPath: string): Promise => { + if (socketPath.startsWith(directory + path.sep)) { + return Promise.reject( + Object.assign(new Error(`listen ${code}: ${socketPath}`), { code, syscall: "listen" }), + ); + } + return listenFor(socketPath); + }; +} + +afterEach(async () => { + await Promise.all( + openServers.splice(0).map((server) => new Promise((resolve) => server.close(resolve))), + ); for (const dir of temporaryDirs.splice(0)) { rmSync(dir, { recursive: true, force: true }); } @@ -142,6 +172,66 @@ describe.skipIf(isPlatform("win32"))("terminal worker socket path", () => { }); }); +describe.skipIf(isPlatform("win32"))("terminal worker socket listen", () => { + it("falls back to the next directory when the home's filesystem refuses the bind", async () => { + const home = makeDir("tw-b-", "/tmp"); + const tmpDir = makeDir("tw-t-", "/tmp"); + const homeSocketDirectory = path.join(home, "terminal-worker", "sock"); + const { socketPath, listener } = await listenOnTerminalWorkerSocket({ + paseoHome: home, + recordId: "test", + workerPid: 11, + tmpDir, + listen: refuseBindUnder(homeSocketDirectory, "EPERM"), + }); + expect(path.dirname(socketPath)).toBe(path.join(tmpDir, `paseo-${process.getuid!()}`)); + expect(listener.address()).toBe(socketPath); + }); + + it("throws a listen error that does not mean the filesystem refuses sockets", async () => { + const home = makeDir("tw-u-", "/tmp"); + const homeSocketDirectory = path.join(home, "terminal-worker", "sock"); + await expect( + listenOnTerminalWorkerSocket({ + paseoHome: home, + recordId: "test", + workerPid: 12, + tmpDir: makeDir("tw-t-", "/tmp"), + listen: refuseBindUnder(homeSocketDirectory, "EADDRINUSE"), + }), + ).rejects.toMatchObject({ code: "EADDRINUSE" }); + }); + + it("names every candidate's bind error when no directory accepts a socket", async () => { + const home = makeDir("tw-n-", "/tmp"); + const tmpDir = makeDir("tw-t-", "/tmp"); + let caught: unknown; + try { + await listenOnTerminalWorkerSocket({ + paseoHome: home, + recordId: "test", + workerPid: 13, + tmpDir, + listen: refuseBindUnder("", "EPERM"), + }); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(TerminalWorkerSocketPathError); + const rejected = (caught as TerminalWorkerSocketPathError).rejected; + expect(rejected.slice(0, 2)).toEqual([ + expect.objectContaining({ + path: expect.stringContaining(path.join(home, "terminal-worker", "sock")), + reason: "cannot bind a socket here (EPERM)", + }), + expect.objectContaining({ + path: expect.stringContaining(path.join(tmpDir, `paseo-${process.getuid!()}`)), + reason: "cannot bind a socket here (EPERM)", + }), + ]); + }); +}); + it("uses a separate socket for each record even when its pid is reused", () => { const paseoHome = makeDir("tw-id-"); const first = resolveTerminalWorkerSocketPath({ paseoHome, workerPid: 42, recordId: "old" }); diff --git a/packages/server/src/terminal/terminal-worker-socket-path.ts b/packages/server/src/terminal/terminal-worker-socket-path.ts index d9a153c598..0ad94a1ced 100644 --- a/packages/server/src/terminal/terminal-worker-socket-path.ts +++ b/packages/server/src/terminal/terminal-worker-socket-path.ts @@ -14,6 +14,9 @@ const SOCKET_PATH_BYTE_BUDGET: Partial> = { }; const DEFAULT_SOCKET_PATH_BYTE_BUDGET = 103; const OWNER_ONLY_MODE = 0o700; +// Bind errors that mean the directory's filesystem cannot hold a Unix socket. AFS and some other +// network filesystems pass the directory checks and then refuse AF_UNIX bind with EPERM. +const SOCKET_UNSUPPORTED_BIND_CODES = new Set(["EPERM", "EACCES", "EOPNOTSUPP", "ENOTSUP"]); export interface TerminalWorkerSocketPathInput { paseoHome: string; @@ -24,6 +27,16 @@ export interface TerminalWorkerSocketPathInput { uid?: number; } +export interface TerminalWorkerSocketListenInput extends TerminalWorkerSocketPathInput { + /** Binds a fresh listener at the path, removing any stale socket there first. */ + listen: (socketPath: string) => Promise; +} + +export interface TerminalWorkerSocketListener { + socketPath: string; + listener: T; +} + export interface RejectedSocketCandidate { path: string; byteLength: number; @@ -36,7 +49,7 @@ export class TerminalWorkerSocketPathError extends Error { constructor(rejected: RejectedSocketCandidate[], budget: number) { super( [ - `No terminal worker socket path fits ${budget} bytes in an owner-only directory:`, + `No terminal worker socket path fits ${budget} bytes in an owner-only directory that accepts a socket:`, ...rejected.map( (candidate) => ` ${candidate.path} (${candidate.byteLength} bytes): ${candidate.reason}`, ), @@ -57,18 +70,73 @@ export function terminalWorkerSocketId(paseoHome: string, workerPid: number): st * Picks (and creates) the socket path a worker listens on. Windows uses a named pipe; POSIX * tries $PASEO_HOME/terminal-worker/sock, then /paseo-, then /tmp/paseo-. * A shared root is never chmodded: an existing directory there is used only when it is a real - * directory owned by this user with mode 0700. + * directory owned by this user with mode 0700. It does not bind; the worker listens through + * `listenOnTerminalWorkerSocket`, which also skips directories that refuse a socket. */ export function resolveTerminalWorkerSocketPath(input: TerminalWorkerSocketPathInput): string { const platform = input.platform ?? process.platform; - const id = terminalWorkerSocketId(input.paseoHome, input.workerPid) + "-" + input.recordId; if (platform === "win32") { - return `\\\\.\\pipe\\paseo-tw-${id}`; + return windowsPipePath(input); } + const budget = socketPathByteBudget(platform); + const rejected: RejectedSocketCandidate[] = []; + for (const candidate of eligibleUnixSocketPaths(input, budget, rejected)) { + return candidate.path; + } + throw new TerminalWorkerSocketPathError(rejected, budget); +} +/** + * Listens on the first candidate path that accepts a socket. A bind refused by the filesystem + * moves on to the next candidate; any other listen error (EADDRINUSE, ...) is thrown as is. + */ +export async function listenOnTerminalWorkerSocket( + input: TerminalWorkerSocketListenInput, +): Promise> { + const { listen } = input; + const platform = input.platform ?? process.platform; + if (platform === "win32") { + const socketPath = windowsPipePath(input); + return { socketPath, listener: await listen(socketPath) }; + } + const budget = socketPathByteBudget(platform); + const rejected: RejectedSocketCandidate[] = []; + for (const candidate of eligibleUnixSocketPaths(input, budget, rejected)) { + try { + return { socketPath: candidate.path, listener: await listen(candidate.path) }; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (!code || !SOCKET_UNSUPPORTED_BIND_CODES.has(code)) throw error; + rejected.push({ + path: candidate.path, + byteLength: candidate.byteLength, + reason: `cannot bind a socket here (${code})`, + }); + } + } + throw new TerminalWorkerSocketPathError(rejected, budget); +} + +function windowsPipePath(input: TerminalWorkerSocketPathInput): string { + return `\\\\.\\pipe\\paseo-tw-${socketFileId(input)}`; +} + +function socketFileId(input: TerminalWorkerSocketPathInput): string { + return terminalWorkerSocketId(input.paseoHome, input.workerPid) + "-" + input.recordId; +} + +function socketPathByteBudget(platform: NodeJS.Platform): number { + return SOCKET_PATH_BYTE_BUDGET[platform] ?? DEFAULT_SOCKET_PATH_BYTE_BUDGET; +} + +/** Yields each candidate that fits the budget in an owner-only directory, in order. */ +function* eligibleUnixSocketPaths( + input: TerminalWorkerSocketPathInput, + budget: number, + rejected: RejectedSocketCandidate[], +): Generator<{ path: string; byteLength: number }> { const uid = input.uid ?? process.getuid?.() ?? 0; - const budget = SOCKET_PATH_BYTE_BUDGET[platform] ?? DEFAULT_SOCKET_PATH_BYTE_BUDGET; - const fileName = `tw-${id}.sock`; + const fileName = `tw-${socketFileId(input)}.sock`; const homeSocketDirectory = path.join(terminalWorkerDirectory(input.paseoHome), "sock"); const candidates: Array<{ directory: string; prepare: () => string | null }> = [ { @@ -87,7 +155,6 @@ export function resolveTerminalWorkerSocketPath(input: TerminalWorkerSocketPathI }), ]; - const rejected: RejectedSocketCandidate[] = []; const seen = new Set(); for (const candidate of candidates) { const socketPath = path.join(candidate.directory, fileName); @@ -108,9 +175,8 @@ export function resolveTerminalWorkerSocketPath(input: TerminalWorkerSocketPathI rejected.push({ path: socketPath, byteLength, reason: problem }); continue; } - return socketPath; + yield { path: socketPath, byteLength }; } - throw new TerminalWorkerSocketPathError(rejected, budget); } function prepareSharedRootDirectory(directory: string, uid: number): string | null {