fix(server): fall back to the next socket directory when the bind is refused (#9214)

A PASEO_HOME on AFS passes the socket directory checks, then refuses the
Unix socket bind with EPERM, and the terminal worker failed to start. The
worker now binds each eligible candidate in turn and moves on when the
filesystem refuses the socket (EPERM, EACCES, EOPNOTSUPP, ENOTSUP). Other
listen errors still fail the start.

Fixes #9213

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit ade3817a45)
This commit is contained in:
Josh Bendavid
2026-11-18 14:15:27 +01:00
committed by Mohamed Boudra
parent c09a3d81e7
commit 293fbf825d
4 changed files with 184 additions and 25 deletions
+1
View File
@@ -29,6 +29,7 @@ The worker writes `$PASEO_HOME/terminal-worker/<pid>.json` (owner-only) with its
- Workers never query their own birth time or command line. Readers accept legacy identity fields but ignore them. Daemons through 0.15.1 report new records unreadable and cannot adopt them.
- Windows terminal programs have no recorded birth time and terminal lifecycle operations never launch PowerShell. Linux and macOS retain program birth-time checks through `ps` with `LC_ALL=C`; Linux also walks sessions to find surviving jobs.
- Unix socket paths have a byte budget (103 on macOS, 107 on Linux). Long dev homes use an owner-only fallback under `$TMPDIR/paseo-<uid>/` or `/tmp/paseo-<uid>/`. An existing directory must be a real `0700` directory owned by you; a shared root is never chmodded.
- A home on a filesystem that refuses Unix sockets (AFS, some network mounts) passes those checks and then fails the bind with `EPERM`, `EACCES`, `EOPNOTSUPP` or `ENOTSUP`. The worker moves on to the next directory on those codes only; any other listen error fails the start. The record carries the path the worker bound, so nothing else recomputes it.
- A kill cleanup that cannot confirm every process gone writes `closing-<recordId>.json` beside the worker record. The worker rewrites its own record whenever a terminal starts or exits, so the kill intent has to live in a file it never writes. A closing record is never adopted; the next stop or start retries it.
- An unreadable record of either kind makes `daemon status` and a kill restart fail with its path: a kill cleanup that met it could not check what it names, and leaves nothing else behind. A read that fails with `EPERM`, `EBUSY` or `EACCES` is retried for 250ms first: on Windows, opening a record while the worker renames its new copy over it fails that way for a moment.
@@ -19,7 +19,7 @@ import {
type TerminalWorkerRecord,
} from "./terminal-worker-record.js";
import { createTerminalWorkerRecordPublisher } from "./terminal-worker-record-publisher.js";
import { resolveTerminalWorkerSocketPath } from "./terminal-worker-socket-path.js";
import { listenOnTerminalWorkerSocket } from "./terminal-worker-socket-path.js";
import {
TERMINAL_WORKER_PROTOCOL_OVERRIDE_ENV,
TERMINAL_WORKER_PROTOCOL_VERSION,
@@ -700,23 +700,25 @@ function runWorker(argv: string[]): void {
}
async function start(): Promise<void> {
const socketPath = resolveTerminalWorkerSocketPath({
const { socketPath, listener } = await listenOnTerminalWorkerSocket({
paseoHome: args.paseoHome,
workerPid: process.pid,
recordId: args.recordId,
listen: (candidatePath) => {
if (process.platform !== "win32") {
rmSync(candidatePath, { force: true });
}
const listening = createServer(handleConnection);
return new Promise<Server>((resolve, reject) => {
listening.once("error", reject);
listening.listen(candidatePath, () => {
listening.off("error", reject);
resolve(listening);
});
});
},
});
if (process.platform !== "win32") {
rmSync(socketPath, { force: true });
}
const listening = createServer(handleConnection);
await new Promise<void>((resolve, reject) => {
listening.once("error", reject);
listening.listen(socketPath, () => {
listening.off("error", reject);
resolve();
});
});
server = listening;
server = listener;
if (process.platform !== "win32") {
chmodSync(socketPath, PRIVATE_FILE_MODE);
}
@@ -1,10 +1,11 @@
import { afterEach, describe, expect, it } from "vitest";
import { chmodSync, mkdirSync, mkdtempSync, rmSync, symlinkSync } from "node:fs";
import { createServer } from "node:net";
import { createServer, type Server } from "node:net";
import { tmpdir } from "node:os";
import path from "node:path";
import { isPlatform } from "../test-utils/platform.js";
import {
listenOnTerminalWorkerSocket,
resolveTerminalWorkerSocketPath,
TerminalWorkerSocketPathError,
} from "./terminal-worker-socket-path.js";
@@ -25,7 +26,36 @@ function longHome(root: string): string {
return home;
}
afterEach(() => {
const openServers: Server[] = [];
function listenFor(socketPath: string): Promise<Server> {
const server = createServer();
return new Promise((resolve, reject) => {
server.once("error", reject);
server.listen(socketPath, () => {
openServers.push(server);
resolve(server);
});
});
}
// Stands in for a filesystem that passes the directory checks but refuses AF_UNIX bind, as AFS
// does with EPERM.
function refuseBindUnder(directory: string, code: string) {
return (socketPath: string): Promise<Server> => {
if (socketPath.startsWith(directory + path.sep)) {
return Promise.reject(
Object.assign(new Error(`listen ${code}: ${socketPath}`), { code, syscall: "listen" }),
);
}
return listenFor(socketPath);
};
}
afterEach(async () => {
await Promise.all(
openServers.splice(0).map((server) => new Promise((resolve) => server.close(resolve))),
);
for (const dir of temporaryDirs.splice(0)) {
rmSync(dir, { recursive: true, force: true });
}
@@ -142,6 +172,66 @@ describe.skipIf(isPlatform("win32"))("terminal worker socket path", () => {
});
});
describe.skipIf(isPlatform("win32"))("terminal worker socket listen", () => {
it("falls back to the next directory when the home's filesystem refuses the bind", async () => {
const home = makeDir("tw-b-", "/tmp");
const tmpDir = makeDir("tw-t-", "/tmp");
const homeSocketDirectory = path.join(home, "terminal-worker", "sock");
const { socketPath, listener } = await listenOnTerminalWorkerSocket({
paseoHome: home,
recordId: "test",
workerPid: 11,
tmpDir,
listen: refuseBindUnder(homeSocketDirectory, "EPERM"),
});
expect(path.dirname(socketPath)).toBe(path.join(tmpDir, `paseo-${process.getuid!()}`));
expect(listener.address()).toBe(socketPath);
});
it("throws a listen error that does not mean the filesystem refuses sockets", async () => {
const home = makeDir("tw-u-", "/tmp");
const homeSocketDirectory = path.join(home, "terminal-worker", "sock");
await expect(
listenOnTerminalWorkerSocket({
paseoHome: home,
recordId: "test",
workerPid: 12,
tmpDir: makeDir("tw-t-", "/tmp"),
listen: refuseBindUnder(homeSocketDirectory, "EADDRINUSE"),
}),
).rejects.toMatchObject({ code: "EADDRINUSE" });
});
it("names every candidate's bind error when no directory accepts a socket", async () => {
const home = makeDir("tw-n-", "/tmp");
const tmpDir = makeDir("tw-t-", "/tmp");
let caught: unknown;
try {
await listenOnTerminalWorkerSocket({
paseoHome: home,
recordId: "test",
workerPid: 13,
tmpDir,
listen: refuseBindUnder("", "EPERM"),
});
} catch (error) {
caught = error;
}
expect(caught).toBeInstanceOf(TerminalWorkerSocketPathError);
const rejected = (caught as TerminalWorkerSocketPathError).rejected;
expect(rejected.slice(0, 2)).toEqual([
expect.objectContaining({
path: expect.stringContaining(path.join(home, "terminal-worker", "sock")),
reason: "cannot bind a socket here (EPERM)",
}),
expect.objectContaining({
path: expect.stringContaining(path.join(tmpDir, `paseo-${process.getuid!()}`)),
reason: "cannot bind a socket here (EPERM)",
}),
]);
});
});
it("uses a separate socket for each record even when its pid is reused", () => {
const paseoHome = makeDir("tw-id-");
const first = resolveTerminalWorkerSocketPath({ paseoHome, workerPid: 42, recordId: "old" });
@@ -14,6 +14,9 @@ const SOCKET_PATH_BYTE_BUDGET: Partial<Record<NodeJS.Platform, number>> = {
};
const DEFAULT_SOCKET_PATH_BYTE_BUDGET = 103;
const OWNER_ONLY_MODE = 0o700;
// Bind errors that mean the directory's filesystem cannot hold a Unix socket. AFS and some other
// network filesystems pass the directory checks and then refuse AF_UNIX bind with EPERM.
const SOCKET_UNSUPPORTED_BIND_CODES = new Set(["EPERM", "EACCES", "EOPNOTSUPP", "ENOTSUP"]);
export interface TerminalWorkerSocketPathInput {
paseoHome: string;
@@ -24,6 +27,16 @@ export interface TerminalWorkerSocketPathInput {
uid?: number;
}
export interface TerminalWorkerSocketListenInput<T> extends TerminalWorkerSocketPathInput {
/** Binds a fresh listener at the path, removing any stale socket there first. */
listen: (socketPath: string) => Promise<T>;
}
export interface TerminalWorkerSocketListener<T> {
socketPath: string;
listener: T;
}
export interface RejectedSocketCandidate {
path: string;
byteLength: number;
@@ -36,7 +49,7 @@ export class TerminalWorkerSocketPathError extends Error {
constructor(rejected: RejectedSocketCandidate[], budget: number) {
super(
[
`No terminal worker socket path fits ${budget} bytes in an owner-only directory:`,
`No terminal worker socket path fits ${budget} bytes in an owner-only directory that accepts a socket:`,
...rejected.map(
(candidate) => ` ${candidate.path} (${candidate.byteLength} bytes): ${candidate.reason}`,
),
@@ -57,18 +70,73 @@ export function terminalWorkerSocketId(paseoHome: string, workerPid: number): st
* Picks (and creates) the socket path a worker listens on. Windows uses a named pipe; POSIX
* tries $PASEO_HOME/terminal-worker/sock, then <tmpdir>/paseo-<uid>, then /tmp/paseo-<uid>.
* A shared root is never chmodded: an existing directory there is used only when it is a real
* directory owned by this user with mode 0700.
* directory owned by this user with mode 0700. It does not bind; the worker listens through
* `listenOnTerminalWorkerSocket`, which also skips directories that refuse a socket.
*/
export function resolveTerminalWorkerSocketPath(input: TerminalWorkerSocketPathInput): string {
const platform = input.platform ?? process.platform;
const id = terminalWorkerSocketId(input.paseoHome, input.workerPid) + "-" + input.recordId;
if (platform === "win32") {
return `\\\\.\\pipe\\paseo-tw-${id}`;
return windowsPipePath(input);
}
const budget = socketPathByteBudget(platform);
const rejected: RejectedSocketCandidate[] = [];
for (const candidate of eligibleUnixSocketPaths(input, budget, rejected)) {
return candidate.path;
}
throw new TerminalWorkerSocketPathError(rejected, budget);
}
/**
* Listens on the first candidate path that accepts a socket. A bind refused by the filesystem
* moves on to the next candidate; any other listen error (EADDRINUSE, ...) is thrown as is.
*/
export async function listenOnTerminalWorkerSocket<T>(
input: TerminalWorkerSocketListenInput<T>,
): Promise<TerminalWorkerSocketListener<T>> {
const { listen } = input;
const platform = input.platform ?? process.platform;
if (platform === "win32") {
const socketPath = windowsPipePath(input);
return { socketPath, listener: await listen(socketPath) };
}
const budget = socketPathByteBudget(platform);
const rejected: RejectedSocketCandidate[] = [];
for (const candidate of eligibleUnixSocketPaths(input, budget, rejected)) {
try {
return { socketPath: candidate.path, listener: await listen(candidate.path) };
} catch (error) {
const code = (error as NodeJS.ErrnoException).code;
if (!code || !SOCKET_UNSUPPORTED_BIND_CODES.has(code)) throw error;
rejected.push({
path: candidate.path,
byteLength: candidate.byteLength,
reason: `cannot bind a socket here (${code})`,
});
}
}
throw new TerminalWorkerSocketPathError(rejected, budget);
}
function windowsPipePath(input: TerminalWorkerSocketPathInput): string {
return `\\\\.\\pipe\\paseo-tw-${socketFileId(input)}`;
}
function socketFileId(input: TerminalWorkerSocketPathInput): string {
return terminalWorkerSocketId(input.paseoHome, input.workerPid) + "-" + input.recordId;
}
function socketPathByteBudget(platform: NodeJS.Platform): number {
return SOCKET_PATH_BYTE_BUDGET[platform] ?? DEFAULT_SOCKET_PATH_BYTE_BUDGET;
}
/** Yields each candidate that fits the budget in an owner-only directory, in order. */
function* eligibleUnixSocketPaths(
input: TerminalWorkerSocketPathInput,
budget: number,
rejected: RejectedSocketCandidate[],
): Generator<{ path: string; byteLength: number }> {
const uid = input.uid ?? process.getuid?.() ?? 0;
const budget = SOCKET_PATH_BYTE_BUDGET[platform] ?? DEFAULT_SOCKET_PATH_BYTE_BUDGET;
const fileName = `tw-${id}.sock`;
const fileName = `tw-${socketFileId(input)}.sock`;
const homeSocketDirectory = path.join(terminalWorkerDirectory(input.paseoHome), "sock");
const candidates: Array<{ directory: string; prepare: () => string | null }> = [
{
@@ -87,7 +155,6 @@ export function resolveTerminalWorkerSocketPath(input: TerminalWorkerSocketPathI
}),
];
const rejected: RejectedSocketCandidate[] = [];
const seen = new Set<string>();
for (const candidate of candidates) {
const socketPath = path.join(candidate.directory, fileName);
@@ -108,9 +175,8 @@ export function resolveTerminalWorkerSocketPath(input: TerminalWorkerSocketPathI
rejected.push({ path: socketPath, byteLength, reason: problem });
continue;
}
return socketPath;
yield { path: socketPath, byteLength };
}
throw new TerminalWorkerSocketPathError(rejected, budget);
}
function prepareSharedRootDirectory(directory: string, uid: number): string | null {