From 46bd5d8a3ed7dc2dae4f918c600ced41bcf7a27e Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Wed, 18 Nov 2026 19:24:41 +0700 Subject: [PATCH] Keep Hub daemons connected after execution permissions change (#10113) * Use current Hub permissions when reconnecting a daemon * Back off when the Hub repeatedly rejects session permissions * Show the permission recovery commands in Hub status (cherry picked from commit 0e67045c4b220162637c54ff470468fbafe201c9) --- .../hub/relationship-controller.test.ts | 34 +++++++++++++++++ .../src/server/hub/relationship-controller.ts | 38 +++++++++++-------- .../server/hub/relationship-remote.test.ts | 37 ++++++++++++++++-- .../src/server/hub/relationship-remote.ts | 6 +-- .../hub/test-utils/relationship-harness.ts | 4 +- 5 files changed, 96 insertions(+), 23 deletions(-) diff --git a/packages/server/src/server/hub/relationship-controller.test.ts b/packages/server/src/server/hub/relationship-controller.test.ts index d994a7280e..82d41f3d36 100644 --- a/packages/server/src/server/hub/relationship-controller.test.ts +++ b/packages/server/src/server/hub/relationship-controller.test.ts @@ -153,6 +153,40 @@ describe("Hub relationship", () => { }); }); + test.each([false, true])( + "reconnect advertises the current execution grant (initially %s)", + async (initiallyGranted) => { + relationship = await HubRelationshipHarness.start(); + await relationship.beginConnect("token", "https://hub.example", initiallyGranted).result; + relationship.connectLatestSocket(); + if (initiallyGranted) await relationship.revokePermission("hub.execute"); + else await relationship.grantPermission("hub.execute"); + relationship.closeLatestSocket(1006); + await relationship.retry(); + relationship.connectLatestSocket(); + + expect(relationship.serverInfoPermissions().at(-1)).toEqual( + initiallyGranted ? [] : ["hub.execute"], + ); + }, + ); + + test("a permission mismatch retries the handshake with the same enrollment", async () => { + relationship = await HubRelationshipHarness.start(); + await relationship.beginConnect().result; + relationship.connectLatestSocket(); + const reason = "daemon session permissions do not match enrollment"; + relationship.closeLatestSocket(4403, reason); + expect(await relationship.status()).toMatchObject({ + state: "reconnecting", + lastError: expect.stringContaining("paseo hub permissions"), + }); + await relationship.retry(); + await relationship.revokePermission("hub.execute"); + relationship.connectLatestSocket(); + expect(relationship.serverInfoPermissions().at(-1)).toEqual([]); + }); + test("Hub enrollment cannot widen the daemon's locally granted permissions", async () => { relationship = await HubRelationshipHarness.start(); relationship.returnEnrollmentPermissions(["hub.execute", "daemon.manage"]); diff --git a/packages/server/src/server/hub/relationship-controller.ts b/packages/server/src/server/hub/relationship-controller.ts index 89a86ca6c3..af671ae8ea 100644 --- a/packages/server/src/server/hub/relationship-controller.ts +++ b/packages/server/src/server/hub/relationship-controller.ts @@ -246,7 +246,7 @@ export class HubRelationshipController implements HubRelationshipManagement { } async start(): Promise { - if (this.record?.state === "active") this.openSocket(this.record, false); + if (this.record?.state === "active") this.openSocket(false); if (this.record?.state === "pending") { const enrollmentGeneration = this.beginEnrollmentAttempt(); try { @@ -441,7 +441,7 @@ export class HubRelationshipController implements HubRelationshipManagement { this.record = active; this.retry = null; this.retryAttempt = 0; - this.openSocket(active, false); + this.openSocket(false); } catch (error) { if (enrollmentGeneration !== this.enrollmentGeneration) return; if (error instanceof HubEnrollmentRejectedError) { @@ -455,15 +455,17 @@ export class HubRelationshipController implements HubRelationshipManagement { } } - private openSocket(record: ActiveRecord, reconnecting: boolean): void { + private openSocket(reconnecting: boolean): void { + const record = this.record; + if (record?.state !== "active") return; const generation = ++this.generation; this.state = reconnecting ? "reconnecting" : "connecting"; const events: HubSocketEvents = { connected: (socket, sessionProtocol) => - this.socketConnected(generation, record, socket, sessionProtocol), + this.socketConnected(generation, socket, sessionProtocol), rejected: (statusCode) => this.socketRejected(generation, statusCode), - closed: (code) => this.socketClosed(generation, record, code), - failed: (error) => this.socketFailed(generation, record, error), + closed: (code, reason) => this.socketClosed(generation, code, reason), + failed: (error) => this.socketFailed(generation, error), }; this.socket = this.options.remote.openSocket( { @@ -477,15 +479,14 @@ export class HubRelationshipController implements HubRelationshipManagement { private socketConnected( generation: number, - record: ActiveRecord, socket: WebSocketLike, sessionProtocol: "legacy" | "session-v1", ): void { - if (generation !== this.generation) { + const record = this.record; + if (generation !== this.generation || record?.state !== "active") { socket.close(); return; } - this.retryAttempt = 0; this.state = "connected"; this.connectedAt = this.clock.now().toISOString(); this.lastError = null; @@ -516,24 +517,31 @@ export class HubRelationshipController implements HubRelationshipManagement { this.revoke(`Hub rejected socket authentication (${statusCode})`); } - private socketClosed(generation: number, record: ActiveRecord, code: number): void { + private socketClosed(generation: number, code: number, reason?: string): void { if (generation !== this.generation) return; + if (code === 4403 && reason === "daemon session permissions do not match enrollment") { + this.lastError = + "Hub and daemon permissions differ. Run `paseo hub permissions grant ` or `paseo hub permissions revoke ` on this daemon to reapply your intended permissions."; + this.scheduleSocket(true); + return; + } if (code === 4403) { this.revoke("Hub revoked this relationship"); return; } - if (this.record?.state === "active") this.scheduleSocket(record); + if (this.record?.state === "active") this.scheduleSocket(); } - private socketFailed(generation: number, record: ActiveRecord, error: Error): void { + private socketFailed(generation: number, error: Error): void { if (generation !== this.generation) return; this.lastError = error.message; - if (this.record?.state === "active") this.scheduleSocket(record); + if (this.record?.state === "active") this.scheduleSocket(); } - private scheduleSocket(record: ActiveRecord): void { + private scheduleSocket(permissionMismatch = false): void { + if (this.state === "connected" && !permissionMismatch) this.retryAttempt = 0; this.state = "reconnecting"; - this.schedule(() => this.openSocket(record, true)); + this.schedule(() => this.openSocket(true)); } private scheduleEnrollment(record: PendingRecord, enrollmentGeneration: number): void { diff --git a/packages/server/src/server/hub/relationship-remote.test.ts b/packages/server/src/server/hub/relationship-remote.test.ts index 452947d4c8..eb38b52ea2 100644 --- a/packages/server/src/server/hub/relationship-remote.test.ts +++ b/packages/server/src/server/hub/relationship-remote.test.ts @@ -330,6 +330,27 @@ test.each([401, 403] as const)( }, ); +test("repeated permission mismatch closes back off until the Hub accepts the permissions", async () => { + const mismatch = { + closeCode: 4403, + reason: "daemon session permissions do not match enrollment", + }; + const hub = await UpgradeRejectingHub.start([ + mismatch, + mismatch, + { closeCode: 1001, reason: "shutdown" }, + ]); + const clock = new ManualRelationshipClock(); + const controller = await connectController(hub, clock); + await expect.poll(() => clock.scheduledAttempts()).toEqual([0]); + clock.runNext(); + await expect.poll(() => clock.scheduledAttempts()).toEqual([0, 1]); + expect(controller.status().state).toBe("reconnecting"); + clock.runNext(); + await expect.poll(() => clock.scheduledAttempts()).toEqual([0, 1, 0]); + await controller.stop(); +}); + test("controller redials after a transient socket response and releases both failed upgrades", async () => { const hub = await UpgradeRejectingHub.start([503, 401]); const clock = new ManualRelationshipClock(); @@ -475,6 +496,8 @@ class SocketOutcomes { } } +type UpgradeOutcome = number | "stall" | { closeCode: number; reason: string }; + class UpgradeRejectingHub { private readonly server = createServer(async (request, response) => { let body = ""; @@ -493,8 +516,8 @@ class UpgradeRejectingHub { private attemptObserved = deferred(); private attempts = 0; - private constructor(private readonly statuses: Array) { - this.server.on("upgrade", (_request, socket) => { + private constructor(private readonly statuses: UpgradeOutcome[]) { + this.server.on("upgrade", (request, socket, head) => { const attempt = ++this.attempts; this.attemptObserved.resolve(); this.attemptObserved = deferred(); @@ -506,6 +529,14 @@ class UpgradeRejectingHub { released.resolve(); }); const status = this.statuses[attempt - 1] ?? this.statuses.at(-1) ?? 503; + if (typeof status === "object") { + const ws = new WebSocketServer({ noServer: true }); + ws.handleUpgrade(request, socket, head, (peer) => + peer.close(status.closeCode, status.reason), + ); + ws.close(); + return; + } if (status === "stall") { socket.resume(); return; @@ -520,7 +551,7 @@ class UpgradeRejectingHub { }); } - static async start(statuses: Array): Promise { + static async start(statuses: UpgradeOutcome[]): Promise { const hub = new UpgradeRejectingHub(statuses); openUpgradeHubs.push(hub); openServers.push(hub.server); diff --git a/packages/server/src/server/hub/relationship-remote.ts b/packages/server/src/server/hub/relationship-remote.ts index 6427b71fa6..7481a9d80b 100644 --- a/packages/server/src/server/hub/relationship-remote.ts +++ b/packages/server/src/server/hub/relationship-remote.ts @@ -46,7 +46,7 @@ export interface HubSocketCredentials { export interface HubSocketEvents { connected(socket: WebSocketLike, sessionProtocol: "legacy" | "session-v1"): void; rejected(statusCode: 401 | 403): void; - closed(code: number): void; + closed(code: number, reason?: string): void; failed(error: Error): void; } @@ -227,11 +227,11 @@ export class DirectHubRelationshipRemote implements HubRelationshipRemote { } events.closed(1006); }); - socket.once("close", (code) => { + socket.once("close", (code, reason) => { stopHeartbeat(); if (settled) return; settled = true; - events.closed(code); + events.closed(code, reason.toString()); }); socket.once("error", fail); return socket; diff --git a/packages/server/src/server/hub/test-utils/relationship-harness.ts b/packages/server/src/server/hub/test-utils/relationship-harness.ts index f2e0e4dff6..5da000a3ea 100644 --- a/packages/server/src/server/hub/test-utils/relationship-harness.ts +++ b/packages/server/src/server/hub/test-utils/relationship-harness.ts @@ -720,9 +720,9 @@ export class HubRelationshipHarness { this.rejectLatestSocket(code); } - closeLatestSocket(code: number): void { + closeLatestSocket(code: number, reason?: string): void { const socket = this.latestSocket(); - socket.events.closed(code); + socket.events.closed(code, reason); socket.socket.close(code); }