Keep Hub daemons connected after execution permissions change (#10113)

* Use current Hub permissions when reconnecting a daemon

* Back off when the Hub repeatedly rejects session permissions

* Show the permission recovery commands in Hub status

(cherry picked from commit 0e67045c4b)
This commit is contained in:
Mohamed Boudra
2026-11-18 14:15:27 +01:00
parent 9895e0c957
commit 46bd5d8a3e
5 changed files with 96 additions and 23 deletions
@@ -153,6 +153,40 @@ describe("Hub relationship", () => {
});
});
test.each([false, true])(
"reconnect advertises the current execution grant (initially %s)",
async (initiallyGranted) => {
relationship = await HubRelationshipHarness.start();
await relationship.beginConnect("token", "https://hub.example", initiallyGranted).result;
relationship.connectLatestSocket();
if (initiallyGranted) await relationship.revokePermission("hub.execute");
else await relationship.grantPermission("hub.execute");
relationship.closeLatestSocket(1006);
await relationship.retry();
relationship.connectLatestSocket();
expect(relationship.serverInfoPermissions().at(-1)).toEqual(
initiallyGranted ? [] : ["hub.execute"],
);
},
);
test("a permission mismatch retries the handshake with the same enrollment", async () => {
relationship = await HubRelationshipHarness.start();
await relationship.beginConnect().result;
relationship.connectLatestSocket();
const reason = "daemon session permissions do not match enrollment";
relationship.closeLatestSocket(4403, reason);
expect(await relationship.status()).toMatchObject({
state: "reconnecting",
lastError: expect.stringContaining("paseo hub permissions"),
});
await relationship.retry();
await relationship.revokePermission("hub.execute");
relationship.connectLatestSocket();
expect(relationship.serverInfoPermissions().at(-1)).toEqual([]);
});
test("Hub enrollment cannot widen the daemon's locally granted permissions", async () => {
relationship = await HubRelationshipHarness.start();
relationship.returnEnrollmentPermissions(["hub.execute", "daemon.manage"]);
@@ -246,7 +246,7 @@ export class HubRelationshipController implements HubRelationshipManagement {
}
async start(): Promise<void> {
if (this.record?.state === "active") this.openSocket(this.record, false);
if (this.record?.state === "active") this.openSocket(false);
if (this.record?.state === "pending") {
const enrollmentGeneration = this.beginEnrollmentAttempt();
try {
@@ -441,7 +441,7 @@ export class HubRelationshipController implements HubRelationshipManagement {
this.record = active;
this.retry = null;
this.retryAttempt = 0;
this.openSocket(active, false);
this.openSocket(false);
} catch (error) {
if (enrollmentGeneration !== this.enrollmentGeneration) return;
if (error instanceof HubEnrollmentRejectedError) {
@@ -455,15 +455,17 @@ export class HubRelationshipController implements HubRelationshipManagement {
}
}
private openSocket(record: ActiveRecord, reconnecting: boolean): void {
private openSocket(reconnecting: boolean): void {
const record = this.record;
if (record?.state !== "active") return;
const generation = ++this.generation;
this.state = reconnecting ? "reconnecting" : "connecting";
const events: HubSocketEvents = {
connected: (socket, sessionProtocol) =>
this.socketConnected(generation, record, socket, sessionProtocol),
this.socketConnected(generation, socket, sessionProtocol),
rejected: (statusCode) => this.socketRejected(generation, statusCode),
closed: (code) => this.socketClosed(generation, record, code),
failed: (error) => this.socketFailed(generation, record, error),
closed: (code, reason) => this.socketClosed(generation, code, reason),
failed: (error) => this.socketFailed(generation, error),
};
this.socket = this.options.remote.openSocket(
{
@@ -477,15 +479,14 @@ export class HubRelationshipController implements HubRelationshipManagement {
private socketConnected(
generation: number,
record: ActiveRecord,
socket: WebSocketLike,
sessionProtocol: "legacy" | "session-v1",
): void {
if (generation !== this.generation) {
const record = this.record;
if (generation !== this.generation || record?.state !== "active") {
socket.close();
return;
}
this.retryAttempt = 0;
this.state = "connected";
this.connectedAt = this.clock.now().toISOString();
this.lastError = null;
@@ -516,24 +517,31 @@ export class HubRelationshipController implements HubRelationshipManagement {
this.revoke(`Hub rejected socket authentication (${statusCode})`);
}
private socketClosed(generation: number, record: ActiveRecord, code: number): void {
private socketClosed(generation: number, code: number, reason?: string): void {
if (generation !== this.generation) return;
if (code === 4403 && reason === "daemon session permissions do not match enrollment") {
this.lastError =
"Hub and daemon permissions differ. Run `paseo hub permissions grant <permission>` or `paseo hub permissions revoke <permission>` on this daemon to reapply your intended permissions.";
this.scheduleSocket(true);
return;
}
if (code === 4403) {
this.revoke("Hub revoked this relationship");
return;
}
if (this.record?.state === "active") this.scheduleSocket(record);
if (this.record?.state === "active") this.scheduleSocket();
}
private socketFailed(generation: number, record: ActiveRecord, error: Error): void {
private socketFailed(generation: number, error: Error): void {
if (generation !== this.generation) return;
this.lastError = error.message;
if (this.record?.state === "active") this.scheduleSocket(record);
if (this.record?.state === "active") this.scheduleSocket();
}
private scheduleSocket(record: ActiveRecord): void {
private scheduleSocket(permissionMismatch = false): void {
if (this.state === "connected" && !permissionMismatch) this.retryAttempt = 0;
this.state = "reconnecting";
this.schedule(() => this.openSocket(record, true));
this.schedule(() => this.openSocket(true));
}
private scheduleEnrollment(record: PendingRecord, enrollmentGeneration: number): void {
@@ -330,6 +330,27 @@ test.each([401, 403] as const)(
},
);
test("repeated permission mismatch closes back off until the Hub accepts the permissions", async () => {
const mismatch = {
closeCode: 4403,
reason: "daemon session permissions do not match enrollment",
};
const hub = await UpgradeRejectingHub.start([
mismatch,
mismatch,
{ closeCode: 1001, reason: "shutdown" },
]);
const clock = new ManualRelationshipClock();
const controller = await connectController(hub, clock);
await expect.poll(() => clock.scheduledAttempts()).toEqual([0]);
clock.runNext();
await expect.poll(() => clock.scheduledAttempts()).toEqual([0, 1]);
expect(controller.status().state).toBe("reconnecting");
clock.runNext();
await expect.poll(() => clock.scheduledAttempts()).toEqual([0, 1, 0]);
await controller.stop();
});
test("controller redials after a transient socket response and releases both failed upgrades", async () => {
const hub = await UpgradeRejectingHub.start([503, 401]);
const clock = new ManualRelationshipClock();
@@ -475,6 +496,8 @@ class SocketOutcomes {
}
}
type UpgradeOutcome = number | "stall" | { closeCode: number; reason: string };
class UpgradeRejectingHub {
private readonly server = createServer(async (request, response) => {
let body = "";
@@ -493,8 +516,8 @@ class UpgradeRejectingHub {
private attemptObserved = deferred<void>();
private attempts = 0;
private constructor(private readonly statuses: Array<number | "stall">) {
this.server.on("upgrade", (_request, socket) => {
private constructor(private readonly statuses: UpgradeOutcome[]) {
this.server.on("upgrade", (request, socket, head) => {
const attempt = ++this.attempts;
this.attemptObserved.resolve();
this.attemptObserved = deferred<void>();
@@ -506,6 +529,14 @@ class UpgradeRejectingHub {
released.resolve();
});
const status = this.statuses[attempt - 1] ?? this.statuses.at(-1) ?? 503;
if (typeof status === "object") {
const ws = new WebSocketServer({ noServer: true });
ws.handleUpgrade(request, socket, head, (peer) =>
peer.close(status.closeCode, status.reason),
);
ws.close();
return;
}
if (status === "stall") {
socket.resume();
return;
@@ -520,7 +551,7 @@ class UpgradeRejectingHub {
});
}
static async start(statuses: Array<number | "stall">): Promise<UpgradeRejectingHub> {
static async start(statuses: UpgradeOutcome[]): Promise<UpgradeRejectingHub> {
const hub = new UpgradeRejectingHub(statuses);
openUpgradeHubs.push(hub);
openServers.push(hub.server);
@@ -46,7 +46,7 @@ export interface HubSocketCredentials {
export interface HubSocketEvents {
connected(socket: WebSocketLike, sessionProtocol: "legacy" | "session-v1"): void;
rejected(statusCode: 401 | 403): void;
closed(code: number): void;
closed(code: number, reason?: string): void;
failed(error: Error): void;
}
@@ -227,11 +227,11 @@ export class DirectHubRelationshipRemote implements HubRelationshipRemote {
}
events.closed(1006);
});
socket.once("close", (code) => {
socket.once("close", (code, reason) => {
stopHeartbeat();
if (settled) return;
settled = true;
events.closed(code);
events.closed(code, reason.toString());
});
socket.once("error", fail);
return socket;
@@ -720,9 +720,9 @@ export class HubRelationshipHarness {
this.rejectLatestSocket(code);
}
closeLatestSocket(code: number): void {
closeLatestSocket(code: number, reason?: string): void {
const socket = this.latestSocket();
socket.events.closed(code);
socket.events.closed(code, reason);
socket.socket.close(code);
}