mirror of
https://github.com/getpaseo/paseo.git
synced 2026-11-19 23:05:21 +00:00
Keep Hub daemons connected after execution permissions change (#10113)
* Use current Hub permissions when reconnecting a daemon
* Back off when the Hub repeatedly rejects session permissions
* Show the permission recovery commands in Hub status
(cherry picked from commit 0e67045c4b)
This commit is contained in:
@@ -153,6 +153,40 @@ describe("Hub relationship", () => {
|
||||
});
|
||||
});
|
||||
|
||||
test.each([false, true])(
|
||||
"reconnect advertises the current execution grant (initially %s)",
|
||||
async (initiallyGranted) => {
|
||||
relationship = await HubRelationshipHarness.start();
|
||||
await relationship.beginConnect("token", "https://hub.example", initiallyGranted).result;
|
||||
relationship.connectLatestSocket();
|
||||
if (initiallyGranted) await relationship.revokePermission("hub.execute");
|
||||
else await relationship.grantPermission("hub.execute");
|
||||
relationship.closeLatestSocket(1006);
|
||||
await relationship.retry();
|
||||
relationship.connectLatestSocket();
|
||||
|
||||
expect(relationship.serverInfoPermissions().at(-1)).toEqual(
|
||||
initiallyGranted ? [] : ["hub.execute"],
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
test("a permission mismatch retries the handshake with the same enrollment", async () => {
|
||||
relationship = await HubRelationshipHarness.start();
|
||||
await relationship.beginConnect().result;
|
||||
relationship.connectLatestSocket();
|
||||
const reason = "daemon session permissions do not match enrollment";
|
||||
relationship.closeLatestSocket(4403, reason);
|
||||
expect(await relationship.status()).toMatchObject({
|
||||
state: "reconnecting",
|
||||
lastError: expect.stringContaining("paseo hub permissions"),
|
||||
});
|
||||
await relationship.retry();
|
||||
await relationship.revokePermission("hub.execute");
|
||||
relationship.connectLatestSocket();
|
||||
expect(relationship.serverInfoPermissions().at(-1)).toEqual([]);
|
||||
});
|
||||
|
||||
test("Hub enrollment cannot widen the daemon's locally granted permissions", async () => {
|
||||
relationship = await HubRelationshipHarness.start();
|
||||
relationship.returnEnrollmentPermissions(["hub.execute", "daemon.manage"]);
|
||||
|
||||
@@ -246,7 +246,7 @@ export class HubRelationshipController implements HubRelationshipManagement {
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
if (this.record?.state === "active") this.openSocket(this.record, false);
|
||||
if (this.record?.state === "active") this.openSocket(false);
|
||||
if (this.record?.state === "pending") {
|
||||
const enrollmentGeneration = this.beginEnrollmentAttempt();
|
||||
try {
|
||||
@@ -441,7 +441,7 @@ export class HubRelationshipController implements HubRelationshipManagement {
|
||||
this.record = active;
|
||||
this.retry = null;
|
||||
this.retryAttempt = 0;
|
||||
this.openSocket(active, false);
|
||||
this.openSocket(false);
|
||||
} catch (error) {
|
||||
if (enrollmentGeneration !== this.enrollmentGeneration) return;
|
||||
if (error instanceof HubEnrollmentRejectedError) {
|
||||
@@ -455,15 +455,17 @@ export class HubRelationshipController implements HubRelationshipManagement {
|
||||
}
|
||||
}
|
||||
|
||||
private openSocket(record: ActiveRecord, reconnecting: boolean): void {
|
||||
private openSocket(reconnecting: boolean): void {
|
||||
const record = this.record;
|
||||
if (record?.state !== "active") return;
|
||||
const generation = ++this.generation;
|
||||
this.state = reconnecting ? "reconnecting" : "connecting";
|
||||
const events: HubSocketEvents = {
|
||||
connected: (socket, sessionProtocol) =>
|
||||
this.socketConnected(generation, record, socket, sessionProtocol),
|
||||
this.socketConnected(generation, socket, sessionProtocol),
|
||||
rejected: (statusCode) => this.socketRejected(generation, statusCode),
|
||||
closed: (code) => this.socketClosed(generation, record, code),
|
||||
failed: (error) => this.socketFailed(generation, record, error),
|
||||
closed: (code, reason) => this.socketClosed(generation, code, reason),
|
||||
failed: (error) => this.socketFailed(generation, error),
|
||||
};
|
||||
this.socket = this.options.remote.openSocket(
|
||||
{
|
||||
@@ -477,15 +479,14 @@ export class HubRelationshipController implements HubRelationshipManagement {
|
||||
|
||||
private socketConnected(
|
||||
generation: number,
|
||||
record: ActiveRecord,
|
||||
socket: WebSocketLike,
|
||||
sessionProtocol: "legacy" | "session-v1",
|
||||
): void {
|
||||
if (generation !== this.generation) {
|
||||
const record = this.record;
|
||||
if (generation !== this.generation || record?.state !== "active") {
|
||||
socket.close();
|
||||
return;
|
||||
}
|
||||
this.retryAttempt = 0;
|
||||
this.state = "connected";
|
||||
this.connectedAt = this.clock.now().toISOString();
|
||||
this.lastError = null;
|
||||
@@ -516,24 +517,31 @@ export class HubRelationshipController implements HubRelationshipManagement {
|
||||
this.revoke(`Hub rejected socket authentication (${statusCode})`);
|
||||
}
|
||||
|
||||
private socketClosed(generation: number, record: ActiveRecord, code: number): void {
|
||||
private socketClosed(generation: number, code: number, reason?: string): void {
|
||||
if (generation !== this.generation) return;
|
||||
if (code === 4403 && reason === "daemon session permissions do not match enrollment") {
|
||||
this.lastError =
|
||||
"Hub and daemon permissions differ. Run `paseo hub permissions grant <permission>` or `paseo hub permissions revoke <permission>` on this daemon to reapply your intended permissions.";
|
||||
this.scheduleSocket(true);
|
||||
return;
|
||||
}
|
||||
if (code === 4403) {
|
||||
this.revoke("Hub revoked this relationship");
|
||||
return;
|
||||
}
|
||||
if (this.record?.state === "active") this.scheduleSocket(record);
|
||||
if (this.record?.state === "active") this.scheduleSocket();
|
||||
}
|
||||
|
||||
private socketFailed(generation: number, record: ActiveRecord, error: Error): void {
|
||||
private socketFailed(generation: number, error: Error): void {
|
||||
if (generation !== this.generation) return;
|
||||
this.lastError = error.message;
|
||||
if (this.record?.state === "active") this.scheduleSocket(record);
|
||||
if (this.record?.state === "active") this.scheduleSocket();
|
||||
}
|
||||
|
||||
private scheduleSocket(record: ActiveRecord): void {
|
||||
private scheduleSocket(permissionMismatch = false): void {
|
||||
if (this.state === "connected" && !permissionMismatch) this.retryAttempt = 0;
|
||||
this.state = "reconnecting";
|
||||
this.schedule(() => this.openSocket(record, true));
|
||||
this.schedule(() => this.openSocket(true));
|
||||
}
|
||||
|
||||
private scheduleEnrollment(record: PendingRecord, enrollmentGeneration: number): void {
|
||||
|
||||
@@ -330,6 +330,27 @@ test.each([401, 403] as const)(
|
||||
},
|
||||
);
|
||||
|
||||
test("repeated permission mismatch closes back off until the Hub accepts the permissions", async () => {
|
||||
const mismatch = {
|
||||
closeCode: 4403,
|
||||
reason: "daemon session permissions do not match enrollment",
|
||||
};
|
||||
const hub = await UpgradeRejectingHub.start([
|
||||
mismatch,
|
||||
mismatch,
|
||||
{ closeCode: 1001, reason: "shutdown" },
|
||||
]);
|
||||
const clock = new ManualRelationshipClock();
|
||||
const controller = await connectController(hub, clock);
|
||||
await expect.poll(() => clock.scheduledAttempts()).toEqual([0]);
|
||||
clock.runNext();
|
||||
await expect.poll(() => clock.scheduledAttempts()).toEqual([0, 1]);
|
||||
expect(controller.status().state).toBe("reconnecting");
|
||||
clock.runNext();
|
||||
await expect.poll(() => clock.scheduledAttempts()).toEqual([0, 1, 0]);
|
||||
await controller.stop();
|
||||
});
|
||||
|
||||
test("controller redials after a transient socket response and releases both failed upgrades", async () => {
|
||||
const hub = await UpgradeRejectingHub.start([503, 401]);
|
||||
const clock = new ManualRelationshipClock();
|
||||
@@ -475,6 +496,8 @@ class SocketOutcomes {
|
||||
}
|
||||
}
|
||||
|
||||
type UpgradeOutcome = number | "stall" | { closeCode: number; reason: string };
|
||||
|
||||
class UpgradeRejectingHub {
|
||||
private readonly server = createServer(async (request, response) => {
|
||||
let body = "";
|
||||
@@ -493,8 +516,8 @@ class UpgradeRejectingHub {
|
||||
private attemptObserved = deferred<void>();
|
||||
private attempts = 0;
|
||||
|
||||
private constructor(private readonly statuses: Array<number | "stall">) {
|
||||
this.server.on("upgrade", (_request, socket) => {
|
||||
private constructor(private readonly statuses: UpgradeOutcome[]) {
|
||||
this.server.on("upgrade", (request, socket, head) => {
|
||||
const attempt = ++this.attempts;
|
||||
this.attemptObserved.resolve();
|
||||
this.attemptObserved = deferred<void>();
|
||||
@@ -506,6 +529,14 @@ class UpgradeRejectingHub {
|
||||
released.resolve();
|
||||
});
|
||||
const status = this.statuses[attempt - 1] ?? this.statuses.at(-1) ?? 503;
|
||||
if (typeof status === "object") {
|
||||
const ws = new WebSocketServer({ noServer: true });
|
||||
ws.handleUpgrade(request, socket, head, (peer) =>
|
||||
peer.close(status.closeCode, status.reason),
|
||||
);
|
||||
ws.close();
|
||||
return;
|
||||
}
|
||||
if (status === "stall") {
|
||||
socket.resume();
|
||||
return;
|
||||
@@ -520,7 +551,7 @@ class UpgradeRejectingHub {
|
||||
});
|
||||
}
|
||||
|
||||
static async start(statuses: Array<number | "stall">): Promise<UpgradeRejectingHub> {
|
||||
static async start(statuses: UpgradeOutcome[]): Promise<UpgradeRejectingHub> {
|
||||
const hub = new UpgradeRejectingHub(statuses);
|
||||
openUpgradeHubs.push(hub);
|
||||
openServers.push(hub.server);
|
||||
|
||||
@@ -46,7 +46,7 @@ export interface HubSocketCredentials {
|
||||
export interface HubSocketEvents {
|
||||
connected(socket: WebSocketLike, sessionProtocol: "legacy" | "session-v1"): void;
|
||||
rejected(statusCode: 401 | 403): void;
|
||||
closed(code: number): void;
|
||||
closed(code: number, reason?: string): void;
|
||||
failed(error: Error): void;
|
||||
}
|
||||
|
||||
@@ -227,11 +227,11 @@ export class DirectHubRelationshipRemote implements HubRelationshipRemote {
|
||||
}
|
||||
events.closed(1006);
|
||||
});
|
||||
socket.once("close", (code) => {
|
||||
socket.once("close", (code, reason) => {
|
||||
stopHeartbeat();
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
events.closed(code);
|
||||
events.closed(code, reason.toString());
|
||||
});
|
||||
socket.once("error", fail);
|
||||
return socket;
|
||||
|
||||
@@ -720,9 +720,9 @@ export class HubRelationshipHarness {
|
||||
this.rejectLatestSocket(code);
|
||||
}
|
||||
|
||||
closeLatestSocket(code: number): void {
|
||||
closeLatestSocket(code: number, reason?: string): void {
|
||||
const socket = this.latestSocket();
|
||||
socket.events.closed(code);
|
||||
socket.events.closed(code, reason);
|
||||
socket.socket.close(code);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user