fix(hub): contain enrollment retry failures

packages/server/src/server/hub/relationship-controller.test.ts

@@ -3,6 +3,19 @@ import { platform } from "node:os";
 import { afterEach, describe, expect, test } from "vitest";
 import { HubRelationshipHarness } from "./test-utils/relationship-harness.js";
 
+async function captureUnhandledRejections(action: () => Promise<void>): Promise<unknown[]> {
+  const rejections: unknown[] = [];
+  const capture = (reason: unknown) => rejections.push(reason);
+  process.on("unhandledRejection", capture);
+  try {
+    await action();
+    await new Promise((resolve) => setImmediate(resolve));
+  } finally {
+    process.off("unhandledRejection", capture);
+  }
+  return rejections;
+}
+
 describe("Hub relationship", () => {
   let relationship: HubRelationshipHarness | null = null;
 
@@ -211,6 +224,24 @@ describe("Hub relationship", () => {
     expect(relationship.relationshipFile()).toBeNull();
   });
 
+  test("a scheduled enrollment rejection is contained after removing pending authority", async () => {
+    relationship = await HubRelationshipHarness.start();
+    relationship.holdEnrollment();
+    const connecting = relationship.beginConnect("expired-token");
+    await relationship.enrollmentBegins();
+    relationship.loseEnrollmentResponse();
+    await connecting.result;
+    relationship.rejectNextEnrollment(403);
+
+    const unhandledRejections = await captureUnhandledRejections(() => relationship!.retry());
+    const status = await relationship.status();
+
+    expect(status).toMatchObject({ state: "not_connected" });
+    expect(relationship.relationshipFile()).toBeNull();
+    expect(relationship.pendingRelationshipRetries()).toBe(0);
+    expect(unhandledRejections).toEqual([]);
+  });
+
   test.each(["{not-json", JSON.stringify({ version: 1, state: "unknown" })])(
     "an invalid relationship file is quarantined without blocking daemon startup",
     async (contents) => {

packages/server/src/server/hub/relationship-controller.ts

@@ -394,7 +394,10 @@ export class HubRelationshipController implements HubRelationshipManagement {
     const delay = this.retryPolicy.delay(this.retryAttempt++);
     this.retry = this.clock.schedule(delay, () => {
       if (enrollmentGeneration !== this.enrollmentGeneration) return;
-      void this.tryEnrollment(record, enrollmentGeneration);
+      void this.tryEnrollment(record, enrollmentGeneration).catch((error: unknown) => {
+        if (error instanceof HubEnrollmentRejectedError) return;
+        this.options.logger.error({ err: error }, "Scheduled Hub enrollment retry failed");
+      });
     });
   }
 

packages/server/src/server/hub/test-utils/relationship-harness.ts

@@ -1283,13 +1283,18 @@ export class HubRelationshipHarness {
   }
 
   private async removeRoot(): Promise<void> {
-    const attempts = platform() === "win32" ? 10 : 1;
+    let retryableCode: string | null = null;
+    if (platform() === "win32") retryableCode = "EBUSY";
+    if (platform() === "darwin") retryableCode = "ENOTEMPTY";
+    const attempts = retryableCode ? 10 : 1;
     for (let attempt = 1; attempt <= attempts; attempt++) {
       try {
         await rm(this.root, { recursive: true, force: true });
         return;
       } catch (error) {
-        if ((error as NodeJS.ErrnoException).code !== "EBUSY" || attempt === attempts) throw error;
+        if ((error as NodeJS.ErrnoException).code !== retryableCode || attempt === attempts) {
+          throw error;
+        }
         await new Promise((resolve) => setTimeout(resolve, 100 * attempt));
       }
     }