diff --git a/packages/app/src/runtime/host-runtime.test.ts b/packages/app/src/runtime/host-runtime.test.ts index c802a945c3..1981a68975 100644 --- a/packages/app/src/runtime/host-runtime.test.ts +++ b/packages/app/src/runtime/host-runtime.test.ts @@ -2149,6 +2149,68 @@ describe("HostRuntimeController", () => { }); }); + it("wakes an active routed client before an unrelated direct probe settles", async () => { + useHostRuntimeClock(); + const route: HostConnection = { + id: "via:srv_parent", + type: "routed", + originServerId: "srv_parent", + }; + const host = makeHost({ connections: [route] }); + const client = makeConnectedProbeClient(5); + const directProbeStarted = createDeferred(); + const directProbe = createDeferred(); + let directProbeCalls = 0; + const controller = new HostRuntimeController({ + host, + deps: { + getClientId: async () => "blocked-direct-probe", + createClient: () => new FakeDaemonClient() as unknown as DaemonClient, + connectToDaemon: async ({ connection }) => { + if (connection.type === "directTcp") { + directProbeCalls += 1; + directProbeStarted.resolve(); + await directProbe.promise; + throw new Error("Direct connection unavailable"); + } + return { + client: client as unknown as DaemonClient, + serverId: host.serverId, + hostname: host.label, + }; + }, + }, + }); + await controller.start({ autoProbe: false }); + client.setConnectionState({ + status: "disconnected", + reason: "origin offline", + nextRetryAt: new Date(Date.now() + 12000).toISOString(), + }); + const probing = controller.updateHost({ + ...host, + connections: [ + route, + { id: "direct:unreachable", type: "directTcp", endpoint: "unreachable:6767" }, + ], + }); + await directProbeStarted.promise; + const recovering = controller.reconnectVia("srv_parent"); + try { + expect(client.ensureConnectedCalls).toBe(1); + expect(controller.getSnapshot().connectionStatus).toBe("online"); + expect(directProbeCalls).toBe(1); + await vi.advanceTimersByTimeAsync(0); + expect(client.ensureConnectedCalls).toBe(1); + expect(directProbeCalls).toBe(1); + } finally { + directProbe.resolve(); + await Promise.all([probing, recovering]); + await controller.stop(); + } + expect(directProbeCalls).toBe(1); + }); + it("keeps active client generation stable during background probe cycles", async () => { useHostRuntimeClock(); const host = makeHost({ @@ -2195,6 +2257,146 @@ describe("HostRuntimeController", () => { }); describe("HostRuntimeStore", () => { + function gatewayRecoveryFixture() { + const parent = new FakeDaemonClient(); + parent.connectedHosts = true; + parent.hostDirectory = Promise.resolve({ + self: { serverId: "srv_parent", label: "parent" }, + hosts: ["srv_child", "srv_healthy"].map((serverId) => ({ + registrationId: `reg_${serverId}`, + serverId, + label: serverId, + source: { id: "user", key: serverId }, + metadata: {}, + enabled: true, + status: "online" as const, + targetDescription: "relay", + lastOnlineAt: null, + nextRetryAt: null, + error: null, + })), + }); + const parentReady = createDeferred(); + const child = makeConnectedProbeClient(5); + const healthy = makeConnectedProbeClient(5); + const unrelated = makeConnectedProbeClient(5); + const attempts = new Map(); + const hosts = [ + makeHost({ + serverId: "srv_parent", + connections: [ + { + id: "relay:parent", + type: "relay", + relayEndpoint: "relay.paseo.sh:443", + daemonPublicKeyB64: "pk_parent", + }, + ], + }), + ...["srv_child", "srv_healthy"].map((serverId) => + makeHost({ + serverId, + connections: [ + { + id: "via:srv_parent", + type: "routed" as const, + originServerId: "srv_parent", + }, + ], + }), + ), + makeHost({ + serverId: "srv_unrelated", + connections: [ + { + id: "direct:unrelated:6767", + type: "directTcp", + endpoint: "unrelated:6767", + }, + ], + }), + ]; + const store = new HostRuntimeStore({ + storage: createMemoryHostRuntimeStorage({ + "@paseo:e2e": "1", + "@paseo:daemon-registry": JSON.stringify(hosts), + }), + deps: { + getClientId: async () => "gateway-recovery", + createClient: () => new FakeDaemonClient() as unknown as DaemonClient, + connectToDaemon: async ({ host, connection }) => { + attempts.set(host.serverId, (attempts.get(host.serverId) ?? 0) + 1); + let client: FakeDaemonClient; + if (host.serverId === "srv_parent") { + await parentReady.promise; + parent.setConnectionState({ status: "connected" }); + client = parent; + } else if (connection.type === "routed") { + if (!parent.isConnected) throw new Error("Origin host srv_parent is unavailable"); + client = host.serverId === "srv_child" ? child : healthy; + } else { + client = unrelated; + } + return { + client: client as unknown as DaemonClient, + serverId: host.serverId, + hostname: host.label, + }; + }, + }, + }); + return { store, parent, parentReady, child, healthy, unrelated, attempts }; + } + + it("retries saved child routes immediately when their relay gateway first connects", async () => { + useHostRuntimeClock(); + const { store, parentReady, attempts } = gatewayRecoveryFixture(); + try { + await store.boot(); + await vi.advanceTimersByTimeAsync(100); + expect(attempts.get("srv_child")).toBe(1); + expect(store.getSnapshot("srv_child")?.connectionStatus).not.toBe("online"); + parentReady.resolve(); + await vi.advanceTimersByTimeAsync(0); + expect(store.getSnapshot("srv_parent")?.connectionStatus).toBe("online"); + expect(store.getSnapshot("srv_child")?.connectionStatus).toBe("online"); + expect(store.getSnapshot("srv_healthy")?.connectionStatus).toBe("online"); + expect(attempts.get("srv_child")).toBe(2); + expect(attempts.get("srv_healthy")).toBe(2); + expect(attempts.get("srv_unrelated")).toBe(1); + } finally { + store.syncHosts([]); + } + }); + + it("wakes disconnected routed clients on gateway recovery without disturbing healthy hosts", async () => { + useHostRuntimeClock(); + const { store, parent, parentReady, child, healthy, unrelated } = gatewayRecoveryFixture(); + try { + parentReady.resolve(); + await store.boot(); + await vi.advanceTimersByTimeAsync(2000); + expect(store.getSnapshot("srv_child")?.connectionStatus).toBe("online"); + parent.setConnectionState({ status: "disconnected", reason: "relay lost" }); + child.setConnectionState({ + status: "disconnected", + reason: "origin offline", + nextRetryAt: new Date(Date.now() + 12000).toISOString(), + }); + parent.setConnectionState({ status: "connected" }); + await vi.advanceTimersByTimeAsync(0); + expect(child.ensureConnectedCalls).toBe(1); + expect(store.getSnapshot("srv_child")?.connectionStatus).toBe("online"); + expect(healthy.ensureConnectedCalls).toBe(0); + expect(unrelated.ensureConnectedCalls).toBe(0); + parent.setConnectionState({ status: "connected" }); + await vi.advanceTimersByTimeAsync(0); + expect(child.ensureConnectedCalls).toBe(1); + } finally { + store.syncHosts([]); + } + }); + it.each([false, true])( "keeps B and C distinct with routed and direct candidates (B direct first: %s)", async (directFirst) => { diff --git a/packages/app/src/runtime/host-runtime.ts b/packages/app/src/runtime/host-runtime.ts index 15a86b7bb4..b2d6871f0c 100644 --- a/packages/app/src/runtime/host-runtime.ts +++ b/packages/app/src/runtime/host-runtime.ts @@ -869,6 +869,22 @@ export class HostRuntimeController { this.activeClient?.ensureConnected(options); } + async reconnectVia(originServerId: string): Promise { + const routes = this.host.connections.filter( + (connection) => + connection.type === "routed" && + connection.originServerId === originServerId && + !this.authRejectedConnectionIds.has(connection.id), + ); + if (routes.length === 0 || !this.started || isHostRuntimeConnected(this.snapshot)) return; + if (routes.some((route) => route.id === this.snapshot.activeConnectionId)) + this.ensureConnected(); + await this.probeCycleInFlight; + if (!this.started || this.snapshot.connectionStatus === "online") return; + for (const route of routes) this.connectionLastProbedAt.delete(route.id); + await this.runProbeCycleNow(); + } + markAgentDirectorySyncLoading(): void { const status = this.snapshot.hasEverLoadedAgentDirectory ? "revalidating" : "initial_loading"; this.updateSnapshot({ @@ -2601,6 +2617,7 @@ export class HostRuntimeStore { snapshot.connectionStatus === "online" && previousStatus !== "online"; if (didTransitionOnline) { useSessionStore.getState().bumpHistorySyncGeneration(serverId); + for (const child of this.controllers.values()) void child.reconnectVia(serverId); // Checkout git data is push-driven; pushes emitted while disconnected are gone for // good (the daemon dedupes by snapshot fingerprint). Mark the caches stale so active // queries refetch now and evicted ones on their next mount. diff --git a/packages/client/src/daemon-client.test.ts b/packages/client/src/daemon-client.test.ts index 2ddef8cc61..057e6550c8 100644 --- a/packages/client/src/daemon-client.test.ts +++ b/packages/client/src/daemon-client.test.ts @@ -12,6 +12,7 @@ import { type PluginClientBundle, type PluginClientBundleStore, } from "./daemon-client"; +import { createHostChannelTransport } from "./hosts/index.js"; import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities"; import { ChunkedTransport } from "@getpaseo/protocol/chunked-transport"; import { BROWSER_AUTOMATION_COMMAND_NAMES } from "@getpaseo/protocol/browser-automation/rpc-schemas"; @@ -9562,7 +9563,6 @@ test.each(["latency", "dictation start", "dictation finish"])( ); test("routed channels inherit an origin without drain support", async () => { - const { createHostChannelTransport } = await import("./hosts/index.js"); const mock = createMockTransport(); mock.transport.supportsDrain = false; const client = new DaemonClient({ @@ -10133,3 +10133,69 @@ describe("provider options across daemon versions", () => { }); }); }); + +test.each([ + { name: "route", initialRouteId: null, link: false }, + { name: "link", initialRouteId: null, link: true }, + { name: "initial route", initialRouteId: "route", link: false }, +])( + "$name fails promptly when its origin is offline during initialization", + async ({ initialRouteId, link }) => { + vi.useFakeTimers(); + const origin = new DaemonClient({ + url: "ws://origin", + clientId: "offline-origin", + transportFactory: () => createMockTransport().transport, + reconnect: { enabled: false }, + logger: createMockLogger(), + }); + clients.push(origin); + const unsubscribe = vi.fn(); + const subscribe = origin.subscribeConnectionStatus.bind(origin); + vi.spyOn(origin, "subscribeConnectionStatus").mockImplementation((listener) => { + const release = subscribe(listener); + return () => { + release(); + unsubscribe(); + }; + }); + const child = new DaemonClient({ + url: "ws://routed-host/ws", + clientId: "offline-origin-child", + transportFactory: createHostChannelTransport(origin, "child", "scope", initialRouteId, link), + reconnect: { enabled: false }, + logger: createMockLogger(), + }); + clients.push(child); + const rejected = expect(child.connect()).rejects.toThrow(); + await vi.advanceTimersByTimeAsync(0); + expect(child.getConnectionState().status).toBe("disconnected"); + await rejected; + expect(unsubscribe).toHaveBeenCalledTimes(1); + }, +); + +test("disposing a route before its deferred close does not reopen it on origin recovery", async () => { + vi.useFakeTimers(); + const mock = createMockTransport(); + const origin = new DaemonClient({ + url: "ws://origin", + clientId: "disposed-route-origin", + transportFactory: () => mock.transport, + reconnect: { enabled: false }, + logger: createMockLogger(), + }); + clients.push(origin); + const route = createHostChannelTransport(origin, "child", "scope", null)({ url: "ws://route" }); + const opened = vi.fn(); + const closed = vi.fn(); + route.onOpen(opened); + route.onClose(closed); + route.close(); + const connecting = origin.connect(); + mock.triggerOpen({ features: { ownedSubscriptions: true, connectedHosts: true } }); + await connecting; + await vi.advanceTimersByTimeAsync(0); + expect(opened).not.toHaveBeenCalled(); + expect(closed).toHaveBeenCalledTimes(1); +}); diff --git a/packages/client/src/hosts/index.ts b/packages/client/src/hosts/index.ts index 150cf48071..47504f0ebe 100644 --- a/packages/client/src/hosts/index.ts +++ b/packages/client/src/hosts/index.ts @@ -86,7 +86,8 @@ export function createHostChannelTransport( ); cleanups.push( origin.subscribeConnectionStatus((connectionState) => { - if (connectionState.status !== "connected") close(); + // The initial status arrives before the client installs transport handlers. + if (connectionState.status !== "connected") queueMicrotask(close); }), ); let opening: Promise;