fix(client): recover routed hosts when their gateway reconnects (#9558)

* client: recover routed hosts when their gateway reconnects

Assisted-By: OpenAI GPT-6.1 Sol high in omp/18.16.0

* client: wake routed clients before waiting for probes

Assisted-By: OpenAI GPT-6.1 Sol high in omp/18.18.9
This commit is contained in:
Adam C. Stephens
2026-11-15 07:06:52 +00:00
committed by GitHub
parent fee32c9594
commit dc2dfdab93
4 changed files with 288 additions and 2 deletions
@@ -2149,6 +2149,68 @@ describe("HostRuntimeController", () => {
});
});
it("wakes an active routed client before an unrelated direct probe settles", async () => {
useHostRuntimeClock();
const route: HostConnection = {
id: "via:srv_parent",
type: "routed",
originServerId: "srv_parent",
};
const host = makeHost({ connections: [route] });
const client = makeConnectedProbeClient(5);
const directProbeStarted = createDeferred<void>();
const directProbe = createDeferred<void>();
let directProbeCalls = 0;
const controller = new HostRuntimeController({
host,
deps: {
getClientId: async () => "blocked-direct-probe",
createClient: () => new FakeDaemonClient() as unknown as DaemonClient,
connectToDaemon: async ({ connection }) => {
if (connection.type === "directTcp") {
directProbeCalls += 1;
directProbeStarted.resolve();
await directProbe.promise;
throw new Error("Direct connection unavailable");
}
return {
client: client as unknown as DaemonClient,
serverId: host.serverId,
hostname: host.label,
};
},
},
});
await controller.start({ autoProbe: false });
client.setConnectionState({
status: "disconnected",
reason: "origin offline",
nextRetryAt: new Date(Date.now() + 12000).toISOString(),
});
const probing = controller.updateHost({
...host,
connections: [
route,
{ id: "direct:unreachable", type: "directTcp", endpoint: "unreachable:6767" },
],
});
await directProbeStarted.promise;
const recovering = controller.reconnectVia("srv_parent");
try {
expect(client.ensureConnectedCalls).toBe(1);
expect(controller.getSnapshot().connectionStatus).toBe("online");
expect(directProbeCalls).toBe(1);
await vi.advanceTimersByTimeAsync(0);
expect(client.ensureConnectedCalls).toBe(1);
expect(directProbeCalls).toBe(1);
} finally {
directProbe.resolve();
await Promise.all([probing, recovering]);
await controller.stop();
}
expect(directProbeCalls).toBe(1);
});
it("keeps active client generation stable during background probe cycles", async () => {
useHostRuntimeClock();
const host = makeHost({
@@ -2195,6 +2257,146 @@ describe("HostRuntimeController", () => {
});
describe("HostRuntimeStore", () => {
function gatewayRecoveryFixture() {
const parent = new FakeDaemonClient();
parent.connectedHosts = true;
parent.hostDirectory = Promise.resolve({
self: { serverId: "srv_parent", label: "parent" },
hosts: ["srv_child", "srv_healthy"].map((serverId) => ({
registrationId: `reg_${serverId}`,
serverId,
label: serverId,
source: { id: "user", key: serverId },
metadata: {},
enabled: true,
status: "online" as const,
targetDescription: "relay",
lastOnlineAt: null,
nextRetryAt: null,
error: null,
})),
});
const parentReady = createDeferred<void>();
const child = makeConnectedProbeClient(5);
const healthy = makeConnectedProbeClient(5);
const unrelated = makeConnectedProbeClient(5);
const attempts = new Map<string, number>();
const hosts = [
makeHost({
serverId: "srv_parent",
connections: [
{
id: "relay:parent",
type: "relay",
relayEndpoint: "relay.paseo.sh:443",
daemonPublicKeyB64: "pk_parent",
},
],
}),
...["srv_child", "srv_healthy"].map((serverId) =>
makeHost({
serverId,
connections: [
{
id: "via:srv_parent",
type: "routed" as const,
originServerId: "srv_parent",
},
],
}),
),
makeHost({
serverId: "srv_unrelated",
connections: [
{
id: "direct:unrelated:6767",
type: "directTcp",
endpoint: "unrelated:6767",
},
],
}),
];
const store = new HostRuntimeStore({
storage: createMemoryHostRuntimeStorage({
"@paseo:e2e": "1",
"@paseo:daemon-registry": JSON.stringify(hosts),
}),
deps: {
getClientId: async () => "gateway-recovery",
createClient: () => new FakeDaemonClient() as unknown as DaemonClient,
connectToDaemon: async ({ host, connection }) => {
attempts.set(host.serverId, (attempts.get(host.serverId) ?? 0) + 1);
let client: FakeDaemonClient;
if (host.serverId === "srv_parent") {
await parentReady.promise;
parent.setConnectionState({ status: "connected" });
client = parent;
} else if (connection.type === "routed") {
if (!parent.isConnected) throw new Error("Origin host srv_parent is unavailable");
client = host.serverId === "srv_child" ? child : healthy;
} else {
client = unrelated;
}
return {
client: client as unknown as DaemonClient,
serverId: host.serverId,
hostname: host.label,
};
},
},
});
return { store, parent, parentReady, child, healthy, unrelated, attempts };
}
it("retries saved child routes immediately when their relay gateway first connects", async () => {
useHostRuntimeClock();
const { store, parentReady, attempts } = gatewayRecoveryFixture();
try {
await store.boot();
await vi.advanceTimersByTimeAsync(100);
expect(attempts.get("srv_child")).toBe(1);
expect(store.getSnapshot("srv_child")?.connectionStatus).not.toBe("online");
parentReady.resolve();
await vi.advanceTimersByTimeAsync(0);
expect(store.getSnapshot("srv_parent")?.connectionStatus).toBe("online");
expect(store.getSnapshot("srv_child")?.connectionStatus).toBe("online");
expect(store.getSnapshot("srv_healthy")?.connectionStatus).toBe("online");
expect(attempts.get("srv_child")).toBe(2);
expect(attempts.get("srv_healthy")).toBe(2);
expect(attempts.get("srv_unrelated")).toBe(1);
} finally {
store.syncHosts([]);
}
});
it("wakes disconnected routed clients on gateway recovery without disturbing healthy hosts", async () => {
useHostRuntimeClock();
const { store, parent, parentReady, child, healthy, unrelated } = gatewayRecoveryFixture();
try {
parentReady.resolve();
await store.boot();
await vi.advanceTimersByTimeAsync(2000);
expect(store.getSnapshot("srv_child")?.connectionStatus).toBe("online");
parent.setConnectionState({ status: "disconnected", reason: "relay lost" });
child.setConnectionState({
status: "disconnected",
reason: "origin offline",
nextRetryAt: new Date(Date.now() + 12000).toISOString(),
});
parent.setConnectionState({ status: "connected" });
await vi.advanceTimersByTimeAsync(0);
expect(child.ensureConnectedCalls).toBe(1);
expect(store.getSnapshot("srv_child")?.connectionStatus).toBe("online");
expect(healthy.ensureConnectedCalls).toBe(0);
expect(unrelated.ensureConnectedCalls).toBe(0);
parent.setConnectionState({ status: "connected" });
await vi.advanceTimersByTimeAsync(0);
expect(child.ensureConnectedCalls).toBe(1);
} finally {
store.syncHosts([]);
}
});
it.each([false, true])(
"keeps B and C distinct with routed and direct candidates (B direct first: %s)",
async (directFirst) => {
+17
View File
@@ -869,6 +869,22 @@ export class HostRuntimeController {
this.activeClient?.ensureConnected(options);
}
async reconnectVia(originServerId: string): Promise<void> {
const routes = this.host.connections.filter(
(connection) =>
connection.type === "routed" &&
connection.originServerId === originServerId &&
!this.authRejectedConnectionIds.has(connection.id),
);
if (routes.length === 0 || !this.started || isHostRuntimeConnected(this.snapshot)) return;
if (routes.some((route) => route.id === this.snapshot.activeConnectionId))
this.ensureConnected();
await this.probeCycleInFlight;
if (!this.started || this.snapshot.connectionStatus === "online") return;
for (const route of routes) this.connectionLastProbedAt.delete(route.id);
await this.runProbeCycleNow();
}
markAgentDirectorySyncLoading(): void {
const status = this.snapshot.hasEverLoadedAgentDirectory ? "revalidating" : "initial_loading";
this.updateSnapshot({
@@ -2601,6 +2617,7 @@ export class HostRuntimeStore {
snapshot.connectionStatus === "online" && previousStatus !== "online";
if (didTransitionOnline) {
useSessionStore.getState().bumpHistorySyncGeneration(serverId);
for (const child of this.controllers.values()) void child.reconnectVia(serverId);
// Checkout git data is push-driven; pushes emitted while disconnected are gone for
// good (the daemon dedupes by snapshot fingerprint). Mark the caches stale so active
// queries refetch now and evicted ones on their next mount.
+67 -1
View File
@@ -12,6 +12,7 @@ import {
type PluginClientBundle,
type PluginClientBundleStore,
} from "./daemon-client";
import { createHostChannelTransport } from "./hosts/index.js";
import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities";
import { ChunkedTransport } from "@getpaseo/protocol/chunked-transport";
import { BROWSER_AUTOMATION_COMMAND_NAMES } from "@getpaseo/protocol/browser-automation/rpc-schemas";
@@ -9562,7 +9563,6 @@ test.each(["latency", "dictation start", "dictation finish"])(
);
test("routed channels inherit an origin without drain support", async () => {
const { createHostChannelTransport } = await import("./hosts/index.js");
const mock = createMockTransport();
mock.transport.supportsDrain = false;
const client = new DaemonClient({
@@ -10133,3 +10133,69 @@ describe("provider options across daemon versions", () => {
});
});
});
test.each([
{ name: "route", initialRouteId: null, link: false },
{ name: "link", initialRouteId: null, link: true },
{ name: "initial route", initialRouteId: "route", link: false },
])(
"$name fails promptly when its origin is offline during initialization",
async ({ initialRouteId, link }) => {
vi.useFakeTimers();
const origin = new DaemonClient({
url: "ws://origin",
clientId: "offline-origin",
transportFactory: () => createMockTransport().transport,
reconnect: { enabled: false },
logger: createMockLogger(),
});
clients.push(origin);
const unsubscribe = vi.fn();
const subscribe = origin.subscribeConnectionStatus.bind(origin);
vi.spyOn(origin, "subscribeConnectionStatus").mockImplementation((listener) => {
const release = subscribe(listener);
return () => {
release();
unsubscribe();
};
});
const child = new DaemonClient({
url: "ws://routed-host/ws",
clientId: "offline-origin-child",
transportFactory: createHostChannelTransport(origin, "child", "scope", initialRouteId, link),
reconnect: { enabled: false },
logger: createMockLogger(),
});
clients.push(child);
const rejected = expect(child.connect()).rejects.toThrow();
await vi.advanceTimersByTimeAsync(0);
expect(child.getConnectionState().status).toBe("disconnected");
await rejected;
expect(unsubscribe).toHaveBeenCalledTimes(1);
},
);
test("disposing a route before its deferred close does not reopen it on origin recovery", async () => {
vi.useFakeTimers();
const mock = createMockTransport();
const origin = new DaemonClient({
url: "ws://origin",
clientId: "disposed-route-origin",
transportFactory: () => mock.transport,
reconnect: { enabled: false },
logger: createMockLogger(),
});
clients.push(origin);
const route = createHostChannelTransport(origin, "child", "scope", null)({ url: "ws://route" });
const opened = vi.fn();
const closed = vi.fn();
route.onOpen(opened);
route.onClose(closed);
route.close();
const connecting = origin.connect();
mock.triggerOpen({ features: { ownedSubscriptions: true, connectedHosts: true } });
await connecting;
await vi.advanceTimersByTimeAsync(0);
expect(opened).not.toHaveBeenCalled();
expect(closed).toHaveBeenCalledTimes(1);
});
+2 -1
View File
@@ -86,7 +86,8 @@ export function createHostChannelTransport(
);
cleanups.push(
origin.subscribeConnectionStatus((connectionState) => {
if (connectionState.status !== "connected") close();
// The initial status arrives before the client installs transport handlers.
if (connectionState.status !== "connected") queueMicrotask(close);
}),
);
let opening: Promise<string>;