test(server): reproduce missed cross-client agent deletion

Retain the failing reproduction for #4870. Deletion reaches the requesting subscription but leaves other connected agent subscriptions stale. No production behavior is changed.

nix/npm-deps.hash

@@ -1 +1 @@
-sha256-O8sNh6oXFIddUljYiC6RbVZF5/7a7A7W9UWrOd0tIV4=
+sha256-tSE4nb4E06GFZ8I9UEbo93nlPiAKtW00nkywgKQmMhI=

packages/app/e2e/browser/sidebar-project-grouping.spec.ts

@@ -9,6 +9,8 @@ import {
   expectProjectWorkspaceCountForHost,
   expectSeparateProjects,
   openGroupedProjectSettings,
+  openGroupedProjectWorkspace,
+  expectWorkspaceHost,
   openProjectsForSettingsHost,
   openProjectDirectory,
   openProjectDirectoryWithHosts,
@@ -418,4 +420,29 @@ test.describe("Sidebar project grouping", () => {
       count: 2,
     });
   });
+  test("creates grouped project workspaces on the active workspace host", async ({
+    page,
+    crossHostProject,
+  }) => {
+    await openScenario(page, crossHostProject);
+    await openGroupedProjectWorkspace(page, GROUPED_PROJECT_NAME, "Secondary workspace");
+    await beginWorkspaceFromProject(page, GROUPED_PROJECT_NAME);
+    await expectWorkspaceHost(page, SECONDARY_HOST_LABEL);
+    await createWorkspaceWithoutAgent(page);
+    await expectProjectWorkspaceCountForHost(page, {
+      projectName: GROUPED_PROJECT_NAME,
+      hostName: SECONDARY_HOST_LABEL,
+      count: 2,
+    });
+
+    await openGroupedProjectWorkspace(page, GROUPED_PROJECT_NAME, "Primary workspace");
+    await beginWorkspaceFromProject(page, GROUPED_PROJECT_NAME);
+    await expectWorkspaceHost(page, PRIMARY_HOST_LABEL);
+    await createWorkspaceWithoutAgent(page);
+    await expectProjectWorkspaceCountForHost(page, {
+      projectName: GROUPED_PROJECT_NAME,
+      hostName: PRIMARY_HOST_LABEL,
+      count: 2,
+    });
+  });
 });

packages/app/e2e/support/helpers/project-grouping.ts

@@ -134,3 +134,19 @@ export async function expectProjectSettingsName(page: Page, projectName: string)
     timeout: PROJECT_VISIBILITY_TIMEOUT,
   });
 }
+
+export async function openGroupedProjectWorkspace(
+  page: Page,
+  projectName: string,
+  workspaceName: string,
+): Promise<void> {
+  await projectGroup(page, projectName)
+    .getByRole("button")
+    .filter({ hasText: workspaceName })
+    .click();
+  await expect(page).toHaveURL(/\/workspace\//);
+}
+
+export async function expectWorkspaceHost(page: Page, hostName: string): Promise<void> {
+  await expect(page.getByRole("button", { name: "Host", exact: true })).toContainText(hostName);
+}

packages/app/src/components/sidebar-workspace-list.tsx

@@ -41,7 +41,11 @@ import { ExternalLink, Settings, MoreVertical, Plus, Trash2 } from "lucide-react
 import { NestableScrollContainer } from "react-native-draggable-flatlist";
 import { DraggableList, type DraggableRenderItemInfo } from "./draggable-list";
 import type { DraggableListDragHandleProps } from "./draggable-list.types";
-import { getHostRuntimeStore, useHosts } from "@/runtime/host-runtime";
+import {
+  getHostRuntimeStore,
+  useHosts,
+  useHostRuntimeConnectionStatus,
+} from "@/runtime/host-runtime";
 import type { PinnedSidebarGroups } from "@/hooks/use-sidebar-pins";
 import {
   useSidebarWorkspacePinController,
@@ -1596,14 +1600,18 @@ function ProjectBlock({
     canToggle: canToggleWorkspaces,
     toggleExpanded: toggleWorkspacesExpanded,
   } = useLimitedSidebarGroup(project.workspaces);
+  const activeServerId = activeWorkspaceSelection?.serverId ?? null;
+  const activeHostStatus = useHostRuntimeConnectionStatus(activeServerId ?? "");
+  const preferredServerId = activeHostStatus === "online" ? activeServerId : null;
   const rowModel = useMemo(
     () =>
       buildSidebarProjectRowModel({
         project,
         collapsed,
         supportsMultiplicityByServerId,
+        preferredServerId,
       }),
-    [collapsed, project, supportsMultiplicityByServerId],
+    [collapsed, preferredServerId, project, supportsMultiplicityByServerId],
   );
 
   // Collapsed rows hide their workspace rows, so the project row carries the most urgent
@@ -1848,6 +1856,7 @@ function areProjectBlockPropsEqual(previous: ProjectBlockProps, next: ProjectBlo
     previous.useNestable === next.useNestable &&
     previous.dragGestureHostActive === next.dragGestureHostActive &&
     previous.creatingWorkspaceIds === next.creatingWorkspaceIds &&
+    previous.activeWorkspaceSelection?.serverId === next.activeWorkspaceSelection?.serverId &&
     areProjectBlockSelectionsEqual(previous, next)
   );
 }

packages/app/src/components/sidebar/empty-states.browser.test.tsx

@@ -0,0 +1,85 @@
+import React, { act } from "react";
+import { createRoot, type Root } from "react-dom/client";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { getAllByRole } from "@testing-library/dom";
+import { i18n as testI18n } from "@/i18n/i18next";
+import { DEFAULT_SIDEBAR_WIDTH, MIN_SIDEBAR_WIDTH } from "@/stores/panel-store";
+import { SidebarProjectEmptyState } from "./empty-states";
+
+// App sources compile against the classic JSX runtime, which expects React on the global.
+beforeEach(() => vi.stubGlobal("React", React));
+
+interface Mounted {
+  root: Root;
+  container: HTMLDivElement;
+}
+
+const mounted: Mounted[] = [];
+
+afterEach(async () => {
+  for (const entry of mounted.splice(0)) {
+    act(() => entry.root.unmount());
+    entry.container.remove();
+  }
+  await testI18n.changeLanguage("en");
+});
+
+async function mountEmptyState(language: string, width: number): Promise<HTMLElement> {
+  await testI18n.changeLanguage(language);
+
+  const container = document.createElement("div");
+  container.style.width = `${width}px`;
+  // Match the workspace list's horizontal inset around the card.
+  container.style.paddingInline = "6px";
+  container.style.boxSizing = "border-box";
+  document.body.appendChild(container);
+  const root = createRoot(container);
+  act(() => root.render(<SidebarProjectEmptyState />));
+  mounted.push({ root, container });
+
+  const card = container.querySelector('[data-testid="sidebar-project-empty-state"]');
+  if (!(card instanceof HTMLElement)) {
+    throw new Error("SidebarProjectEmptyState did not render its card");
+  }
+  return card;
+}
+
+function expectActionsInsideCard(card: HTMLElement) {
+  const cardRect = card.getBoundingClientRect();
+  const buttons = getAllByRole(card, "button");
+
+  expect(buttons).toHaveLength(2);
+  for (const button of buttons) {
+    const rect = button.getBoundingClientRect();
+    expect(rect.left).toBeGreaterThanOrEqual(cardRect.left);
+    expect(rect.right).toBeLessThanOrEqual(cardRect.right);
+
+    // Containing the button must also keep the whole label readable, without clipping.
+    const label = document.createRange();
+    label.selectNodeContents(button);
+    for (const fragment of Array.from(label.getClientRects())) {
+      expect(fragment.left).toBeGreaterThanOrEqual(rect.left);
+      expect(fragment.right).toBeLessThanOrEqual(rect.right);
+      expect(fragment.top).toBeGreaterThanOrEqual(rect.top);
+      expect(fragment.bottom).toBeLessThanOrEqual(rect.bottom);
+    }
+  }
+}
+
+describe("SidebarProjectEmptyState", () => {
+  const cases = ["en", "ru", "fr", "ja", "es", "pt-BR", "ar", "ko", "zh-CN"].flatMap((language) =>
+    [MIN_SIDEBAR_WIDTH, DEFAULT_SIDEBAR_WIDTH].map((width) => ({ language, width })),
+  );
+  it.each(cases)(
+    "keeps every action inside the card ($language, $width px sidebar)",
+    async ({ language, width }) => {
+      expectActionsInsideCard(await mountEmptyState(language, width));
+    },
+  );
+
+  it("keeps English actions on one row when they fit", async () => {
+    const card = await mountEmptyState("en", DEFAULT_SIDEBAR_WIDTH);
+    const [addProject, importSession] = getAllByRole(card, "button");
+    expect(addProject.getBoundingClientRect().top).toBe(importSession.getBoundingClientRect().top);
+  });
+});

packages/app/src/components/sidebar/empty-states.tsx

@@ -54,7 +54,13 @@ export function SidebarFilterEmptyState() {
       title={t("sidebar.filterEmpty.title")}
       description={t("sidebar.filterEmpty.description")}
     >
-      <Button variant="secondary" size="xs" onPress={clearFilters}>
+      <Button
+        variant="secondary"
+        size="xs"
+        style={styles.action}
+        textStyle={styles.actionText}
+        onPress={clearFilters}
+      >
         {t("sidebar.filterEmpty.clear")}
       </Button>
     </SidebarEmptyStateCard>
@@ -76,10 +82,24 @@ export function SidebarProjectEmptyState({
       title={t("sidebar.project.empty.title")}
       description={t("sidebar.project.empty.description")}
     >
-      <Button variant="secondary" size="xs" leftIcon={Plus} onPress={onAddProject}>
+      <Button
+        variant="secondary"
+        size="xs"
+        leftIcon={Plus}
+        style={styles.action}
+        textStyle={styles.actionText}
+        onPress={onAddProject}
+      >
         {t("sidebar.actions.addProject")}
       </Button>
-      <Button variant="outline" size="xs" leftIcon={Import} onPress={onImportSession}>
+      <Button
+        variant="outline"
+        size="xs"
+        leftIcon={Import}
+        style={styles.action}
+        textStyle={styles.actionText}
+        onPress={onImportSession}
+      >
         {t("importSession.title")}
       </Button>
     </SidebarEmptyStateCard>
@@ -99,9 +119,16 @@ const styles = StyleSheet.create((theme) => ({
   },
   actions: {
     flexDirection: "row",
+    flexWrap: "wrap",
     gap: theme.spacing[2],
     marginTop: theme.spacing[4],
   },
+  action: {
+    maxWidth: "100%",
+  },
+  actionText: {
+    flexShrink: 1,
+  },
   title: {
     color: theme.colors.foreground,
     fontSize: theme.fontSize.sm,

packages/app/src/utils/sidebar-project-row-model.test.ts

@@ -65,6 +65,45 @@ function project(overrides: ProjectOverrides = {}): SidebarProjectEntry {
 }
 
 describe("buildSidebarProjectRowModel", () => {
+  it("prefers the active host when the grouped project can create there", () => {
+    const result = buildSidebarProjectRowModel({
+      project: project({
+        hosts: [
+          { serverId: "host-a", iconWorkingDir: "/repo/a", worktreeSupport: "supported" },
+          { serverId: "host-b", iconWorkingDir: "/repo/b", worktreeSupport: "supported" },
+        ],
+      }),
+      collapsed: false,
+      preferredServerId: "host-b",
+    });
+
+    expect(result.trailingAction).toEqual({
+      kind: "new_workspace",
+      target: { serverId: "host-b", projectId: "project-host-b", iconWorkingDir: "/repo/b" },
+    });
+  });
+
+  it.each(["host-b", "another-host"])(
+    "keeps the first eligible placement when preferred host %s cannot create here",
+    (preferredServerId) => {
+      const result = buildSidebarProjectRowModel({
+        project: project({
+          hosts: [
+            { serverId: "host-a", iconWorkingDir: "/repo/a", worktreeSupport: "supported" },
+            { serverId: "host-b", iconWorkingDir: "/repo/b", worktreeSupport: "unsupported" },
+          ],
+        }),
+        collapsed: false,
+        preferredServerId,
+      });
+
+      expect(result.trailingAction).toEqual({
+        kind: "new_workspace",
+        target: { serverId: "host-a", projectId: "project-host-a", iconWorkingDir: "/repo/a" },
+      });
+    },
+  );
+
   it("renders a non-git single-workspace project as an expandable section", () => {
     const result = buildSidebarProjectRowModel({
       project: project({

packages/app/src/utils/sidebar-project-row-model.ts

@@ -85,7 +85,9 @@ export function resolveSidebarProjectLocalPath(
 function resolveNewWorkspaceTarget(
   project: SidebarProjectEntry,
   supportsMultiplicityByServerId: ReadonlyMap<string, boolean>,
+  preferredServerId: string | null | undefined,
 ): SidebarProjectHostTarget | null {
+  let fallbackTarget: SidebarProjectHostTarget | null = null;
   for (const host of project.hosts) {
     if (
       host.worktreeSupport === "unsupported" &&
@@ -94,22 +96,30 @@ function resolveNewWorkspaceTarget(
       continue;
     }
     const target = hostTarget(host);
-    if (target) return target;
+    if (!target) continue;
+    if (host.serverId === preferredServerId) return target;
+    fallbackTarget ??= target;
   }
-  return null;
+  return fallbackTarget;
 }
 
 function projectTrailingAction(
   project: SidebarProjectEntry,
   supportsMultiplicityByServerId: ReadonlyMap<string, boolean>,
+  preferredServerId: string | null | undefined,
 ): SidebarProjectTrailingAction {
-  const target = resolveNewWorkspaceTarget(project, supportsMultiplicityByServerId);
+  const target = resolveNewWorkspaceTarget(
+    project,
+    supportsMultiplicityByServerId,
+    preferredServerId,
+  );
   return target ? { kind: "new_workspace", target } : { kind: "none" };
 }
 
 export function buildSidebarProjectRowModel(input: {
   project: SidebarProjectEntry;
   collapsed: boolean;
+  preferredServerId?: string | null;
   supportsMultiplicityByServerId?: ReadonlyMap<string, boolean>;
 }): SidebarProjectRowModel {
   return {
@@ -118,6 +128,7 @@ export function buildSidebarProjectRowModel(input: {
     trailingAction: projectTrailingAction(
       input.project,
       input.supportsMultiplicityByServerId ?? EMPTY_MULTIPLICITY_MAP,
+      input.preferredServerId,
     ),
   };
 }

packages/server/src/server/agent/providers/omp/mcp-bridge.test.ts

@@ -285,10 +285,15 @@ describe("OMP MCP host tools", () => {
       expect(omp.registeredHostTools().at(-1)?.[0]?.name).toMatch(/^mcp_local__echo_secret/);
       omp.processExit("crashed");
       await omp.startTurn("again");
-      await vi.waitFor(() => {
-        expect(omp.runtimeSessions()).toHaveLength(2);
-        expect(omp.registeredHostTools().at(-1)?.[0]?.name).toMatch(/^mcp_local__echo_secret/);
-      });
+      // The relaunch spawns a new runtime and MCP child; on a loaded Windows
+      // runner that has taken longer than vi.waitFor's default one second.
+      await vi.waitFor(
+        () => {
+          expect(omp.runtimeSessions()).toHaveLength(2);
+          expect(omp.registeredHostTools().at(-1)?.[0]?.name).toMatch(/^mcp_local__echo_secret/);
+        },
+        { timeout: 10_000 },
+      );
     } finally {
       await omp.close();
     }

packages/server/src/server/daemon-e2e/agent-deletion-broadcast.e2e.test.ts

@@ -0,0 +1,57 @@
+import { expect, test } from "vitest";
+import { createDaemonTestContext } from "../test-utils/index.js";
+import { DaemonClient } from "../test-utils/daemon-client.js";
+import type { SessionOutboundMessage } from "../messages.js";
+
+test("deleting an agent removes it from every connected agent subscription", async () => {
+  const ctx = await createDaemonTestContext();
+  const observer = new DaemonClient({ url: `ws://127.0.0.1:${ctx.daemon.port}/ws` });
+  const updates: SessionOutboundMessage[] = [];
+  const requestorUpdates: SessionOutboundMessage[] = [];
+  try {
+    await observer.connect();
+    const agent = await ctx.client.createAgent({
+      provider: "codex",
+      cwd: ctx.daemon.staticDir,
+      title: "Cross-client deletion",
+    });
+    const snapshot = await observer.fetchAgents({ subscribe: {} });
+    expect(snapshot.entries.some((entry) => entry.agent.id === agent.id)).toBe(true);
+    observer.subscribeRawMessages((message) => updates.push(message));
+    ctx.client.subscribeRawMessages((message) => requestorUpdates.push(message));
+
+    await ctx.client.deleteAgent(agent.id);
+
+    // The record is gone, and a fresh query sees that. An existing subscriber
+    // must learn the same fact without reconnecting or fetching another list.
+    const afterDelete = await ctx.client.fetchAgents();
+    expect(afterDelete.entries.some((entry) => entry.agent.id === agent.id)).toBe(false);
+    expect(
+      requestorUpdates.some(
+        (message) =>
+          message.type === "agent_update" &&
+          message.payload.kind === "remove" &&
+          message.payload.agentId === agent.id,
+      ),
+    ).toBe(true);
+    await expect(observer.fetchAgentTimeline(agent.id, { direction: "tail" })).rejects.toThrow(
+      "Agent not found",
+    );
+
+    await expect
+      .poll(
+        () =>
+          updates.some(
+            (message) =>
+              message.type === "agent_update" &&
+              message.payload.kind === "remove" &&
+              message.payload.agentId === agent.id,
+          ),
+        { timeout: 2_000 },
+      )
+      .toBe(true);
+  } finally {
+    await observer.close();
+    await ctx.cleanup();
+  }
+});

packages/server/src/server/hub/relationship-liveness.test.ts

@@ -0,0 +1,161 @@
+import { createServer as createHttpServer } from "node:http";
+import { connect, createServer as createTcpServer, type AddressInfo, type Socket } from "node:net";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import pino from "pino";
+import { expect, test, vi } from "vitest";
+import { WebSocket, WebSocketServer } from "ws";
+import { HubRelationshipController } from "./relationship-controller.js";
+import { DirectHubRelationshipRemote } from "./relationship-remote.js";
+
+test("a silently lost Hub connection leaves connected state and reconnects when the path recovers", async () => {
+  const home = await mkdtemp(path.join(tmpdir(), "paseo-hub-liveness-"));
+  const hub = createHttpServer(async (request, response) => {
+    let body = "";
+    for await (const chunk of request) body += chunk;
+    const enrollment = JSON.parse(body) as { daemonId: string };
+    response.writeHead(200, { "content-type": "application/json" }).end(
+      JSON.stringify({
+        daemonId: enrollment.daemonId,
+        permissions: ["hub.execute"],
+        webSocketUrl: `ws://127.0.0.1:${proxyPort}/api/daemons/socket`,
+      }),
+    );
+  });
+  const webSockets = new WebSocketServer({ server: hub });
+  const peers = new Set<WebSocket>();
+  let connections = 0;
+  let pongs = 0;
+  webSockets.on("connection", (peer) => {
+    connections += 1;
+    peers.add(peer);
+    peer.on("close", () => peers.delete(peer));
+  });
+  await listen(hub);
+  const hubPort = (hub.address() as AddressInfo).port;
+  const streams = new Set<Socket>();
+  let dropped = false;
+  const proxy = createTcpServer((client) => {
+    const upstream = connect(hubPort, "127.0.0.1");
+    streams.add(client);
+    streams.add(upstream);
+    client.on("data", (data) => {
+      if (!dropped) upstream.write(data);
+    });
+    upstream.on("data", (data) => {
+      if (!dropped) client.write(data);
+    });
+    client.on("end", () => {
+      if (!dropped) upstream.end();
+    });
+    upstream.on("end", () => {
+      if (!dropped) client.end();
+    });
+    for (const socket of [client, upstream]) {
+      socket.on("error", () => socket.destroy());
+      socket.on("close", () => streams.delete(socket));
+    }
+  });
+  await listen(proxy);
+  const proxyPort = (proxy.address() as AddressInfo).port;
+  let retry: (() => void) | undefined;
+  const controller = new HubRelationshipController({
+    paseoHome: home,
+    hostname: "test-daemon",
+    serverId: "server-1",
+    daemonPublicKey: "public-key",
+    logger: pino({ level: "silent" }),
+    remote: new DirectHubRelationshipRemote(),
+    clock: {
+      now: () => new Date(),
+      schedule: (_delay, task) => {
+        retry = task;
+        return { cancel: () => (retry = undefined) };
+      },
+    },
+    attachSocket: async (socket) => {
+      if (!(socket instanceof WebSocket)) throw new Error("Expected a real WebSocket");
+      socket.on("pong", () => (pongs += 1));
+    },
+    updateAttachedPermissions: () => undefined,
+    createExecutionAgents: () => ({
+      create: async () => {
+        throw new Error("Unexpected agent creation");
+      },
+      subscribe: () => () => undefined,
+      invalidateAuthority: async () => undefined,
+    }),
+  });
+
+  try {
+    // Keep TCP and WebSocket I/O real; accelerate only the keepalive interval and its clock.
+    vi.useFakeTimers({ toFake: ["setInterval", "clearInterval", "Date"] });
+    await controller.connect({
+      hubUrl: `http://127.0.0.1:${proxyPort}`,
+      token: "test-token",
+      permissions: ["hub.execute"],
+    });
+    await expect.poll(() => controller.status().state).toBe("connected");
+    const connectedAt = controller.status().connectedAt;
+
+    expect(connections).toBe(1);
+
+    // Model the reported proxy request ending without a close reaching the daemon.
+    dropped = true;
+    for (const peer of peers) peer.terminate();
+    await expect.poll(() => peers.size).toBe(0);
+    await vi.advanceTimersByTimeAsync(60_000);
+    await expect.poll(() => controller.status().state).toBe("reconnecting");
+    expect(controller.status().lastError).toBe("Hub WebSocket heartbeat timed out");
+    expect(retry).toBeTypeOf("function");
+
+    dropped = false;
+    retry?.();
+    await expect.poll(() => controller.status().state).toBe("connected");
+    expect(connections).toBe(2);
+    expect(controller.status().connectedAt).not.toBe(connectedAt);
+    expect(controller.status().lastError).toBeNull();
+
+    // A healthy idle connection survives multiple liveness windows.
+    for (let tick = 0; tick < 12; tick += 1) {
+      const before = pongs;
+      await vi.advanceTimersByTimeAsync(10_000);
+      await expect.poll(() => pongs).toBeGreaterThan(before);
+    }
+    expect(controller.status().state).toBe("connected");
+    expect(connections).toBe(2);
+
+    await controller.stop();
+    await expect.poll(() => peers.size).toBe(0);
+    await expect.poll(() => vi.getTimerCount()).toBe(0);
+  } finally {
+    await controller.stop();
+    vi.useRealTimers();
+    for (const socket of streams) socket.destroy();
+    for (const peer of peers) peer.terminate();
+    webSockets.close();
+    await Promise.all([close(proxy), close(hub)]);
+    await rm(home, { recursive: true, force: true });
+  }
+});
+
+async function listen(
+  server: ReturnType<typeof createTcpServer> | ReturnType<typeof createHttpServer>,
+) {
+  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+}
+
+async function close(
+  server: ReturnType<typeof createTcpServer> | ReturnType<typeof createHttpServer>,
+) {
+  await new Promise<void>((resolve, reject) => {
+    server.close((error) => {
+      if (error) {
+        reject(error);
+        return;
+      }
+      resolve();
+    });
+  });
+}

packages/server/src/server/hub/relationship-remote.ts

@@ -2,6 +2,9 @@ import { WebSocket } from "ws";
 import { z } from "zod";
 import type { WebSocketLike } from "../websocket-server.js";
 
+const SOCKET_PING_INTERVAL_MS = 10_000;
+const SOCKET_PONG_TIMEOUT_MS = 45_000;
+
 export interface HubEnrollment {
   daemonId: string;
   idempotencyKey: string;
@@ -171,13 +174,42 @@ export class DirectHubRelationshipRemote implements HubRelationshipRemote {
       },
     });
     let settled = false;
+    let heartbeat: ReturnType<typeof setInterval> | undefined;
+    const stopHeartbeat = () => {
+      clearInterval(heartbeat);
+      heartbeat = undefined;
+    };
+    const fail = (error: Error) => {
+      if (settled) return;
+      settled = true;
+      stopHeartbeat();
+      socket.terminate();
+      events.failed(error);
+    };
+    let lastPongAt = 0;
+    socket.on("pong", () => {
+      lastPongAt = Date.now();
+    });
     socket.once("upgrade", (response) => {
       if (response.headers["x-paseo-session-protocol"] === "1") {
         sessionProtocol = "session-v1";
       }
     });
     socket.once("open", () => {
-      if (!settled) events.connected(socket as WebSocketLike, sessionProtocol);
+      if (settled) return;
+      lastPongAt = Date.now();
+      heartbeat = setInterval(() => {
+        if (socket.readyState !== WebSocket.OPEN) return;
+        if (Date.now() - lastPongAt >= SOCKET_PONG_TIMEOUT_MS) {
+          fail(new Error("Hub WebSocket heartbeat timed out"));
+          return;
+        }
+        socket.ping(undefined, undefined, (error) => {
+          if (error) fail(error);
+        });
+      }, SOCKET_PING_INTERVAL_MS);
+      heartbeat.unref?.();
+      events.connected(socket as WebSocketLike, sessionProtocol);
     });
     socket.once("unexpected-response", (_request, response) => {
       if (settled) {
@@ -194,16 +226,12 @@ export class DirectHubRelationshipRemote implements HubRelationshipRemote {
       events.closed(1006);
     });
     socket.once("close", (code) => {
+      stopHeartbeat();
       if (settled) return;
       settled = true;
       events.closed(code);
     });
-    socket.once("error", (error) => {
-      if (settled) return;
-      settled = true;
-      socket.terminate();
-      events.failed(error);
-    });
+    socket.once("error", fail);
     return socket;
   }