diff --git a/docs/architecture.md b/docs/architecture.md index 405602def..9d9af4bb7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -134,6 +134,11 @@ Enables remote access when the daemon is behind a firewall. See [SECURITY.md](../SECURITY.md) for the full threat model. +### Paseo Hub + +The optional Hub relationship is daemon-outbound and does not use the relay. Its connection, +authorization, ownership, persistence, and lifecycle contract is documented in [hub.md](hub.md). + ### `packages/desktop` — Desktop app (Electron) Electron wrapper for macOS, Linux, and Windows. diff --git a/docs/hub.md b/docs/hub.md new file mode 100644 index 000000000..4a6d736eb --- /dev/null +++ b/docs/hub.md @@ -0,0 +1,72 @@ +# Paseo Hub relationship + +Paseo Hub is an explicit opt-in connection from one Paseo daemon to one Hub. Running a daemon does +not register it with a Hub. The relationship begins only when a user runs +`paseo hub connect --token ` from the daemon machine. + +## Connection and authority + +The daemon enrolls over HTTP(S), then opens and maintains a direct outbound WebSocket to the Hub. +The Hub never discovers or acquires the daemon through Paseo's relay. The relay remains an optional +encrypted path for normal Paseo clients and has no role in Hub enrollment, authentication, dispatch, +or reconnects. + +The daemon persists a relationship ID and private connection credential before enrollment. The +relationship is independent of its current transport, so a future transport can replace the direct +WebSocket without pairing again. The current foundation supports one Hub relationship per daemon. + +Normal authenticated daemon sessions may run the `hub.management.daemon.connect`, +`hub.management.daemon.get_status`, and `hub.management.daemon.disconnect` RPCs. Hub connections +receive only `hub.execution.*` authority, so execution credentials cannot manage the relationship. + +## Session grants and execution ownership + +Trusted clients and the Hub use the same `Session` implementation. The connection boundary supplies +grants: trusted clients receive `*`, while an enrolled Hub connection receives its persisted +`hub.execution.*` grant. One matcher handles exact RPC names and trailing namespace wildcards for +both inbound requests and outbound messages. A denied request returns the ordinary `rpc_error` +shape. + +The Hub connection still has a narrow lifecycle boundary: it has no trusted-client hello/resume, +browser, binary, retained-session, or broadcast state. Its outbound execution events include only +agents owned by that daemon identity, so unrelated local agents remain outside the Hub surface. + +Each Hub create carries an execution ID. The daemon stores that ID with the agent's relationship +owner before acknowledging creation. Duplicate or replayed creates for the same daemon and +execution resolve to the same durable agent. After a lost response, reconnect, or daemon restart, +the Hub retries `hub.execution.agent.create.request` with the same execution ID. The idempotent +response returns the existing agent and its current state; there is no separate reconciliation RPC. +Transient stream frames are not durably replayed. + +Daemon restart preserves the Hub relationship and owned execution identity, but interrupts any +active turn. The daemon persists that agent as `closed`; an idempotent create retry returns the same +daemon, execution, and agent identity with that terminal state. Paseo never stores or automatically +replays the original prompt. A duplicate create returns the existing agent without starting another +turn. + +Hub creates use the same agent creation path as trusted clients. They may select any existing +worktree target shape and may request `autoArchive`. Worktree creation and terminal auto-archive use +the shared workspace-aware lifecycle policy; Hub does not have a second launch or cleanup path. + +## Disconnect and revocation + +Normal socket loss reconnects the active relationship with bounded exponential backoff and jitter. +Daemon restart loads the same relationship and credential and reconnects without another enrollment +ceremony. + +Hub authentication rejection or close code `4403` permanently revokes the local relationship. The +daemon deletes its credential, stops reconnecting, and retains only the relationship ID, Hub origin, +scopes, and a sanitized reason for status reporting. + +`paseo hub disconnect` disables socket reconnect before requesting remote revocation. If the Hub is +offline, the daemon persists `disconnecting` and retries revocation across daemon restarts without +opening a Hub socket. This also covers an enrollment whose request may have succeeded but whose +response was lost. `--force` removes local authority immediately and warns that remote revocation may +still be pending. + +## Cross-repository compatibility + +The consumer implementation lives in Paseo Cloud. Cloud owns its copy of the Hub wire schemas and +has no Paseo runtime or build dependency. Cross-repository end-to-end verification separately builds +a Paseo source checkout and exercises the real daemon, CLI, direct WebSocket, Cloud service, and +Postgres. That compatibility fixture is not a package dependency or fallback implementation. diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 9c98310ba..318252802 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -9,6 +9,7 @@ import { createScheduleCommand } from "./commands/schedule/index.js"; import { createSpeechCommand } from "./commands/speech/index.js"; import { createTerminalCommand } from "./commands/terminal/index.js"; import { createWorktreeCommand } from "./commands/worktree/index.js"; +import { createHubCommand } from "./commands/hub/index.js"; import { createHooksCommand } from "./commands/hooks.js"; import { startCommand as daemonStartCommand } from "./commands/daemon/start.js"; import { runStatusCommand as runDaemonStatusCommand } from "./commands/daemon/status.js"; @@ -160,6 +161,7 @@ export function createCli(): Command { // Daemon commands program.addCommand(createDaemonCommand()); + program.addCommand(createHubCommand()); // Chat commands program.addCommand(createChatCommand()); diff --git a/packages/cli/src/commands/hub/index.ts b/packages/cli/src/commands/hub/index.ts new file mode 100644 index 000000000..fce9f1c65 --- /dev/null +++ b/packages/cli/src/commands/hub/index.ts @@ -0,0 +1,104 @@ +import { Command } from "commander"; +import { withOutput, type ListResult, type OutputSchema } from "../../output/index.js"; +import { addJsonAndDaemonHostOptions } from "../../utils/command-options.js"; +import { connectToDaemon } from "../../utils/client.js"; + +interface HubRow { + state: string; + daemonId: string | null; + hub: string | null; + scopes: string; + connectedAt: string | null; + error: string | null; + warning?: string; +} + +const schema: OutputSchema = { + idField: "state", + columns: [ + { header: "STATE", field: "state" }, + { header: "HUB", field: "hub" }, + { header: "DAEMON", field: "daemonId" }, + { header: "SCOPES", field: "scopes" }, + { header: "CONNECTED", field: "connectedAt" }, + { header: "ERROR", field: "error" }, + { header: "WARNING", field: "warning" }, + ], +}; + +function result( + status: { + state: string; + daemonId: string | null; + hubOrigin: string | null; + scopes: string[]; + connectedAt: string | null; + lastError: string | null; + }, + warning?: string, +): ListResult { + return { + type: "list", + data: [ + { + state: status.state, + daemonId: status.daemonId, + hub: status.hubOrigin, + scopes: status.scopes.join(", "), + connectedAt: status.connectedAt, + error: status.lastError, + warning, + }, + ], + schema, + }; +} + +async function withClient( + host: string | undefined, + action: (client: Awaited>) => Promise, +): Promise { + const client = await connectToDaemon({ host }); + try { + return await action(client); + } finally { + await client.close().catch(() => undefined); + } +} + +export function createHubCommand(): Command { + const hub = new Command("hub").description("Manage this daemon's Paseo Hub relationship"); + addJsonAndDaemonHostOptions( + hub.command("connect").argument("").requiredOption("--token "), + ).action( + withOutput(async (...args) => { + const url = args[0] as string; + const options = args.at(-2) as { token: string; host?: string }; + return withClient(options.host, async (client) => + result((await client.connectHub(url, options.token)).status), + ); + }), + ); + addJsonAndDaemonHostOptions(hub.command("status")).action( + withOutput(async (...args) => { + const options = args.at(-2) as { host?: string }; + return withClient(options.host, async (client) => + result((await client.getHubStatus()).status), + ); + }), + ); + addJsonAndDaemonHostOptions( + hub + .command("disconnect") + .option("--force", "Remove local authority even if the Hub is offline"), + ).action( + withOutput(async (...args) => { + const options = args.at(-2) as { host?: string; force?: boolean }; + return withClient(options.host, async (client) => { + const response = await client.disconnectHub(options.force ?? false); + return result(response.status, response.warning); + }); + }), + ); + return hub; +} diff --git a/packages/client/src/daemon-client.test.ts b/packages/client/src/daemon-client.test.ts index 723ffd91b..37a766d23 100644 --- a/packages/client/src/daemon-client.test.ts +++ b/packages/client/src/daemon-client.test.ts @@ -221,6 +221,25 @@ test("advertises consumer-provided browser automation capabilities", async () => }); }); +test("Hub management requires daemon support before dispatching requests", async () => { + const mock = createMockTransport(); + const client = new DaemonClient({ + url: "ws://test", + clientId: "hub_feature_gate_unit_test", + transportFactory: () => mock.transport, + reconnect: { enabled: false }, + }); + clients.push(client); + const connecting = client.connect(); + mock.triggerOpen(); + await connecting; + + await expect(client.getHubStatus()).rejects.toThrow( + "Update the host to use Hub relationship management.", + ); + expect(mock.sent).toEqual([]); +}); + test("sets the complete viewed timeline subscription only when the daemon supports it", async () => { const supportedTransport = createMockTransport(); const supportedClient = new DaemonClient({ diff --git a/packages/client/src/daemon-client.ts b/packages/client/src/daemon-client.ts index f7731112a..6eaa0b155 100644 --- a/packages/client/src/daemon-client.ts +++ b/packages/client/src/daemon-client.ts @@ -4269,6 +4269,33 @@ export class DaemonClient { }); } + async connectHub(hubUrl: string, token: string, requestId?: string) { + this.requireHubRelationshipSupport(); + return this.sendCorrelatedSessionRequest({ + requestId, + message: { type: "hub.management.daemon.connect.request", hubUrl, token }, + responseType: "hub.management.daemon.connect.response", + }); + } + + async getHubStatus(requestId?: string) { + this.requireHubRelationshipSupport(); + return this.sendCorrelatedSessionRequest({ + requestId, + message: { type: "hub.management.daemon.get_status.request" }, + responseType: "hub.management.daemon.get_status.response", + }); + } + + async disconnectHub(force = false, requestId?: string) { + this.requireHubRelationshipSupport(); + return this.sendCorrelatedSessionRequest({ + requestId, + message: { type: "hub.management.daemon.disconnect.request", force }, + responseType: "hub.management.daemon.disconnect.response", + }); + } + async getDaemonPairingOffer( options?: DaemonPairingOfferOptions, ): Promise { @@ -5089,6 +5116,13 @@ export class DaemonClient { return this.lastServerInfoMessage; } + private requireHubRelationshipSupport(): void { + // COMPAT(hubRelationship): added in v0.1.X, drop the gate when floor >= v0.1.X. + if (this.lastServerInfoMessage?.features?.hubRelationship !== true) { + throw new Error("Update the host to use Hub relationship management."); + } + } + private resolveTransportUrlForAttempt(): string { return this.config.url; } diff --git a/packages/protocol/src/messages.hub.test.ts b/packages/protocol/src/messages.hub.test.ts new file mode 100644 index 000000000..ae63b0dc6 --- /dev/null +++ b/packages/protocol/src/messages.hub.test.ts @@ -0,0 +1,210 @@ +import { describe, expect, test } from "vitest"; +import { z } from "zod"; + +import { + HubMessageCorrelationError, + SessionInboundMessageSchema, + SessionOutboundMessageSchema, + parseHubExecutionOutboundMessage, +} from "./messages.js"; + +const agent = { + id: "agent-1", + provider: "codex", + cwd: "/workspace", + model: null, + createdAt: "2026-07-13T00:00:00.000Z", + updatedAt: "2026-07-13T00:00:00.000Z", + lastUserMessageAt: null, + status: "idle", + capabilities: { + supportsStreaming: true, + supportsSessionPersistence: true, + supportsDynamicModes: true, + supportsMcpServers: false, + supportsReasoningStream: true, + supportsToolInvocations: true, + supportsRewindConversation: false, + supportsRewindFiles: false, + supportsRewindBoth: false, + }, + currentModeId: null, + availableModes: [], + pendingPermissions: [], + persistence: null, + title: null, + labels: {}, +}; + +// Frozen at the Hub create request shape shipped before worktree and autoArchive. +const PreviousHubAgentCreateRequestSchema = z.object({ + type: z.literal("hub.execution.agent.create.request"), + requestId: z.string(), + executionId: z.string(), + provider: z.string(), + cwd: z.string(), + workspaceId: z.string().optional(), + prompt: z.string(), + model: z.string().optional(), + modeId: z.string().optional(), + thinkingOptionId: z.string().optional(), + featureValues: z.record(z.string(), z.unknown()).optional(), + env: z.record(z.string(), z.string()).optional(), +}); + +describe("Hub session protocol", () => { + test("accepts the Hub execution create request", () => { + const message = { + type: "hub.execution.agent.create.request", + requestId: "request-1", + executionId: "execution-1", + provider: "codex", + cwd: "/workspace", + prompt: "Implement the requested change", + modeId: "code", + }; + + expect(SessionInboundMessageSchema.parse(message)).toEqual(message); + }); + + test.each([ + undefined, + { mode: "branch-off", newBranch: "hub-work", base: "main" }, + { mode: "checkout-branch", branch: "existing-work" }, + { mode: "checkout-pr", prNumber: 42 }, + ])("accepts Hub create worktree target %#", (worktree) => { + const message = { + type: "hub.execution.agent.create.request", + requestId: "hub-worktree", + executionId: "execution-worktree", + provider: "codex", + cwd: "/repo", + prompt: "Work in the requested target", + ...(worktree ? { worktree, autoArchive: true } : {}), + }; + + expect(SessionInboundMessageSchema.parse(message)).toEqual(message); + }); + + test("the previous Hub create parser ignores additive worktree and auto-archive fields", () => { + const newRequest = { + type: "hub.execution.agent.create.request" as const, + requestId: "hub-worktree", + executionId: "execution-worktree", + provider: "codex", + cwd: "/repo", + prompt: "Work in the requested target", + worktree: { mode: "branch-off", newBranch: "hub-work", base: "main" }, + autoArchive: true, + }; + + expect(PreviousHubAgentCreateRequestSchema.parse(newRequest)).toEqual({ + type: "hub.execution.agent.create.request", + requestId: "hub-worktree", + executionId: "execution-worktree", + provider: "codex", + cwd: "/repo", + prompt: "Work in the requested target", + }); + }); + + test.each([ + { + type: "hub.execution.agent.create.response", + payload: { + requestId: "request-1", + executionId: "execution-1", + agentId: "agent-1", + agent, + success: true, + error: null, + }, + }, + { + type: "hub.execution.agent.update", + payload: { executionId: "execution-1", agentId: "agent-1", agent }, + }, + { + type: "hub.execution.agent.stream", + payload: { + executionId: "execution-1", + agentId: "agent-1", + event: { type: "turn_started", provider: "codex" }, + }, + }, + ])("accepts outbound variant $type", (message) => { + expect(SessionOutboundMessageSchema.parse(message)).toEqual(message); + expect(parseHubExecutionOutboundMessage(message)).toEqual(message); + }); + + test("rejects a Hub update whose correlated agent ids disagree", () => { + const malformed = { + type: "hub.execution.agent.update", + payload: { executionId: "execution-1", agentId: "agent-2", agent }, + }; + + expect(SessionOutboundMessageSchema.safeParse(malformed).success).toBe(true); + expect(() => parseHubExecutionOutboundMessage(malformed)).toThrow(HubMessageCorrelationError); + }); + + test.each([ + { + type: "hub.management.daemon.connect.request", + requestId: "r1", + hubUrl: "https://hub.example", + token: "token", + }, + { type: "hub.management.daemon.get_status.request", requestId: "r2" }, + { type: "hub.management.daemon.disconnect.request", requestId: "r3", force: true }, + ])("accepts trusted management request $type", (message) => { + expect(SessionInboundMessageSchema.parse(message)).toEqual(message); + }); + + test.each([ + { + type: "hub.management.daemon.connect.response", + payload: { + requestId: "r1", + status: { + state: "connected", + daemonId: "daemon-1", + hubOrigin: "https://hub.example", + scopes: ["hub.execution.*"], + connectedAt: "2026-07-13T00:00:00.000Z", + lastError: null, + }, + }, + }, + { + type: "hub.management.daemon.get_status.response", + payload: { + requestId: "r2", + status: { + state: "not_connected", + daemonId: null, + hubOrigin: null, + scopes: [], + connectedAt: null, + lastError: null, + }, + }, + }, + { + type: "hub.management.daemon.disconnect.response", + payload: { + requestId: "r3", + status: { + state: "disconnecting", + daemonId: "daemon-1", + hubOrigin: "https://hub.example", + scopes: ["hub.execution.*"], + connectedAt: null, + lastError: "offline", + }, + warning: "pending", + }, + }, + ])("accepts trusted management response $type", (message) => { + expect(SessionOutboundMessageSchema.parse(message)).toEqual(message); + }); +}); diff --git a/packages/protocol/src/messages.ts b/packages/protocol/src/messages.ts index 9918fb533..48e9f106d 100644 --- a/packages/protocol/src/messages.ts +++ b/packages/protocol/src/messages.ts @@ -1135,6 +1135,22 @@ export const DaemonGetPairingOfferRequestSchema = z.object({ requestId: z.string(), }); +export const HubManagementDaemonConnectRequestSchema = z.object({ + type: z.literal("hub.management.daemon.connect.request"), + requestId: z.string(), + hubUrl: z.string(), + token: z.string(), +}); +export const HubManagementDaemonGetStatusRequestSchema = z.object({ + type: z.literal("hub.management.daemon.get_status.request"), + requestId: z.string(), +}); +export const HubManagementDaemonDisconnectRequestSchema = z.object({ + type: z.literal("hub.management.daemon.disconnect.request"), + requestId: z.string(), + force: z.boolean().optional(), +}); + export const DiagnosticsRequestSchema = z.object({ type: z.literal("diagnostics.request"), requestId: z.string(), @@ -2312,7 +2328,27 @@ export const CaptureTerminalRequestSchema = z.object({ requestId: z.string(), }); +export const HubExecutionAgentCreateRequestSchema = z.object({ + type: z.literal("hub.execution.agent.create.request"), + requestId: z.string(), + executionId: z.string(), + provider: z.string(), + cwd: z.string(), + prompt: z.string(), + workspaceId: z.string().optional(), + model: z.string().optional(), + modeId: z.string().optional(), + thinkingOptionId: z.string().optional(), + featureValues: z.record(z.string(), z.unknown()).optional(), + env: z.record(z.string(), z.string()).optional(), + worktree: CreateAgentWorktreeTargetSchema.optional(), + autoArchive: z.boolean().optional(), +}); + +export type HubExecutionAgentCreateRequest = z.infer; + export const SessionInboundMessageSchema = z.discriminatedUnion("type", [ + HubExecutionAgentCreateRequestSchema, BrowserAutomationExecuteResponseSchema, VoiceAudioChunkMessageSchema, AbortRequestMessageSchema, @@ -2337,6 +2373,9 @@ export const SessionInboundMessageSchema = z.discriminatedUnion("type", [ WaitForFinishRequestSchema, DaemonGetStatusRequestSchema, DaemonGetPairingOfferRequestSchema, + HubManagementDaemonConnectRequestSchema, + HubManagementDaemonGetStatusRequestSchema, + HubManagementDaemonDisconnectRequestSchema, DiagnosticsRequestSchema, GetDaemonConfigRequestMessageSchema, SetDaemonConfigRequestMessageSchema, @@ -2669,6 +2708,8 @@ export const ServerInfoStatusPayloadSchema = z providerSubagents: z.boolean().optional(), // COMPAT(workspacePinning): added in v0.1.107, remove gate after 2027-01-12. workspacePinning: z.boolean().optional(), + // COMPAT(hubRelationship): added in v0.1.X, drop the gate when floor >= v0.1.X. + hubRelationship: z.boolean().optional(), // COMPAT(projectGithubClone): added in v0.1.108, remove gate after 2027-01-15. projectGithubClone: z.boolean().optional(), // COMPAT(workspaceGithubRepositorySearch): added in v0.1.108, remove gate after 2027-01-15. @@ -3598,6 +3639,38 @@ export const DaemonGetStatusResponseSchema = z.object({ .passthrough(), }); +export const HubRelationshipStatusSchema = z.object({ + state: z.enum([ + "not_connected", + "connecting", + "connected", + "reconnecting", + "disconnecting", + "revoked", + ]), + daemonId: z.string().nullable(), + hubOrigin: z.string().nullable(), + scopes: z.array(z.string()), + connectedAt: z.string().nullable(), + lastError: z.string().nullable(), +}); +export const HubManagementDaemonConnectResponseSchema = z.object({ + type: z.literal("hub.management.daemon.connect.response"), + payload: z.object({ requestId: z.string(), status: HubRelationshipStatusSchema }), +}); +export const HubManagementDaemonGetStatusResponseSchema = z.object({ + type: z.literal("hub.management.daemon.get_status.response"), + payload: z.object({ requestId: z.string(), status: HubRelationshipStatusSchema }), +}); +export const HubManagementDaemonDisconnectResponseSchema = z.object({ + type: z.literal("hub.management.daemon.disconnect.response"), + payload: z.object({ + requestId: z.string(), + status: HubRelationshipStatusSchema, + warning: z.string().optional(), + }), +}); + export const DaemonGetPairingOfferResponseSchema = z.object({ type: z.literal("daemon.get_pairing_offer.response"), payload: z @@ -4837,9 +4910,76 @@ export const DaemonUpdateProgressMessageSchema = z.object({ }), }); +export const HubExecutionAgentCreateResponseSchema = z.object({ + type: z.literal("hub.execution.agent.create.response"), + payload: z.object({ + requestId: z.string(), + executionId: z.string(), + agentId: z.string().nullable(), + agent: AgentSnapshotPayloadSchema.nullable(), + success: z.boolean(), + error: z.string().nullable(), + }), +}); + +export const HubExecutionAgentUpdateSchema = z.object({ + type: z.literal("hub.execution.agent.update"), + payload: z.object({ + executionId: z.string(), + agentId: z.string(), + agent: AgentSnapshotPayloadSchema, + }), +}); + +export const HubExecutionAgentStreamSchema = z.object({ + type: z.literal("hub.execution.agent.stream"), + payload: z.object({ + executionId: z.string(), + agentId: z.string(), + event: AgentStreamEventPayloadSchema, + }), +}); + +export type HubExecutionAgentCreateResponse = z.infer; +export type HubExecutionAgentUpdate = z.infer; +export type HubExecutionAgentStream = z.infer; + +export const HubExecutionOutboundMessageSchema = z.discriminatedUnion("type", [ + HubExecutionAgentCreateResponseSchema, + HubExecutionAgentUpdateSchema, + HubExecutionAgentStreamSchema, +]); + +export type HubExecutionOutboundMessage = z.infer; + +export class HubMessageCorrelationError extends Error { + constructor(messageType: HubExecutionOutboundMessage["type"]) { + super(`Hub message ${messageType} has mismatched agent correlation`); + this.name = "HubMessageCorrelationError"; + } +} + +export function parseHubExecutionOutboundMessage(value: unknown): HubExecutionOutboundMessage { + const message = HubExecutionOutboundMessageSchema.parse(value); + const payload = message.payload; + if ( + "agent" in payload && + payload.agent !== null && + "agentId" in payload && + payload.agentId !== null && + payload.agent.id !== payload.agentId + ) { + throw new HubMessageCorrelationError(message.type); + } + return message; +} + export type DaemonUpdateProgressMessage = z.infer; export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [ + HubExecutionAgentCreateResponseSchema, + HubExecutionAgentUpdateSchema, + HubExecutionAgentStreamSchema, BrowserAutomationExecuteRequestSchema, ActivityLogMessageSchema, AssistantChunkMessageSchema, @@ -4892,6 +5032,9 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [ SetVoiceModeResponseMessageSchema, DaemonGetStatusResponseSchema, DaemonGetPairingOfferResponseSchema, + HubManagementDaemonConnectResponseSchema, + HubManagementDaemonGetStatusResponseSchema, + HubManagementDaemonDisconnectResponseSchema, DiagnosticsResponseSchema, GetDaemonConfigResponseMessageSchema, SetDaemonConfigResponseMessageSchema, diff --git a/packages/server/src/server/agent/agent-loading.ts b/packages/server/src/server/agent/agent-loading.ts index dfe06fc43..c3a67af2c 100644 --- a/packages/server/src/server/agent/agent-loading.ts +++ b/packages/server/src/server/agent/agent-loading.ts @@ -75,6 +75,7 @@ export async function ensureAgentLoaded( snapshot = await deps.agentManager.createAgent(config, agentId, { labels: record.labels, workspaceId: record.workspaceId, + owner: record.owner, }); deps.logger.info({ agentId, provider: record.provider }, "Agent created from stored config"); } diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index 73f51346e..158cdc811 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -44,6 +44,7 @@ import { } from "./agent-sdk-types.js"; import { buildArchivedAgentRecord, type ArchivedStoredAgentRecord } from "./agent-archive.js"; import type { StoredAgentRecord, AgentStorage } from "./agent-storage.js"; +import type { AgentOwner } from "./agent-owner.js"; import { InMemoryAgentTimelineStore, type SeedAgentTimelineOptions, @@ -232,6 +233,7 @@ export interface CreateAgentOptions { initialTitle?: string | null; // undefined is an explicit decision: the agent never appears in the sidebar. workspaceId: string | undefined; + owner?: AgentOwner; } export interface AgentManagerOptions { @@ -306,6 +308,7 @@ interface ManagedAgentBase { * Null/undefined for legacy agents created before ownership stamping. */ workspaceId?: string; + owner?: AgentOwner; capabilities: AgentCapabilityFlags; config: AgentSessionConfig; runtimeInfo?: AgentRuntimeInfo; @@ -780,6 +783,10 @@ export class AgentManager { }; } + subscriptionCount(): number { + return this.subscribers.size; + } + listAgents(): ManagedAgent[] { return Array.from(this.agents.values()) .filter((agent) => !agent.internal) @@ -1010,6 +1017,7 @@ export class AgentManager { labels: options.labels, initialTitle: options.initialTitle, workspaceId: options.workspaceId, + owner: options.owner, }); } @@ -1033,6 +1041,7 @@ export class AgentManager { lastUserMessageAt?: Date | null; labels?: Record; workspaceId?: string; + owner?: AgentOwner; }, ): Promise { return this.trackAgentRegistrationOperation( @@ -1050,6 +1059,7 @@ export class AgentManager { lastUserMessageAt?: Date | null; labels?: Record; workspaceId?: string; + owner?: AgentOwner; }, ): Promise { this.assertAcceptingAgentRegistrations(); @@ -1229,6 +1239,7 @@ export class AgentManager { return this.registerSession(session, storedConfig, agentId, { labels: existing.labels, workspaceId: existing.workspaceId, + owner: existing.owner, createdAt: existing.createdAt, updatedAt: existing.updatedAt, lastUserMessageAt: existing.lastUserMessageAt, @@ -1298,7 +1309,7 @@ export class AgentManager { } } - async closeAgent(agentId: string): Promise { + async closeAgent(agentId: string, options: { persistClosedState?: boolean } = {}): Promise { const agent = this.requireAgent(agentId); this.logger.trace( { @@ -1318,7 +1329,9 @@ export class AgentManager { for (const event of this.providerSubagents.deleteParent(agentId)) { this.dispatch({ type: "provider_subagent", event }); } - await this.persistSnapshot(closedAgent); + if (options.persistClosedState !== false) { + await this.persistSnapshot(closedAgent); + } this.emitClosedAgent(closedAgent, { persist: false }); this.logger.trace( { @@ -1420,6 +1433,7 @@ export class AgentManager { provider: record.provider, cwd: record.cwd, workspaceId: record.workspaceId, + owner: record.owner, session: null, capabilities: STORED_AGENT_CAPABILITIES, config: buildStoredAgentConfig(record), @@ -2574,6 +2588,7 @@ export class AgentManager { initialTitle?: string | null; publishWhenReady?: boolean; workspaceId?: string; + owner?: AgentOwner; }, ): Promise { let registered = false; @@ -2709,6 +2724,7 @@ export class AgentManager { attention?: AttentionState; persistence?: AgentPersistenceHandle; workspaceId?: string; + owner?: AgentOwner; } | undefined; }): ActiveManagedAgent { @@ -2718,6 +2734,7 @@ export class AgentManager { provider: config.provider, cwd: config.cwd, workspaceId: options?.workspaceId, + owner: options?.owner, session, capabilities: session.capabilities, config, diff --git a/packages/server/src/server/agent/agent-owner.ts b/packages/server/src/server/agent/agent-owner.ts new file mode 100644 index 000000000..7dd9e9442 --- /dev/null +++ b/packages/server/src/server/agent/agent-owner.ts @@ -0,0 +1,16 @@ +import { z } from "zod"; + +export const AgentOwnerSchema = z.discriminatedUnion("kind", [ + z.object({ + kind: z.literal("daemon"), + daemonId: z.string(), + executionId: z.string(), + }), +]); + +export type AgentOwner = z.infer; +export type DaemonAgentOwner = Extract; + +export function daemonExecutionKey(owner: DaemonAgentOwner): string { + return `${owner.daemonId}\0${owner.executionId}`; +} diff --git a/packages/server/src/server/agent/agent-projections.ts b/packages/server/src/server/agent/agent-projections.ts index 6adee4bd5..a01c67b29 100644 --- a/packages/server/src/server/agent/agent-projections.ts +++ b/packages/server/src/server/agent/agent-projections.ts @@ -93,6 +93,7 @@ export function toStoredAgentRecord( ? agent.attention.attentionTimestamp.toISOString() : null, internal: options?.internal, + owner: agent.owner, } satisfies StoredAgentRecord; } diff --git a/packages/server/src/server/agent/agent-storage.ts b/packages/server/src/server/agent/agent-storage.ts index 069865f66..5e504cee6 100644 --- a/packages/server/src/server/agent/agent-storage.ts +++ b/packages/server/src/server/agent/agent-storage.ts @@ -1,4 +1,4 @@ -import { promises as fs } from "node:fs"; +import { promises as fs, type Dirent } from "node:fs"; import path from "node:path"; import { z } from "zod"; import type { Logger } from "pino"; @@ -8,6 +8,7 @@ import { AgentFeatureSchema, AgentStatusSchema } from "../messages.js"; import { toStoredAgentRecord } from "./agent-projections.js"; import type { ManagedAgent } from "./agent-manager.js"; import type { AgentSessionConfig } from "./agent-sdk-types.js"; +import { AgentOwnerSchema, daemonExecutionKey, type DaemonAgentOwner } from "./agent-owner.js"; const SERIALIZABLE_CONFIG_SCHEMA = z .object({ @@ -64,6 +65,7 @@ const STORED_AGENT_SCHEMA = z.object({ attentionTimestamp: z.string().nullable().optional(), internal: z.boolean().optional(), archivedAt: z.string().nullable().optional(), + owner: AgentOwnerSchema.optional(), }); export type SerializableAgentConfig = Pick< @@ -88,6 +90,8 @@ export class AgentStorage { private pathsById: Map> = new Map(); private pendingWrites: Map> = new Map(); private deleting: Set = new Set(); + private daemonAgentIdsByExecution: Map = new Map(); + private daemonExecutionKeysByAgentId: Map = new Map(); private loaded = false; private baseDir: string; private loadPromise: Promise | null = null; @@ -112,6 +116,12 @@ export class AgentStorage { return this.cache.get(agentId) ?? null; } + async findByDaemonExecution(owner: DaemonAgentOwner): Promise { + await this.load(); + const agentId = this.daemonAgentIdsByExecution.get(daemonExecutionKey(owner)); + return agentId ? (this.cache.get(agentId) ?? null) : null; + } + async upsert(record: StoredAgentRecord): Promise { await this.load(); await this.queueRecordWrite(record); @@ -157,6 +167,7 @@ export class AgentStorage { } this.cache.set(agentId, record); + this.indexOwner(record); this.pathById.set(agentId, nextPath); } @@ -186,6 +197,7 @@ export class AgentStorage { ); this.cache.delete(agentId); + this.removeOwnerIndex(agentId); this.pathById.delete(agentId); this.pathsById.delete(agentId); } @@ -248,6 +260,8 @@ export class AgentStorage { this.cache.clear(); this.pathById.clear(); this.pathsById.clear(); + this.daemonAgentIdsByExecution.clear(); + this.daemonExecutionKeysByAgentId.clear(); try { const records = await this.scanDisk(); @@ -266,7 +280,7 @@ export class AgentStorage { private async scanDisk(): Promise { const records: StoredAgentRecord[] = []; - let entries: Array = []; + let entries: Dirent[] = []; try { entries = await fs.readdir(this.baseDir, { withFileTypes: true }); } catch (error) { @@ -310,6 +324,7 @@ export class AgentStorage { const { record, filePath } = item; records.push(record); this.cache.set(record.id, record); + this.indexOwner(record); this.pathById.set(record.id, filePath); this.addIndexedPath(record.id, filePath); } @@ -350,6 +365,28 @@ export class AgentStorage { } } + private indexOwner(record: StoredAgentRecord): void { + this.removeOwnerIndex(record.id); + if (record.owner?.kind === "daemon") { + const key = daemonExecutionKey(record.owner); + const previousAgentId = this.daemonAgentIdsByExecution.get(key); + if (previousAgentId && previousAgentId !== record.id) { + this.daemonExecutionKeysByAgentId.delete(previousAgentId); + } + this.daemonAgentIdsByExecution.set(key, record.id); + this.daemonExecutionKeysByAgentId.set(record.id, key); + } + } + + private removeOwnerIndex(agentId: string): void { + const key = this.daemonExecutionKeysByAgentId.get(agentId); + if (!key) return; + if (this.daemonAgentIdsByExecution.get(key) === agentId) { + this.daemonAgentIdsByExecution.delete(key); + } + this.daemonExecutionKeysByAgentId.delete(agentId); + } + private async waitForPendingWrite(agentId: string): Promise { await (this.pendingWrites.get(agentId) ?? Promise.resolve()).catch(() => undefined); } diff --git a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts new file mode 100644 index 000000000..bdb903043 --- /dev/null +++ b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts @@ -0,0 +1,47 @@ +import { expect, test } from "vitest"; + +import type { AgentManagerEvent, AgentSubscriber } from "./agent-manager.js"; +import { registerAgentAutoArchive } from "./create-agent-lifecycle-dispatch.js"; + +class AgentLifecycleEvents { + private readonly listeners = new Set(); + + subscribe(listener: AgentSubscriber): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + + completeTurn(agentId: string): void { + const event: AgentManagerEvent = { + type: "agent_stream", + agentId, + event: { type: "turn_completed", provider: "codex" }, + }; + for (const listener of this.listeners) listener(event); + } + + listenerCount(): number { + return this.listeners.size; + } +} + +test("auto-archive self-releases once and later cancellation waits harmlessly", async () => { + const agentId = "4a7e2521-286d-4ad5-af35-e091c55302e3"; + const agents = new AgentLifecycleEvents(); + let archiveCount = 0; + const registration = registerAgentAutoArchive({ + agentManager: agents, + agentId, + archive: async () => { + archiveCount += 1; + }, + }); + + agents.completeTurn(agentId); + await registration.cancel(); + await registration.cancel(); + agents.completeTurn(agentId); + + expect(archiveCount).toBe(1); + expect(agents.listenerCount()).toBe(0); +}); diff --git a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts index 25f616922..e4c29c958 100644 --- a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts +++ b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts @@ -14,7 +14,7 @@ import type { FirstAgentContext, SessionOutboundMessage, } from "../messages.js"; -import type { AgentManager } from "./agent-manager.js"; +import type { AgentManager, AgentSubscriber, SubscribeOptions } from "./agent-manager.js"; import type { AgentStorage } from "./agent-storage.js"; interface CreateAgentLifecycleDispatchDependencies { @@ -38,6 +38,16 @@ interface CreateAgentLifecycleDispatchDependencies { logger: pino.Logger; } +export interface LifecycleRegistration { + cancel(): Promise; +} + +interface AgentLifecycleEvents { + subscribe(callback: AgentSubscriber, options?: SubscribeOptions): () => void; +} + +const inactiveRegistration: LifecycleRegistration = { cancel: async () => undefined }; + type AutoArchiveTarget = | { kind: "agent-only" } | { kind: "created-worktree"; result: CreatePaseoWorktreeWorkflowResult }; @@ -67,12 +77,12 @@ export class CreateAgentLifecycleDispatch { autoArchive: boolean | undefined; agentId: string; createdWorktree: CreatePaseoWorktreeWorkflowResult | null; - }): void { + }): LifecycleRegistration { if (input.autoArchive !== true) { - return; + return inactiveRegistration; } - this.registerAutoArchiveOnTerminalState( + return this.registerAutoArchiveOnTerminalState( input.agentId, toAutoArchiveTarget(input.createdWorktree), ); @@ -142,24 +152,15 @@ export class CreateAgentLifecycleDispatch { } } - private registerAutoArchiveOnTerminalState(agentId: string, target: AutoArchiveTarget): void { - const unsubscribe = this.dependencies.agentManager.subscribe( - (event) => { - if (event.type !== "agent_stream") { - return; - } - if ( - event.event.type !== "turn_completed" && - event.event.type !== "turn_failed" && - event.event.type !== "turn_canceled" - ) { - return; - } - unsubscribe(); - void this.autoArchiveAgentOnce(agentId, target); - }, - { agentId, replayState: false }, - ); + private registerAutoArchiveOnTerminalState( + agentId: string, + target: AutoArchiveTarget, + ): LifecycleRegistration { + return registerAgentAutoArchive({ + agentManager: this.dependencies.agentManager, + agentId, + archive: () => this.autoArchiveAgentOnce(agentId, target), + }); } private async autoArchiveAgentOnce(agentId: string, target: AutoArchiveTarget): Promise { @@ -226,6 +227,43 @@ export class CreateAgentLifecycleDispatch { } } +export function registerAgentAutoArchive(input: { + agentManager: AgentLifecycleEvents; + agentId: string; + archive: () => Promise; +}): LifecycleRegistration { + let unsubscribe: (() => void) | null = null; + let archiveTask: Promise | null = null; + const release = () => { + if (!unsubscribe) return; + const subscribed = unsubscribe; + unsubscribe = null; + subscribed(); + }; + const registration: LifecycleRegistration = { + async cancel() { + release(); + await archiveTask; + }, + }; + unsubscribe = input.agentManager.subscribe( + (event) => { + if (event.type !== "agent_stream") return; + if ( + event.event.type !== "turn_completed" && + event.event.type !== "turn_failed" && + event.event.type !== "turn_canceled" + ) { + return; + } + release(); + archiveTask = input.archive(); + }, + { agentId: input.agentId, replayState: false }, + ); + return registration; +} + function toAutoArchiveTarget( createdWorktree: CreatePaseoWorktreeWorkflowResult | null, ): AutoArchiveTarget { diff --git a/packages/server/src/server/agent/create-agent/create.test.ts b/packages/server/src/server/agent/create-agent/create.test.ts index 458ed1ccf..e5c90efc2 100644 --- a/packages/server/src/server/agent/create-agent/create.test.ts +++ b/packages/server/src/server/agent/create-agent/create.test.ts @@ -328,6 +328,58 @@ test("mcp create stamps the new worktree's workspaceId, not the parent's", async } }); +test("mcp create exposes the created worktree before dispatching the initial prompt", async () => { + const workdir = mkdtempSync(join(tmpdir(), "create-agent-worktree-callback-test-")); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const agentManager = createRealAgentManager(storage); + const createdWorktree = await fakeWorktreeCreator({ + repoRoot: workdir, + createdWorkspaceId: "ws-created-worktree", + })(); + let observed: + | { + createdWorktree: CreatePaseoWorktreeWorkflowResult | null; + lifecycle: ManagedAgent["lifecycle"] | null; + } + | undefined; + + try { + await createAgentCommand( + { + agentManager, + agentStorage: storage, + logger, + providerSnapshotManager: { + async resolveCreateConfig() { + return {}; + }, + }, + createPaseoWorktree: async () => createdWorktree, + }, + { + kind: "mcp", + provider: "codex", + cwd: workdir, + title: "worktree callback", + initialPrompt: "Say done.", + background: true, + notifyOnFinish: false, + worktree: { worktreeName: "feature", baseBranch: "main" }, + onCreated: ({ agentId, createdWorktree: callbackWorktree }) => { + observed = { + createdWorktree: callbackWorktree, + lifecycle: agentManager.getAgent(agentId)?.lifecycle ?? null, + }; + }, + }, + ); + + expect(observed).toEqual({ createdWorktree, lifecycle: "idle" }); + } finally { + rmSync(workdir, { recursive: true, force: true }); + } +}); + test("session create keeps the prompt title after the initial prompt settles", async () => { const workdir = mkdtempSync(join(tmpdir(), "create-agent-title-test-")); const storage = new AgentStorage(join(workdir, "agents"), logger); diff --git a/packages/server/src/server/agent/create-agent/create.ts b/packages/server/src/server/agent/create-agent/create.ts index ea20d3ffe..539f324d2 100644 --- a/packages/server/src/server/agent/create-agent/create.ts +++ b/packages/server/src/server/agent/create-agent/create.ts @@ -15,6 +15,7 @@ import type { AgentAttachment, FirstAgentContext, GitSetupOptions } from "../../ import type { AgentManager, CreateAgentOptions, ManagedAgent } from "../agent-manager.js"; import type { AgentPromptInput, AgentRunOptions, AgentSessionConfig } from "../agent-sdk-types.js"; import type { AgentStorage } from "../agent-storage.js"; +import type { AgentOwner } from "../agent-owner.js"; import type { ProviderSnapshotManager } from "../provider-snapshot-manager.js"; import { setupFinishNotification, startCreatedAgentInitialPrompt } from "../agent-prompt.js"; import { resolveCreateAgentTitles } from "../create-agent-title.js"; @@ -41,7 +42,7 @@ export interface CreateAgentCommandDependencies { paseoHome?: string; worktreesRoot?: string; terminalManager?: TerminalManager | null; - providerSnapshotManager: ProviderSnapshotManager; + providerSnapshotManager: Pick; createPaseoWorktree?: CreatePaseoWorktreeWorkflowFn; // Mints a fresh directory workspace for a cwd and returns its id. ensureWorkspaceForCreate?: EnsureWorkspaceForCreate; @@ -93,6 +94,13 @@ export interface CreateAgentFromMcpInput { notifyOnFinish: boolean; internal?: boolean; detached?: boolean; + owner?: AgentOwner; + env?: Record; + onCreated?: (created: { + agentId: string; + createdWorktree: CreatePaseoWorktreeWorkflowResult | null; + }) => void; + onWorktreeCreated?: (createdWorktree: CreatePaseoWorktreeWorkflowResult) => void; callerAgentId?: string; callerContext?: { lockedCwd?: string; @@ -118,6 +126,7 @@ export interface CreateAgentCommandResult { background: boolean; initialPromptStarted: boolean; initialPromptError: unknown | null; + createdWorktree?: CreatePaseoWorktreeWorkflowResult; } export type BoundCreateAgentCommand = ( @@ -158,6 +167,7 @@ interface ResolvedCreateAgent { background: boolean; promptFailure: CreateAgentPromptFailureMode; promptLogger?: Logger; + createdWorktree?: CreatePaseoWorktreeWorkflowResult; } export async function createAgentCommand( @@ -182,6 +192,9 @@ export async function createAgentCommand( let liveSnapshot = snapshot; let initialPromptStarted = false; let initialPromptError: unknown | null = null; + if (input.kind === "mcp") { + input.onCreated?.({ agentId: snapshot.id, createdWorktree: resolved.createdWorktree ?? null }); + } if (resolved.prompt !== undefined) { const sendResult = await sendInitialPrompt(dependencies, resolved, snapshot); initialPromptStarted = sendResult.started; @@ -205,6 +218,7 @@ export async function createAgentCommand( background: resolved.background, initialPromptStarted, initialPromptError, + ...(resolved.createdWorktree ? { createdWorktree: resolved.createdWorktree } : {}), }; } @@ -294,12 +308,14 @@ async function resolveMcpCreateAgent( ? requireParentAgent(dependencies.agentManager, input.callerAgentId) : null; const cwd = resolveMcpInitialCwd(input, parentAgent); - const { resolvedCwd, setupContinuation, createdWorkspaceId } = await resolveMcpCwd({ - dependencies, - cwd, - worktree: input.worktree, - initialPrompt: input.initialPrompt ?? "", - }); + const { resolvedCwd, setupContinuation, createdWorkspaceId, createdWorktree } = + await resolveMcpCwd({ + dependencies, + cwd, + worktree: input.worktree, + initialPrompt: input.initialPrompt ?? "", + }); + if (createdWorktree) input.onWorktreeCreated?.(createdWorktree); const workspaceId = await resolveMcpWorkspaceId({ dependencies, @@ -338,9 +354,12 @@ async function resolveMcpCreateAgent( createOptions: { ...(labels ? { labels } : {}), workspaceId: requireResolvedWorkspaceId(workspaceId), + owner: input.owner, + env: input.env, }, prompt: trimmedPrompt ? trimmedPrompt : undefined, setupContinuation, + createdWorktree, background: input.background, promptFailure: input.promptFailure ?? "log", }; @@ -518,6 +537,7 @@ async function resolveMcpCwd(params: { resolvedCwd: string; setupContinuation?: AgentWorktreeSetupContinuation; createdWorkspaceId?: string; + createdWorktree?: CreatePaseoWorktreeWorkflowResult; }> { const { dependencies, worktree } = params; if (!worktree) { @@ -576,6 +596,7 @@ async function resolveMcpCwd(params: { resolvedCwd: createdWorktree.workspace.cwd, setupContinuation: createdWorktree.setupContinuation, createdWorkspaceId: createdWorktree.workspace.workspaceId, + createdWorktree, }; } diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 135f0efc4..0157f81e2 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -202,6 +202,18 @@ import { createAgentCommand, type CreateAgentCommandDependencies, } from "./agent/create-agent/create.js"; +import { archiveAgentCommand } from "./agent/lifecycle-command.js"; +import { CreateAgentLifecycleDispatch } from "./agent/create-agent-lifecycle-dispatch.js"; +import { + HubRelationshipController, + type HubRelationshipClock, + type HubRelationshipRetryPolicy, +} from "./hub/relationship-controller.js"; +import { + DirectHubRelationshipRemote, + type HubRelationshipRemote, +} from "./hub/relationship-remote.js"; +import { DaemonExecutions } from "./hub/daemon-executions.js"; const MAX_MCP_DEBUG_BATCH_ITEMS = 10; const REDACTED_LOG_VALUE = "[redacted]"; @@ -434,6 +446,13 @@ export interface PaseoDaemon { getListenTarget(): ListenTarget | null; } +export interface PaseoDaemonDependencies { + hubRelationshipRemote?: HubRelationshipRemote; + hubRelationshipClock?: HubRelationshipClock; + hubRelationshipRetryPolicy?: HubRelationshipRetryPolicy; + createHubDaemonId?: () => string; +} + function createBootstrapManagedProcessRegistry( config: Pick, logger: Logger, @@ -511,6 +530,7 @@ function createInitialMutableDaemonConfig(config: PaseoDaemonConfig): MutableDae export async function createPaseoDaemon( config: PaseoDaemonConfig, rootLogger: Logger, + dependencies: PaseoDaemonDependencies = {}, ): Promise { const logger = rootLogger.child({ module: "bootstrap" }); const bootstrapStart = performance.now(); @@ -577,7 +597,7 @@ export async function createPaseoDaemon( serviceProxy, onChange: createScriptStatusEmitter({ sessions: () => - wsServer?.listActiveSessions().map((session) => ({ + wsServer?.listTrustedSessions().map((session) => ({ emit: (message) => session.emitServerMessage(message), })) ?? [], serviceProxy, @@ -827,7 +847,7 @@ export async function createPaseoDaemon( onProjectUpdate: (update) => wsServer?.publishProjectUpdate(update), onWorkspacesChanged: async (workspaceIds) => { await fanOutReconciledWorkspaceUpdates({ - sessions: wsServer?.listActiveSessions() ?? [], + sessions: wsServer?.listTrustedSessions() ?? [], workspaceIds, logger, }); @@ -845,7 +865,7 @@ export async function createPaseoDaemon( workspaceGitService, }); const archiveWorkspaceRecordExternal = async (workspaceId: string) => { - const sessions = wsServer?.listActiveSessions() ?? []; + const sessions = wsServer?.listTrustedSessions() ?? []; if (sessions.length > 0) { await Promise.all( sessions.map((session) => session.archiveWorkspaceRecordForExternalMutation(workspaceId)), @@ -895,20 +915,20 @@ export async function createPaseoDaemon( }; const markWorkspaceArchivingExternal = (workspaceIds: Iterable, archivingAt: string) => { const workspaceIdList = Array.from(workspaceIds); - for (const session of wsServer?.listActiveSessions() ?? []) { + for (const session of wsServer?.listTrustedSessions() ?? []) { session.markWorkspaceArchivingForExternalMutation(workspaceIdList, archivingAt); } }; const clearWorkspaceArchivingExternal = (workspaceIds: Iterable) => { const workspaceIdList = Array.from(workspaceIds); - for (const session of wsServer?.listActiveSessions() ?? []) { + for (const session of wsServer?.listTrustedSessions() ?? []) { session.clearWorkspaceArchivingForExternalMutation(workspaceIdList); } }; const emitWorkspaceUpdatesExternal = async (workspaceIds: Iterable) => { const workspaceIdList = Array.from(workspaceIds); await Promise.all( - (wsServer?.listActiveSessions() ?? []).map((session) => + (wsServer?.listTrustedSessions() ?? []).map((session) => session.emitWorkspaceUpdatesForExternalWorkspaceIds(workspaceIdList), ), ); @@ -986,7 +1006,7 @@ export async function createPaseoDaemon( warmWorkspaceGitData: async (workspace) => { await Promise.all( wsServer - ?.listActiveSessions() + ?.listTrustedSessions() .map((session) => session.warmWorkspaceGitDataForWorkspace(workspace)) ?? [], ); }, @@ -1025,6 +1045,57 @@ export async function createPaseoDaemon( }; const createAgent = (input: Parameters[1]) => createAgentCommand(createAgentCommandDependencies, input); + const hubAgentLifecycle = new CreateAgentLifecycleDispatch({ + paseoHome: config.paseoHome, + worktreesRoot: config.worktreesRoot, + agentManager, + agentStorage, + github, + workspaceGitService, + createPaseoWorktreeWorkflow: createPaseoWorktreeForTools, + archiveAgentForClose: (agentId) => + archiveAgentCommand({ agentManager, agentStorage, logger }, agentId), + findWorkspaceIdForCwd: findWorkspaceIdForCwdExternal, + listActiveWorkspaces: listActiveWorkspacesExternal, + archiveWorkspaceRecord: archiveWorkspaceRecordExternal, + emit: emitExternalSessionMessage, + emitAgentRemove: () => undefined, + emitWorkspaceUpdatesForWorkspaceIds: emitWorkspaceUpdatesExternal, + markWorkspaceArchiving: markWorkspaceArchivingExternal, + clearWorkspaceArchiving: clearWorkspaceArchivingExternal, + killTerminalsForWorkspace: (workspaceId) => + killTerminalsForWorkspace({ terminalManager, sessionLogger: logger }, workspaceId), + logger, + }); + const hubRelationships = new HubRelationshipController({ + paseoHome: config.paseoHome, + serverId, + daemonPublicKey: daemonKeyPair.publicKeyB64, + logger, + remote: dependencies.hubRelationshipRemote ?? new DirectHubRelationshipRemote(), + clock: dependencies.hubRelationshipClock, + retryPolicy: dependencies.hubRelationshipRetryPolicy, + createDaemonId: dependencies.createHubDaemonId, + attachSocket: async (socket, options) => { + if (!wsServer) throw new Error("WebSocket server is not running"); + await wsServer.attachHubSocket(socket, options); + }, + createExecutionAgents: (daemonId) => + new DaemonExecutions({ + daemonId, + agentManager, + agentStorage, + createAgent, + registerAutoArchive: ({ agentId, createdWorktree }) => + hubAgentLifecycle.registerAutoArchiveIfRequested({ + autoArchive: true, + agentId, + createdWorktree, + }), + cleanupFailedCreate: (input) => + hubAgentLifecycle.cleanupCreatedWorktreeAfterFailedAgentCreate(input), + }), + }); const loopService = new LoopService({ paseoHome: config.paseoHome, @@ -1417,7 +1488,9 @@ export async function createPaseoDaemon( }, serviceProxyPublicBaseUrl, browserToolsBroker, + hubRelationships, ); + await hubRelationships.start(); if (relayEnabled) { const offer = await createConnectionOfferV2({ @@ -1478,6 +1551,7 @@ export async function createPaseoDaemon( }; const stop = async () => { + await hubRelationships.stop(); workspaceReconciliation.dispose(); scriptHealthMonitor.stop(); // Freeze both ingress and registration before taking the agent closure snapshot. diff --git a/packages/server/src/server/daemon-client.e2e.test.ts b/packages/server/src/server/daemon-client.e2e.test.ts index 0065d9e5e..44c4b51be 100644 --- a/packages/server/src/server/daemon-client.e2e.test.ts +++ b/packages/server/src/server/daemon-client.e2e.test.ts @@ -1006,6 +1006,7 @@ test("receives server_info on websocket connect", async () => { expect(serverInfo).not.toBeNull(); expect(serverInfo?.serverId.length).toBeGreaterThan(0); expect(serverInfo?.features?.["terminal-restore-modes"]).toBe(true); + expect(serverInfo?.features?.hubRelationship).toBe(true); expect(serverInfo?.features?.commitsList).toBe(true); expect(serverInfo?.desktopManaged).toBe(false); expect(serverInfo?.features?.daemonSelfUpdate).toBe(true); diff --git a/packages/server/src/server/hub/daemon-executions.test.ts b/packages/server/src/server/hub/daemon-executions.test.ts new file mode 100644 index 000000000..363f341d0 --- /dev/null +++ b/packages/server/src/server/hub/daemon-executions.test.ts @@ -0,0 +1,152 @@ +import { afterEach, expect, test } from "vitest"; +import { HubRelationshipHarness } from "./test-utils/relationship-harness.js"; + +let relationship: HubRelationshipHarness | null = null; + +afterEach(async () => { + await relationship?.close(); + relationship = null; +}); + +async function launchRelationship(): Promise { + const launched = await HubRelationshipHarness.start(); + await launched.beginConnect().result; + launched.connectLatestSocket(); + relationship = launched; + return launched; +} + +test("sequential replay after reconstruction keeps one durable owned agent", async () => { + const hub = await launchRelationship(); + const created = await hub.createOwnedConcurrently(); + + const reconstructed = await hub.reconstructAndReplay(); + + expect(reconstructed.replay.agent.id).toBe(created.first.agentId); + expect(reconstructed.replay.agent.status).toBe("closed"); + expect(reconstructed.durableAgentCount).toBe(1); +}); + +test("removing a daemon-owned agent removes its execution association", async () => { + const hub = await launchRelationship(); + const created = await hub.createOwnedConcurrently(); + + const removed = await hub.removeOwnedAgent(created.first.agentId); + + expect(removed.durableAgentCount).toBe(0); +}); + +test("a failed Hub create removes its auto-created worktree", async () => { + const hub = await launchRelationship(); + hub.beginOwnedCreate("failed-worktree-create", "failed-worktree-execution", { + modeId: "missing-mode", + worktree: { mode: "branch-off", newBranch: "failed-hub-create" }, + }); + + const response = await hub.ownedCreateResult("failed-worktree-create"); + + expect(response).toMatchObject({ + type: "hub.execution.agent.create.response", + payload: { success: false, executionId: "failed-worktree-execution" }, + }); + expect(await hub.listedWorktrees()).toHaveLength(1); + expect(await hub.durableOwnedAgentIds()).toEqual([]); +}); + +test("failed Hub auto-archive creates release their lifecycle subscriptions", async () => { + const hub = await launchRelationship(); + const subscriptionBaseline = hub.agentSubscriptionCount(); + + hub.failProviderPromptStart(); + hub.beginOwnedCreate("failed-prompt-create-1", "failed-prompt-execution-1", { + autoArchive: true, + worktree: { mode: "branch-off", newBranch: "failed-prompt-1" }, + }); + const first = await hub.ownedCreateResult("failed-prompt-create-1"); + + expect(first).toMatchObject({ + type: "hub.execution.agent.create.response", + payload: { success: false, executionId: "failed-prompt-execution-1" }, + }); + expect(hub.activeOwnedAgentIds()).toEqual([]); + expect(await hub.durableOwnedAgentIds()).toEqual([]); + expect(await hub.listedWorktrees()).toHaveLength(1); + expect(hub.agentSubscriptionCount()).toBe(subscriptionBaseline); + + hub.failProviderPromptStart(); + hub.beginOwnedCreate("failed-prompt-create-2", "failed-prompt-execution-2", { + autoArchive: true, + worktree: { mode: "branch-off", newBranch: "failed-prompt-2" }, + }); + const second = await hub.ownedCreateResult("failed-prompt-create-2"); + + expect(second).toMatchObject({ + type: "hub.execution.agent.create.response", + payload: { success: false, executionId: "failed-prompt-execution-2" }, + }); + expect(hub.activeOwnedAgentIds()).toEqual([]); + expect(await hub.durableOwnedAgentIds()).toEqual([]); + expect(await hub.listedWorktrees()).toHaveLength(1); + expect(hub.agentSubscriptionCount()).toBe(subscriptionBaseline); +}); + +test("failed Hub create cleans durable state when provider close rejects", async () => { + const hub = await launchRelationship(); + hub.failProviderPromptStart(); + hub.failNextProviderSessionClose(); + hub.beginOwnedCreate("failed-close-create", "failed-close-execution", { + worktree: { mode: "branch-off", newBranch: "failed-close-worktree" }, + }); + + const response = await hub.ownedCreateResult("failed-close-create"); + + expect(response).toMatchObject({ + type: "hub.execution.agent.create.response", + payload: { success: false, executionId: "failed-close-execution" }, + }); + expect(hub.activeOwnedAgentIds()).toEqual([]); + expect(await hub.durableOwnedAgentIds()).toEqual([]); + expect(await hub.listedWorktrees()).toHaveLength(1); +}); + +test("Hub checkout uses the requested branch ref", async () => { + const hub = await launchRelationship(); + await hub.createBranch("existing-hub-branch"); + hub.beginOwnedCreate("checkout-create", "checkout-execution", { + worktree: { mode: "checkout-branch", branch: "existing-hub-branch" }, + }); + + const response = await hub.ownedCreateResult("checkout-create"); + + expect(response).toMatchObject({ + type: "hub.execution.agent.create.response", + payload: { success: true, executionId: "checkout-execution" }, + }); + expect(await hub.currentBranch(hub.latestCreatedCwd()!)).toBe("existing-hub-branch"); +}); + +test("failed create never archives a reused worktree", async () => { + const hub = await launchRelationship(); + hub.beginOwnedCreate("original-create", "original-execution", { + worktree: { mode: "branch-off", newBranch: "shared-hub-worktree" }, + }); + const original = await hub.ownedCreateResult("original-create"); + const worktreeCwd = original.payload.agent?.cwd; + expect(worktreeCwd).toEqual(expect.any(String)); + await hub.ownedTurnCompletion(original.payload.agentId!); + + const failedPrompt = "Fail the reused worktree create"; + hub.failProviderPromptStart(failedPrompt); + hub.beginOwnedCreate("reused-create", "reused-execution", { + autoArchive: true, + prompt: failedPrompt, + worktree: { mode: "branch-off", newBranch: "shared-hub-worktree" }, + }); + const reused = await hub.ownedCreateResult("reused-create"); + + expect(reused).toMatchObject({ + type: "hub.execution.agent.create.response", + payload: { success: false, executionId: "reused-execution" }, + }); + expect(await hub.worktreeState(worktreeCwd!)).toEqual({ exists: true, listed: true }); +}); diff --git a/packages/server/src/server/hub/daemon-executions.ts b/packages/server/src/server/hub/daemon-executions.ts new file mode 100644 index 000000000..83b4d0693 --- /dev/null +++ b/packages/server/src/server/hub/daemon-executions.ts @@ -0,0 +1,297 @@ +import type { + AgentSnapshotPayload, + AgentStreamEventPayload, + CreateAgentWorktreeTarget, +} from "@getpaseo/protocol/messages"; + +import type { AgentManager, AgentManagerEvent, ManagedAgent } from "../agent/agent-manager.js"; +import type { AgentStorage, StoredAgentRecord } from "../agent/agent-storage.js"; +import type { LifecycleRegistration } from "../agent/create-agent-lifecycle-dispatch.js"; +import type { BoundCreateAgentCommand } from "../agent/create-agent/create.js"; +import type { CreatePaseoWorktreeWorkflowResult } from "../worktree-session.js"; +import { buildStoredAgentPayload } from "../agent/agent-projections.js"; +import { serializeAgentSnapshot, serializeAgentStreamEvent } from "../messages.js"; +import { daemonExecutionKey, type DaemonAgentOwner } from "../agent/agent-owner.js"; + +export interface HubExecutionAgentCreateInput { + executionId: string; + provider: string; + cwd: string; + workspaceId?: string; + prompt: string; + model?: string; + modeId?: string; + thinkingOptionId?: string; + featureValues?: Record; + env?: Record; + worktree?: CreateAgentWorktreeTarget; + autoArchive?: boolean; +} + +export interface OwnedAgentSnapshot { + executionId: string; + agent: AgentSnapshotPayload; +} + +export type OwnedAgentEvent = + | { type: "update"; executionId: string; agent: AgentSnapshotPayload } + | { + type: "stream"; + executionId: string; + agentId: string; + event: AgentStreamEventPayload; + }; + +interface DaemonExecutionsOptions { + daemonId: string; + agentManager: AgentManager; + agentStorage: AgentStorage; + createAgent: BoundCreateAgentCommand; + registerAutoArchive?: (input: { + agentId: string; + createdWorktree: CreatePaseoWorktreeWorkflowResult | null; + }) => LifecycleRegistration; + cleanupFailedCreate?: (input: { + createdWorktree: CreatePaseoWorktreeWorkflowResult | null; + createdAgentId: string | null; + }) => Promise; +} + +export interface HubExecutionAgents { + create(input: HubExecutionAgentCreateInput): Promise; + subscribe(listener: (event: OwnedAgentEvent) => void): () => void; + invalidateAuthority(): Promise; +} + +export class DaemonExecutions implements HubExecutionAgents { + private readonly daemonId: string; + private readonly agentManager: AgentManager; + private readonly agentStorage: AgentStorage; + private readonly createAgentCommand: BoundCreateAgentCommand; + private readonly pendingCreates = new Map>(); + private authorityGeneration = 0; + private authorityActive = true; + private readonly registerAutoArchive: (input: { + agentId: string; + createdWorktree: CreatePaseoWorktreeWorkflowResult | null; + }) => LifecycleRegistration; + private readonly cleanupFailedCreate: NonNullable; + + constructor(options: DaemonExecutionsOptions) { + this.daemonId = options.daemonId; + this.agentManager = options.agentManager; + this.agentStorage = options.agentStorage; + this.createAgentCommand = options.createAgent; + this.registerAutoArchive = + options.registerAutoArchive ?? (() => ({ cancel: async () => undefined })); + this.cleanupFailedCreate = options.cleanupFailedCreate ?? (async () => undefined); + } + + create(input: HubExecutionAgentCreateInput): Promise { + if (!this.authorityActive) { + return Promise.reject(new Error("Hub relationship authority is no longer active")); + } + const owner = this.owner(input.executionId); + const key = daemonExecutionKey(owner); + const pending = this.pendingCreates.get(key); + if (pending) { + return pending; + } + + const authorityGeneration = this.authorityGeneration; + const create = this.createOrResolve(owner, input, authorityGeneration).finally(() => { + if (this.pendingCreates.get(key) === create) { + this.pendingCreates.delete(key); + } + }); + this.pendingCreates.set(key, create); + return create; + } + + async invalidateAuthority(): Promise { + this.authorityActive = false; + this.authorityGeneration++; + await Promise.allSettled(this.pendingCreates.values()); + } + + subscribe(listener: (event: OwnedAgentEvent) => void): () => void { + return this.agentManager.subscribe( + (event) => { + const owned = this.projectEvent(event); + if (owned) { + listener(owned); + } + }, + { replayState: true }, + ); + } + + private async createOrResolve( + owner: DaemonAgentOwner, + input: HubExecutionAgentCreateInput, + authorityGeneration: number, + ): Promise { + const existing = await this.agentStorage.findByDaemonExecution(owner); + if (existing) { + this.requireAuthority(authorityGeneration); + return this.resolveRecord(existing); + } + this.requireAuthority(authorityGeneration); + + let createdWorktree: CreatePaseoWorktreeWorkflowResult | null = null; + let createdAgentId: string | null = null; + let autoArchiveRegistration: LifecycleRegistration = { cancel: async () => undefined }; + let result: Awaited>; + try { + result = await this.createAgentCommand({ + kind: "mcp", + provider: input.model ? `${input.provider}/${input.model}` : input.provider, + title: input.prompt, + initialPrompt: input.prompt, + promptFailure: "throw", + cwd: input.cwd, + workspaceId: input.workspaceId, + mode: input.modeId, + thinking: input.thinkingOptionId, + features: input.featureValues, + env: input.env, + worktree: toCreateAgentWorktree(input.worktree), + background: true, + notifyOnFinish: false, + owner, + onWorktreeCreated: (worktree) => { + createdWorktree = worktree; + }, + onCreated: (created) => { + createdAgentId = created.agentId; + if (input.autoArchive === true) { + autoArchiveRegistration = this.registerAutoArchive({ + ...created, + createdWorktree: ownedCreatedWorktree(created.createdWorktree), + }); + } + }, + }); + this.requireAuthority(authorityGeneration); + } catch (error) { + try { + await autoArchiveRegistration.cancel(); + if (createdAgentId && this.agentManager.getAgent(createdAgentId)) { + await this.agentManager.closeAgent(createdAgentId); + } + } finally { + try { + await this.cleanupFailedCreate({ + createdWorktree: ownedCreatedWorktree(createdWorktree), + createdAgentId: null, + }); + } finally { + if (createdAgentId) { + await this.agentStorage.remove(createdAgentId); + } + } + } + throw error; + } + + return { + executionId: owner.executionId, + agent: serializeAgentSnapshot(result.liveSnapshot), + }; + } + + private resolveRecord(record: StoredAgentRecord): OwnedAgentSnapshot { + return this.projectRecord(record); + } + + private requireAuthority(authorityGeneration: number): void { + if (!this.authorityActive || authorityGeneration !== this.authorityGeneration) { + throw new Error("Hub relationship authority ended during agent creation"); + } + } + + private projectRecord(record: StoredAgentRecord): OwnedAgentSnapshot { + const owner = this.requireOwner(record); + const live = this.agentManager.getAgent(record.id); + return { + executionId: owner.executionId, + agent: live + ? serializeAgentSnapshot(live) + : { + ...buildStoredAgentPayload(record, this.agentManager.getRegisteredProviderIds()), + status: "closed", + }, + }; + } + + private projectEvent(event: AgentManagerEvent): OwnedAgentEvent | null { + if (event.type === "agent_state") { + return this.projectAgentState(event.agent); + } + if (event.type !== "agent_stream") { + return null; + } + const agent = this.agentManager.getAgent(event.agentId); + if (!this.isOwned(agent)) { + return null; + } + const serialized = serializeAgentStreamEvent(event.event); + if (!serialized) { + return null; + } + return { + type: "stream", + executionId: agent.owner.executionId, + agentId: agent.id, + event: serialized, + }; + } + + private projectAgentState(agent: ManagedAgent): OwnedAgentEvent | null { + if (!this.isOwned(agent)) { + return null; + } + return { + type: "update", + executionId: agent.owner.executionId, + agent: serializeAgentSnapshot(agent), + }; + } + + private isOwned(agent: ManagedAgent | null): agent is ManagedAgent & { owner: DaemonAgentOwner } { + return agent?.owner?.kind === "daemon" && agent.owner.daemonId === this.daemonId; + } + + private owner(executionId: string): DaemonAgentOwner { + return { kind: "daemon", daemonId: this.daemonId, executionId }; + } + + private requireOwner(record: StoredAgentRecord): DaemonAgentOwner { + const owner = record.owner; + if (owner?.kind !== "daemon" || owner.daemonId !== this.daemonId) { + throw new Error(`Agent ${record.id} is not owned by daemon ${this.daemonId}`); + } + return owner; + } +} + +function ownedCreatedWorktree( + worktree: CreatePaseoWorktreeWorkflowResult | null, +): CreatePaseoWorktreeWorkflowResult | null { + return worktree?.created === true ? worktree : null; +} + +function toCreateAgentWorktree(target: CreateAgentWorktreeTarget | undefined) { + if (!target) return undefined; + if (target.mode === "branch-off") { + return { + worktreeName: target.newBranch, + baseBranch: target.base, + action: "branch-off" as const, + }; + } + if (target.mode === "checkout-branch") { + return { refName: target.branch, action: "checkout" as const }; + } + return { githubPrNumber: target.prNumber, action: "checkout" as const }; +} diff --git a/packages/server/src/server/hub/execution-controller.test.ts b/packages/server/src/server/hub/execution-controller.test.ts new file mode 100644 index 000000000..585b63d51 --- /dev/null +++ b/packages/server/src/server/hub/execution-controller.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, test } from "vitest"; +import type { + AgentSnapshotPayload, + HubExecutionAgentCreateRequest, + SessionOutboundMessage, +} from "@getpaseo/protocol/messages"; + +import type { + HubExecutionAgents, + OwnedAgentEvent, + OwnedAgentSnapshot, +} from "./daemon-executions.js"; +import { HubExecutionController } from "./execution-controller.js"; + +interface Deferred { + promise: Promise; + resolve(value: T): void; + reject(error: Error): void; +} + +function deferred(): Deferred { + let resolve!: (value: T) => void; + let reject!: (error: Error) => void; + const promise = new Promise((accept, decline) => { + resolve = accept; + reject = decline; + }); + return { promise, resolve, reject }; +} + +class ControlledHubExecutionAgents implements HubExecutionAgents { + private readonly createObserved = deferred(); + private readonly createGate = deferred(); + + create(): Promise { + this.createObserved.resolve(); + return this.createGate.promise; + } + + subscribe(_listener: (event: OwnedAgentEvent) => void): () => void { + return () => undefined; + } + + async invalidateAuthority(): Promise {} + + async creationStarted(): Promise { + await this.createObserved.promise; + } + + finishCreate(): void { + this.createGate.resolve({ + executionId: "execution-shutdown", + agent: { + id: "agent-shutdown", + status: "running", + } as AgentSnapshotPayload, + }); + } +} + +describe("HubExecutionController", () => { + test("cleanup fences in-flight creates before the dead session can receive a response", async () => { + const agents = new ControlledHubExecutionAgents(); + const messages: SessionOutboundMessage[] = []; + const controller = new HubExecutionController({ + agents, + send: (message) => messages.push(message), + }); + + const create = controller.createAgent({ + type: "hub.execution.agent.create.request", + requestId: "shutdown-create", + executionId: "execution-shutdown", + provider: "codex", + cwd: "/tmp/paseo", + prompt: "sleep 30", + } satisfies HubExecutionAgentCreateRequest); + await agents.creationStarted(); + + const cleanup = controller.cleanup(); + agents.finishCreate(); + await Promise.all([create, cleanup]); + + expect(messages).toEqual([]); + }); +}); diff --git a/packages/server/src/server/hub/execution-controller.ts b/packages/server/src/server/hub/execution-controller.ts new file mode 100644 index 000000000..c9b93155e --- /dev/null +++ b/packages/server/src/server/hub/execution-controller.ts @@ -0,0 +1,129 @@ +import { isAbsolute } from "node:path"; +import type { + HubExecutionAgentCreateRequest, + SessionOutboundMessage, +} from "@getpaseo/protocol/messages"; + +import type { HubExecutionAgents, OwnedAgentEvent } from "./daemon-executions.js"; + +interface HubExecutionControllerOptions { + agents: HubExecutionAgents; + send: (message: SessionOutboundMessage) => void; +} + +export class HubExecutionController { + private readonly agents: HubExecutionAgents; + private readonly send: (message: SessionOutboundMessage) => void; + private readonly unsubscribe: () => void; + private readonly pendingCreates = new Set>(); + private cleanupPromise: Promise | null = null; + private closed = false; + + constructor(options: HubExecutionControllerOptions) { + this.agents = options.agents; + this.send = options.send; + this.unsubscribe = this.agents.subscribe((event) => this.sendOwnedEvent(event)); + } + + cleanup(): Promise { + this.cleanupPromise ??= this.cleanupOnce(); + return this.cleanupPromise; + } + + private async cleanupOnce(): Promise { + this.closed = true; + this.unsubscribe(); + await Promise.allSettled(this.pendingCreates); + } + + async createAgent(message: HubExecutionAgentCreateRequest): Promise { + if (this.closed) return; + const create = this.createAgentWithResponse(message); + this.pendingCreates.add(create); + try { + await create; + } finally { + this.pendingCreates.delete(create); + } + } + + private async createAgentWithResponse(message: HubExecutionAgentCreateRequest): Promise { + try { + requireNonBlankHubAgentField("executionId", message.executionId); + requireNonBlankHubAgentField("prompt", message.prompt); + requireNonBlankHubAgentField("cwd", message.cwd); + if (!isAbsolute(message.cwd)) throw new Error("Hub agent cwd must be absolute"); + const result = await this.agents.create({ + executionId: message.executionId, + provider: message.provider, + cwd: message.cwd, + workspaceId: message.workspaceId, + prompt: message.prompt, + model: message.model, + modeId: message.modeId, + thinkingOptionId: message.thinkingOptionId, + featureValues: message.featureValues, + env: message.env, + worktree: message.worktree, + autoArchive: message.autoArchive, + }); + if (this.closed) return; + this.send({ + type: "hub.execution.agent.create.response", + payload: { + requestId: message.requestId, + executionId: message.executionId, + agentId: result.agent.id, + agent: result.agent, + success: true, + error: null, + }, + }); + } catch (error) { + if (this.closed) return; + this.send({ + type: "hub.execution.agent.create.response", + payload: { + requestId: message.requestId, + executionId: message.executionId, + agentId: null, + agent: null, + success: false, + error: error instanceof Error ? error.message : String(error), + }, + }); + } + } + + private sendOwnedEvent(event: OwnedAgentEvent): void { + if (this.closed) return; + if (event.type === "update") { + this.send({ + type: "hub.execution.agent.update", + payload: { + executionId: event.executionId, + agentId: event.agent.id, + agent: event.agent, + }, + }); + return; + } + this.send({ + type: "hub.execution.agent.stream", + payload: { + executionId: event.executionId, + agentId: event.agentId, + event: event.event, + }, + }); + } +} + +function requireNonBlankHubAgentField( + field: "executionId" | "prompt" | "cwd", + value: string, +): void { + if (value.trim().length === 0) { + throw new Error(`Hub agent ${field} cannot be blank`); + } +} diff --git a/packages/server/src/server/hub/execution-session.websocket.test.ts b/packages/server/src/server/hub/execution-session.websocket.test.ts new file mode 100644 index 000000000..caeccd6c6 --- /dev/null +++ b/packages/server/src/server/hub/execution-session.websocket.test.ts @@ -0,0 +1,145 @@ +import { afterEach, expect, test } from "vitest"; +import { + HubRelationshipHarness, + SetupFailingArchiveWatchFiles, +} from "./test-utils/relationship-harness.js"; + +let relationship: HubRelationshipHarness | null = null; + +afterEach(async () => { + await relationship?.close(); + relationship = null; +}); + +async function launchRelationship(): Promise { + const launched = await HubRelationshipHarness.start(); + await launched.beginConnect().result; + launched.connectLatestSocket(); + relationship = launched; + return launched; +} + +test("Hub retries one durable daemon execution across concurrency and reconstruction", async () => { + const hub = await launchRelationship(); + + const created = await hub.createOwnedConcurrently(); + const update = await hub.ownedUpdate(created.first.agentId); + const stream = await hub.ownedStream(created.first.agentId); + const reconstructed = await hub.reconstructAndReplay(); + + expect(created.duplicate.agentId).toBe(created.first.agentId); + expect(update).toMatchObject({ + executionId: "execution-1", + agentId: created.first.agentId, + agent: { id: created.first.agentId }, + }); + expect(stream).toMatchObject({ executionId: "execution-1", agentId: created.first.agentId }); + expect(reconstructed.replay.agent.id).toBe(created.first.agentId); + expect(reconstructed.durableAgentCount).toBe(1); +}); + +test("Hub denies trusted steering and browser dispatch", async () => { + const hub = await launchRelationship(); + const localAgentId = await hub.createUnrelatedLocalAgent(); + + const steeringDenial = await hub.deniedSteering(localAgentId); + const browserDenial = await hub.deniedBrowserDispatch(); + + expect(steeringDenial).toEqual({ + requestId: "denied-steer", + requestType: "send_agent_message_request", + error: "Session is not authorized for send_agent_message_request", + code: "access_denied", + }); + expect(browserDenial).toEqual({ + requestId: "browser-1", + requestType: "browser.automation.execute.response", + error: "Session is not authorized for browser.automation.execute.response", + code: "access_denied", + }); + expect(hub.observedAgentIds()).not.toContain(localAgentId); + expect(hub.observedTrustedLifecycleMessages()).toEqual([]); +}); + +test("Hub sockets reject trusted hello and capabilities", async () => { + const hub = await launchRelationship(); + + expect(hub.probeTrustedHello()).toBe(4002); +}); + +test("Hub sockets reject trusted binary frames", async () => { + const hub = await launchRelationship(); + + expect(hub.probeBinaryFrame()).toBe(4002); +}); + +test("Hub does not receive trusted broadcasts", async () => { + const hub = await launchRelationship(); + + const trustedBroadcasts = await hub.trustedBroadcastCount(); + const trustedStatus = await hub.trustedDaemonStatus(); + + expect(trustedBroadcasts).toBe(0); + expect(trustedStatus).toMatchObject({ pid: process.pid, relay: { enabled: false } }); + expect(hub.observedTrustedLifecycleMessages()).toEqual([]); +}); + +test("Hub reconnects without retaining trusted session state", async () => { + const hub = await launchRelationship(); + const created = await hub.createOwnedConcurrently(); + + const reconnected = await hub.reconnectAndRetry(); + + expect(reconnected).toMatchObject({ + executionId: "execution-1", + agentId: created.first.agentId, + }); + expect(hub.observedTrustedLifecycleMessages()).toEqual([]); +}); + +test("Hub create forwards worktree and auto-archive through the existing create path", async () => { + const hub = await launchRelationship(); + hub.beginOwnedCreate("worktree-create", "execution-worktree", { + worktree: { mode: "branch-off", newBranch: "hub-created-worktree", base: "main" }, + autoArchive: true, + prompt: "sleep 30", + modeId: "always-ask", + }); + const worktreeCreated = await hub.ownedCreateResult("worktree-create"); + const worktreeCwd = hub.latestCreatedCwd(); + const permission = await hub.ownedPermissionRequest(worktreeCreated.payload.agentId!); + const duringRun = await hub.worktreeState(worktreeCwd!); + const archiveCompletion = hub.waitForOwnedArchiveCompletion(worktreeCreated.payload.agentId!); + await hub.allowOwnedPermission(worktreeCreated.payload.agentId!, permission.id); + await hub.ownedTurnCompletion(worktreeCreated.payload.agentId!); + const archive = await archiveCompletion; + const afterArchive = await hub.worktreeState(worktreeCwd!); + + expect(worktreeCreated).toMatchObject({ + type: "hub.execution.agent.create.response", + payload: { success: true, agent: { cwd: worktreeCwd } }, + }); + expect(worktreeCwd).not.toBe(hub.repoRoot()); + expect(duringRun).toEqual({ exists: true, listed: true }); + expect(afterArchive).toEqual({ exists: false, listed: false }); + expect(archive).toEqual({ + agentArchivedAt: expect.any(String), + workspaceArchivedAt: expect.any(String), + }); +}, 20_000); + +test("archive observation closes its first watcher when the second watcher cannot start", async () => { + const watchFiles = new SetupFailingArchiveWatchFiles(2); + const hub = await HubRelationshipHarness.start(watchFiles); + relationship = hub; + await hub.beginConnect().result; + hub.connectLatestSocket(); + hub.beginOwnedCreate("watch-setup-create", "watch-setup-execution"); + const created = await hub.ownedCreateResult("watch-setup-create"); + + await expect(hub.waitForOwnedArchiveCompletion(created.payload.agentId!)).rejects.toThrow( + "Cannot watch", + ); + + expect(watchFiles.activeDirectories()).toEqual([]); +}); diff --git a/packages/server/src/server/hub/relationship-controller.test.ts b/packages/server/src/server/hub/relationship-controller.test.ts new file mode 100644 index 000000000..d7bbe1f68 --- /dev/null +++ b/packages/server/src/server/hub/relationship-controller.test.ts @@ -0,0 +1,806 @@ +import { createHash } from "node:crypto"; +import { platform } from "node:os"; +import { afterEach, describe, expect, test } from "vitest"; +import { HubRelationshipHarness } from "./test-utils/relationship-harness.js"; + +async function captureUnhandledRejections(action: () => Promise): Promise { + const rejections: unknown[] = []; + const capture = (reason: unknown) => rejections.push(reason); + process.on("unhandledRejection", capture); + try { + await action(); + await new Promise((resolve) => setImmediate(resolve)); + } finally { + process.off("unhandledRejection", capture); + } + return rejections; +} + +describe("Hub relationship", () => { + let relationship: HubRelationshipHarness | null = null; + + afterEach(async () => { + await relationship?.close(); + relationship = null; + }); + + test("the CLI connects, reports status, and disconnects through the daemon", async () => { + relationship = await HubRelationshipHarness.start(); + const connected = await relationship.beginConnect().result; + relationship.connectLatestSocket(); + + const status = await relationship.status(); + const enrollment = relationship.enrollmentAttempts()[0]; + const secret = relationship.relationshipFile()?.credential?.secret; + const disconnected = await relationship.disconnect(); + + expect(connected.state).toBe("connecting"); + expect(status.state).toBe("connected"); + expect(relationship.loggableValues(status)).not.toContain(secret); + expect(relationship.loggableValues(status)).not.toContain(enrollment.credentialVerifier); + expect(relationship.loggableValues(status)).not.toContain(enrollment.token); + expect(relationship.loggableValues(status)).not.toContain(enrollment.idempotencyKey); + expect(disconnected.state).toBe("not_connected"); + }, 30_000); + + test("Hub URLs cannot persist embedded credentials", async () => { + relationship = await HubRelationshipHarness.start(); + + await expect( + relationship.beginConnect("ceremony-token", "https://user:password@hub.example").result, + ).rejects.toThrow(); + + expect(relationship.relationshipFile()).toBeNull(); + expect(relationship.enrollmentAttempts()).toEqual([]); + }); + + test("an authenticated external socket can manage the Hub relationship", async () => { + relationship = await HubRelationshipHarness.start(); + + const responses = await relationship.manageRelationshipFromExternalSocket(); + + expect(responses.map((response) => response.type)).toEqual([ + "hub.management.daemon.connect.response", + "hub.management.daemon.get_status.response", + "hub.management.daemon.disconnect.response", + ]); + expect(relationship.enrollmentAttempts()).toHaveLength(1); + expect(relationship.relationshipFile()).toBeNull(); + }); + + test("an authenticated browser socket can manage the Hub relationship", async () => { + relationship = await HubRelationshipHarness.start(); + + const response = await relationship.connectFromBrowserSocket(); + + expect(response.type).toBe("hub.management.daemon.connect.response"); + expect(relationship.enrollmentAttempts()).toHaveLength(1); + expect(relationship.relationshipFile()?.state).toBe("active"); + }); + + test("persists private generated authority before enrollment and active before dialing", async () => { + relationship = await HubRelationshipHarness.start(); + const privateFileMode = platform() === "win32" ? 0o666 : 0o600; + relationship.holdEnrollment(); + const connecting = relationship.beginConnect("one-time-token"); + + const enrollment = await relationship.enrollmentBegins(); + const pending = relationship.enrollmentInvocation(); + + expect(pending).toMatchObject({ + record: { + state: "pending", + relationship: { + daemonId: enrollment.daemonId, + idempotencyKey: enrollment.idempotencyKey, + }, + credential: { secret: expect.any(String) }, + enrollment: { token: "one-time-token" }, + identity: { serverId: expect.any(String), daemonPublicKey: expect.any(String) }, + }, + }); + expect(pending.mode).toBe(privateFileMode); + const secret = pending.record.credential?.secret; + expect(secret).toEqual(expect.any(String)); + expect(enrollment.credentialVerifier).toBe( + createHash("sha256") + .update(secret ?? "") + .digest("base64url"), + ); + relationship.completeEnrollment(); + await connecting.result; + await relationship.socketDialed(); + expect(relationship.socketInvocation()).toMatchObject({ + mode: privateFileMode, + record: { state: "active", relationship: { daemonId: enrollment.daemonId } }, + }); + }); + + test("Hub enrollment cannot widen the daemon's locally granted scopes", async () => { + relationship = await HubRelationshipHarness.start(); + relationship.returnEnrollmentScopes(["hub.execution.*", "*"]); + + await relationship.beginConnect().result; + relationship.connectLatestSocket(); + const responses = relationship.sendHubRequestOnLatest({ + type: "daemon.get_status.request", + requestId: "scope-escalation", + }); + + expect(relationship.relationshipFile()?.relationship.scopes).toEqual(["hub.execution.*"]); + expect(responses).toContainEqual({ + type: "rpc_error", + payload: { + requestId: "scope-escalation", + requestType: "daemon.get_status.request", + error: "Session is not authorized for daemon.get_status.request", + code: "access_denied", + }, + }); + }); + + test("a lost enrollment response reuses the exact ceremony", async () => { + relationship = await HubRelationshipHarness.start(); + relationship.holdEnrollment(); + const connecting = relationship.beginConnect("same-token"); + await relationship.enrollmentBegins(); + relationship.loseEnrollmentResponse(); + await connecting.result; + + await relationship.retry(); + const attempts = relationship.enrollmentAttempts(); + + expect(attempts).toHaveLength(2); + expect(attempts[1]).toEqual(attempts[0]); + expect(relationship.enrolledRelationships()).toBe(1); + }); + + test("a fresh token replaces the token for a pending enrollment ceremony", async () => { + relationship = await HubRelationshipHarness.start(); + relationship.holdEnrollment(); + const firstConnect = relationship.beginConnect("expired-token"); + const firstResult = expect(firstConnect.result).resolves.toMatchObject({ + state: "reconnecting", + }); + await relationship.enrollmentBegins(); + relationship.loseEnrollmentResponse(); + await firstResult; + + relationship.holdEnrollment(); + const secondConnect = relationship.beginConnect("fresh-token"); + const secondResult = expect(secondConnect.result).resolves.toMatchObject({ + state: "connecting", + }); + const retried = await relationship.enrollmentBegins(); + + expect(retried.token).toBe("fresh-token"); + expect(relationship.relationshipFile()?.enrollment?.token).toBe("fresh-token"); + relationship.completeEnrollment(); + await secondResult; + }); + + test("a stale enrollment rejection cannot discard a fresh pending ceremony", async () => { + relationship = await HubRelationshipHarness.start(); + relationship.holdEnrollment(); + const expiredConnect = relationship.beginConnect("expired-token"); + await relationship.enrollmentBegins(); + + relationship.holdEnrollment(); + const freshConnect = relationship.beginConnect("fresh-token"); + const freshEnrollment = await relationship.enrollmentBegins(); + relationship.rejectEnrollment(0, 403); + await expiredConnect.result; + + expect(relationship.relationshipFile()?.enrollment?.token).toBe("fresh-token"); + expect(relationship.pendingRelationshipRetries()).toBe(0); + + relationship.completeEnrollment(); + await freshConnect.result; + + expect(freshEnrollment.token).toBe("fresh-token"); + expect(relationship.relationshipFile()?.state).toBe("active"); + expect(relationship.enrollmentAttempts()).toHaveLength(2); + expect(relationship.pendingRelationshipRetries()).toBe(0); + }); + + test("a rejected pending enrollment is discarded without blocking daemon restart", async () => { + relationship = await HubRelationshipHarness.start(); + relationship.holdEnrollment(); + const connecting = relationship.beginConnect("expired-token"); + await relationship.enrollmentBegins(); + relationship.loseEnrollmentResponse(); + await connecting.result; + relationship.rejectNextEnrollment(401); + + await relationship.restartDaemon(); + + expect(await relationship.status()).toMatchObject({ state: "not_connected" }); + expect(relationship.relationshipFile()).toBeNull(); + }); + + test("a scheduled enrollment rejection is contained after removing pending authority", async () => { + relationship = await HubRelationshipHarness.start(); + relationship.holdEnrollment(); + const connecting = relationship.beginConnect("expired-token"); + await relationship.enrollmentBegins(); + relationship.loseEnrollmentResponse(); + await connecting.result; + relationship.rejectNextEnrollment(403); + + const unhandledRejections = await captureUnhandledRejections(() => relationship!.retry()); + const status = await relationship.status(); + + expect(status).toMatchObject({ state: "not_connected" }); + expect(relationship.relationshipFile()).toBeNull(); + expect(relationship.pendingRelationshipRetries()).toBe(0); + expect(unhandledRejections).toEqual([]); + }); + + test.each(["{not-json", JSON.stringify({ version: 1, state: "unknown" })])( + "an invalid relationship file is quarantined without blocking daemon startup", + async (contents) => { + relationship = await HubRelationshipHarness.start(); + await relationship.corruptRelationshipFile(contents); + + await relationship.startStoppedDaemon(); + + expect(await relationship.status()).toMatchObject({ state: "not_connected" }); + expect(relationship.relationshipFile()).toBeNull(); + expect(await relationship.quarantinedRelationshipFiles()).toHaveLength(1); + }, + ); + + test.each([ + ["a non-HTTP scheme", "ftp://hub.test"], + ["embedded credentials", "https://user:password@hub.test"], + ["a query", "https://hub.test?token=secret"], + ["a fragment", "https://hub.test#secret"], + ])("a persisted Hub origin with %s is quarantined before startup", async (_, hubOrigin) => { + relationship = await HubRelationshipHarness.start(); + await relationship.corruptRelationshipFile( + JSON.stringify({ + version: 1, + state: "pending", + relationship: { + daemonId: "daemon-1", + idempotencyKey: "ceremony-1", + hubOrigin, + createdAt: "2026-07-13T00:00:00.000Z", + scopes: ["hub.execution.*"], + }, + credential: { secret: "credential" }, + enrollment: { token: "enrollment-token" }, + identity: { serverId: "server-1", daemonPublicKey: "public-key" }, + }), + ); + + await relationship.startStoppedDaemon(); + + expect(await relationship.status()).toMatchObject({ state: "not_connected" }); + expect(relationship.relationshipFile()).toBeNull(); + expect(await relationship.quarantinedRelationshipFiles()).toHaveLength(1); + expect(relationship.enrollmentAttempts()).toEqual([]); + }); + + test("a persisted Hub relationship cannot widen its local execution scope", async () => { + relationship = await HubRelationshipHarness.start(); + await relationship.beginConnect().result; + const persisted = relationship.relationshipFile(); + expect(persisted).not.toBeNull(); + persisted!.relationship.scopes = ["*"]; + await relationship.corruptRelationshipFile(JSON.stringify(persisted)); + const socketAttemptsBeforeRestart = relationship.socketAttempts(); + + await relationship.startStoppedDaemon(); + + expect(await relationship.status()).toMatchObject({ state: "not_connected" }); + expect(relationship.relationshipFile()).toBeNull(); + expect(await relationship.quarantinedRelationshipFiles()).toHaveLength(1); + expect(relationship.socketAttempts()).toBe(socketAttemptsBeforeRestart); + }); + + test("a persisted non-WebSocket transport is quarantined before daemon startup", async () => { + relationship = await HubRelationshipHarness.start(); + await relationship.corruptRelationshipFile( + JSON.stringify({ + version: 1, + state: "active", + relationship: { + id: "relationship-1", + idempotencyKey: "ceremony-1", + hubOrigin: "https://hub.test", + createdAt: "2026-07-13T00:00:00.000Z", + scopes: ["hub.execution.*"], + }, + credential: { secret: "credential" }, + transport: { kind: "direct_websocket", webSocketUrl: "ftp://hub.test/daemon" }, + }), + ); + + await relationship.startStoppedDaemon(); + + expect(await relationship.status()).toMatchObject({ state: "not_connected" }); + expect(relationship.relationshipFile()).toBeNull(); + expect(await relationship.quarantinedRelationshipFiles()).toHaveLength(1); + expect(relationship.socketAttempts()).toBe(0); + }); + + test("a persisted WebSocket transport with a fragment is quarantined before startup", async () => { + relationship = await HubRelationshipHarness.start(); + await relationship.corruptRelationshipFile( + JSON.stringify({ + version: 1, + state: "active", + relationship: { + id: "relationship-1", + idempotencyKey: "ceremony-1", + hubOrigin: "https://hub.test", + createdAt: "2026-07-13T00:00:00.000Z", + scopes: ["hub.execution.*"], + }, + credential: { secret: "credential" }, + transport: { + kind: "direct_websocket", + webSocketUrl: "wss://hub.test/daemon#fragment", + }, + }), + ); + + await relationship.startStoppedDaemon(); + + expect(await relationship.status()).toMatchObject({ state: "not_connected" }); + expect(relationship.relationshipFile()).toBeNull(); + expect(await relationship.quarantinedRelationshipFiles()).toHaveLength(1); + expect(relationship.socketAttempts()).toBe(0); + }); + + test("disconnect revokes an ambiguous pending enrollment before removing local authority", async () => { + relationship = await HubRelationshipHarness.start(); + relationship.holdEnrollment(); + const connecting = relationship.beginConnect("one-time-token"); + const enrollment = await relationship.enrollmentBegins(); + const credential = relationship.relationshipFile()?.credential?.secret; + relationship.loseEnrollmentResponse(); + await connecting.result; + relationship.failRevocations(2); + + const disconnected = await relationship.disconnect(); + expect(disconnected.state).toBe("disconnecting"); + expect(relationship.relationshipFile()?.state).toBe("disconnecting"); + + await relationship.restartDaemon(); + await relationship.retry(); + + expect(relationship.revocationAttempts()).toBe(3); + expect(relationship.latestRevocation()).toEqual( + expect.objectContaining({ + daemonId: enrollment.daemonId, + hubOrigin: enrollment.hubOrigin, + credential, + }), + ); + expect(relationship.relationshipFile()).toBeNull(); + }); + + test("disconnect revokes only after an in-flight enrollment settles", async () => { + relationship = await HubRelationshipHarness.start(); + relationship.holdEnrollment(); + const connecting = relationship.beginConnect("one-time-token"); + const enrollment = await relationship.enrollmentBegins(); + + const disconnecting = relationship.beginDisconnect(); + await relationship.relationshipStateBecomes("disconnecting"); + expect(relationship.revocationAttempts()).toBe(0); + + relationship.completeEnrollment(); + await connecting.result; + const disconnected = await disconnecting.result; + + expect(disconnected.state).toBe("not_connected"); + expect(relationship.latestRevocation()?.daemonId).toBe(enrollment.daemonId); + expect(relationship.revocationAttempts()).toBe(1); + expect(relationship.socketAttempts()).toBe(0); + expect(relationship.relationshipFile()).toBeNull(); + }); + + test("daemon restart reconnects the same durable relationship", async () => { + relationship = await HubRelationshipHarness.start(); + await relationship.beginConnect().result; + const id = relationship.relationshipFile()?.relationship.daemonId; + relationship.connectLatestSocket(); + await relationship.socketDialed(); + + await relationship.restartDaemon(); + await relationship.socketDialed(); + + expect(relationship.relationshipFile()?.relationship.daemonId).toBe(id); + expect(relationship.socketAttempts()).toBe(2); + }); + + test("daemon restart closes an interrupted owned turn without replaying its prompt", async () => { + relationship = await HubRelationshipHarness.start(); + await relationship.beginConnect().result; + await relationship.socketDialed(); + relationship.connectLatestSocket(); + const daemonId = relationship.relationshipFile()?.relationship.daemonId; + const prompt = "sleep 30"; + relationship.beginOwnedCreate("running-create", "execution-running", { + prompt, + modeId: "full-access", + }); + const created = await relationship.ownedCreateResult("running-create"); + const running = await relationship.ownedRunningUpdate(created.payload.agentId!); + + await relationship.restartDaemon(); + await relationship.socketDialed(); + relationship.connectLatestSocket(); + relationship.beginOwnedCreate("running-duplicate", "execution-running", { prompt }); + const duplicate = await relationship.ownedCreateResult("running-duplicate"); + expect(duplicate).toMatchObject({ + payload: { + success: true, + executionId: "execution-running", + agentId: created.payload.agentId, + agent: { id: created.payload.agentId, status: "closed" }, + }, + }); + const durableAgentIds = await relationship.durableOwnedAgentIds(); + + expect(running).toMatchObject({ + executionId: "execution-running", + agentId: created.payload.agentId, + agent: { id: created.payload.agentId, status: "running" }, + }); + expect(relationship.relationshipFile()?.relationship.daemonId).toBe(daemonId); + expect(durableAgentIds).toEqual([created.payload.agentId]); + expect(relationship.providerCreations()).toBe(1); + expect(relationship.providerResumes()).toBe(0); + expect(relationship.providerPromptTexts()).toEqual([prompt]); + expect(relationship.latestOwnedTurnCompletions(created.payload.agentId!)).toBe(0); + }); + + test("an owned execution does not persist a daemon-restart prompt intent", async () => { + relationship = await HubRelationshipHarness.start(); + await relationship.beginConnect().result; + relationship.connectLatestSocket(); + relationship.beginOwnedCreate("intent-create", "execution-intent", { prompt: "sleep 30" }); + await relationship.ownedCreateResult("intent-create"); + + expect(await relationship.hubExecutionIntentFiles()).toEqual([]); + }); + + test("an ordinary duplicate create does not replay a completed owned turn", async () => { + relationship = await HubRelationshipHarness.start(); + await relationship.beginConnect().result; + relationship.connectLatestSocket(); + const prompt = "respond with exactly: completed once"; + relationship.beginOwnedCreate("completed-create", "execution-completed", { prompt }); + const created = await relationship.ownedCreateResult("completed-create"); + await relationship.ownedTurnCompletion(created.payload.agentId!); + + relationship.beginOwnedCreate("completed-duplicate", "execution-completed", { prompt }); + const duplicate = await relationship.ownedCreateResult("completed-duplicate"); + + expect(duplicate).toMatchObject({ + type: "hub.execution.agent.create.response", + payload: { + success: true, + executionId: "execution-completed", + agentId: created.payload.agentId, + agent: { status: "idle" }, + }, + }); + expect(relationship.providerPromptTexts()).toEqual([prompt]); + expect(await relationship.durableOwnedAgentIds()).toEqual([created.payload.agentId]); + expect(relationship.latestOwnedTurnCompletions(created.payload.agentId!)).toBe(1); + }); + + test("daemon restart closes a completed owned session without replaying its prompt", async () => { + relationship = await HubRelationshipHarness.start(); + await relationship.beginConnect().result; + relationship.connectLatestSocket(); + const prompt = "respond with exactly: already completed"; + relationship.beginOwnedCreate("idle-create", "execution-idle", { prompt }); + const created = await relationship.ownedCreateResult("idle-create"); + await relationship.ownedTurnCompletion(created.payload.agentId!); + + await relationship.restartDaemon(); + await relationship.socketDialed(); + relationship.connectLatestSocket(); + relationship.beginOwnedCreate("idle-retry", "execution-idle", { prompt }); + const retried = await relationship.ownedCreateResult("idle-retry"); + + expect(retried).toMatchObject({ + payload: { + executionId: "execution-idle", + agentId: created.payload.agentId, + agent: { id: created.payload.agentId, status: "closed" }, + }, + }); + expect(relationship.providerPromptTexts()).toEqual([prompt]); + expect(relationship.providerResumes()).toBe(0); + expect(relationship.latestOwnedTurnCompletions(created.payload.agentId!)).toBe(0); + }); + + test("daemon restart closes a failed owned session without replaying its prompt", async () => { + relationship = await HubRelationshipHarness.start(); + await relationship.beginConnect().result; + relationship.connectLatestSocket(); + const prompt = "emit a turn failure"; + relationship.beginOwnedCreate("failed-create", "execution-failed", { prompt }); + const created = await relationship.ownedCreateResult("failed-create"); + const failed = await relationship.ownedTurnFailure(created.payload.agentId!); + + await relationship.restartDaemon(); + await relationship.socketDialed(); + relationship.connectLatestSocket(); + relationship.beginOwnedCreate("failed-retry", "execution-failed", { prompt }); + const retried = await relationship.ownedCreateResult("failed-retry"); + + expect(failed).toMatchObject({ + executionId: "execution-failed", + agentId: created.payload.agentId, + event: { type: "turn_failed" }, + }); + expect(retried).toMatchObject({ + payload: { + executionId: "execution-failed", + agentId: created.payload.agentId, + agent: { id: created.payload.agentId, status: "closed" }, + }, + }); + expect(relationship.providerPromptTexts()).toEqual([prompt]); + expect(relationship.providerResumes()).toBe(0); + expect(relationship.latestOwnedTurnCompletions(created.payload.agentId!)).toBe(0); + }); + + test("Hub revocation leaves no execution intent artifacts", async () => { + relationship = await HubRelationshipHarness.start(); + await relationship.beginConnect().result; + relationship.connectLatestSocket(); + relationship.beginOwnedCreate("revoked-create", "execution-revoked", { prompt: "sleep 30" }); + await relationship.ownedCreateResult("revoked-create"); + + relationship.rejectRelationship(401); + + expect(await relationship.hubExecutionIntentFiles()).toEqual([]); + }); + + test("stale socket generations cannot replace or unregister the current socket", async () => { + relationship = await HubRelationshipHarness.start(); + await relationship.beginConnect().result; + relationship.closeSocket(0, 1006); + await relationship.retry(); + relationship.connectSocket(1); + + relationship.connectSocket(0); + relationship.closeSocket(0, 1000); + const messages = relationship.sendHubRequestOnLatest({ + type: "daemon.get_status.request", + requestId: "still-current", + }); + + expect(messages).toContainEqual({ + type: "rpc_error", + payload: { + requestId: "still-current", + requestType: "daemon.get_status.request", + error: "Session is not authorized for daemon.get_status.request", + code: "access_denied", + }, + }); + expect(relationship.socketAttempts()).toBe(2); + }); + + test("replayed create across socket generations shares one pending durable execution", async () => { + relationship = await HubRelationshipHarness.start(); + await relationship.beginConnect().result; + relationship.connectLatestSocket(); + relationship.holdAgentCreation(); + relationship.beginOwnedCreate("first-create"); + await relationship.agentCreationAttempts(1); + + relationship.closeLatestSocket(1006); + await relationship.retry(); + relationship.connectLatestSocket(); + relationship.beginOwnedCreate("replayed-create"); + relationship.finishAgentCreation(); + + const replayed = await relationship.ownedCreateResult("replayed-create"); + const durableAgentIds = await relationship.durableOwnedAgentIds(); + + expect(relationship.socketDeliveredResponse(0, "first-create")).toBe(false); + expect(replayed).toMatchObject({ + type: "hub.execution.agent.create.response", + payload: { + success: true, + executionId: "execution-race", + agentId: durableAgentIds[0], + }, + }); + expect(relationship.providerCreations()).toBe(1); + expect(durableAgentIds).toHaveLength(1); + }); + + test("idempotent retry waits for a pending create across socket generations", async () => { + relationship = await HubRelationshipHarness.start(); + await relationship.beginConnect().result; + relationship.connectLatestSocket(); + relationship.holdAgentCreation(); + relationship.beginOwnedCreate("pending-create", "pending-execution"); + await relationship.agentCreationAttempts(1); + + relationship.closeLatestSocket(1006); + await relationship.retry(); + relationship.connectLatestSocket(); + relationship.beginOwnedCreate("pending-retry", "pending-execution"); + relationship.finishAgentCreation(); + + await expect(relationship.ownedCreateResult("pending-retry")).resolves.toMatchObject({ + payload: { + executionId: "pending-execution", + agentId: expect.any(String), + agent: { id: expect.any(String) }, + }, + }); + expect(relationship.providerCreations()).toBe(1); + expect(await relationship.durableOwnedAgentIds()).toHaveLength(1); + }); + + test("force disconnect fences a pending create before removing authority", async () => { + relationship = await HubRelationshipHarness.start(); + await relationship.beginConnect().result; + relationship.connectLatestSocket(); + relationship.holdAgentCreation(); + relationship.beginOwnedCreate("cancelled-create", "cancelled-execution", { + worktree: { mode: "branch-off", newBranch: "cancelled-hub-create" }, + }); + await relationship.agentCreationAttempts(1); + + const disconnecting = relationship.beginDisconnect(true); + await relationship.relationshipStateBecomes(null); + relationship.finishAgentCreation(); + await disconnecting.result; + + expect(relationship.activeOwnedAgentIds()).toEqual([]); + expect(await relationship.durableOwnedAgentIds()).toEqual([]); + expect(await relationship.listedWorktrees()).toHaveLength(1); + }); + + test("re-enrollment gives the same daemon fresh execution authority", async () => { + relationship = await HubRelationshipHarness.start(); + await relationship.beginConnect().result; + relationship.connectLatestSocket(); + relationship.beginOwnedCreate("first-create", "first-execution"); + const first = await relationship.ownedCreateResult("first-create"); + const daemonId = relationship.relationshipFile()?.relationship.daemonId; + + await relationship.disconnect(); + await relationship.beginConnect("fresh-token").result; + relationship.connectLatestSocket(); + relationship.beginOwnedCreate("second-create", "second-execution"); + const second = await relationship.ownedCreateResult("second-create"); + + expect(relationship.relationshipFile()?.relationship.daemonId).toBe(daemonId); + expect(first).toMatchObject({ payload: { success: true, executionId: "first-execution" } }); + expect(second).toMatchObject({ payload: { success: true, executionId: "second-execution" } }); + expect(relationship.providerCreations()).toBe(2); + }); + + test("daemon shutdown fences a pending create before closing owned agents", async () => { + relationship = await HubRelationshipHarness.start(); + await relationship.beginConnect().result; + relationship.connectLatestSocket(); + relationship.holdAgentCreation(); + relationship.beginOwnedCreate("shutdown-create", "execution-shutdown", { prompt: "sleep 30" }); + await relationship.agentCreationAttempts(1); + + const shutdown = relationship.shutdownDaemon(); + relationship.finishAgentCreation(); + await shutdown; + + expect(relationship.socketDeliveredResponse(0, "shutdown-create")).toBe(false); + expect(await relationship.durableOwnedAgentIdsOnDisk()).toEqual([]); + }); + + test.each([ + [4403, "Hub revoked this relationship"], + [401, "Hub rejected socket authentication (401)"], + [403, "Hub rejected socket authentication (403)"], + ] as const)("authentication rejection %s revokes permanently", async (code, reason) => { + relationship = await HubRelationshipHarness.start(); + await relationship.beginConnect("private-token").result; + const enrollment = relationship.enrollmentAttempts()[0]; + const secret = relationship.relationshipFile()?.credential?.secret; + relationship.rejectRelationship(code); + + await relationship.restartDaemon(); + const status = await relationship.status(); + const persisted = relationship.relationshipFile(); + + expect(status).toMatchObject({ + state: "revoked", + daemonId: enrollment.daemonId, + hub: "https://hub.test", + scopes: "hub.execution.*", + error: reason, + }); + expect(persisted?.state).toBe("revoked"); + expect(persisted?.relationship).toMatchObject({ + daemonId: enrollment.daemonId, + hubOrigin: "https://hub.test", + scopes: ["hub.execution.*"], + }); + expect(persisted?.reason).toBe(reason); + expect(persisted).not.toHaveProperty("credential"); + expect(persisted).not.toHaveProperty("relationship.idempotencyKey"); + expect(relationship.socketAttempts()).toBe(1); + const loggable = relationship.loggableValues(status); + const reconstructed = JSON.stringify({ status, persisted }); + expect(reconstructed).not.toContain(secret); + expect(reconstructed).not.toContain(enrollment.credentialVerifier); + expect(reconstructed).not.toContain(enrollment.token); + expect(reconstructed).not.toContain(enrollment.idempotencyKey); + expect(loggable).not.toContain(secret); + expect(loggable).not.toContain(enrollment.credentialVerifier); + expect(loggable).not.toContain(enrollment.token); + expect(loggable).not.toContain(enrollment.idempotencyKey); + }); + + test("offline disconnect retries across runtime and restart without opening a socket", async () => { + relationship = await HubRelationshipHarness.start(); + await relationship.beginConnect().result; + relationship.connectLatestSocket(); + relationship.failRevocations(3); + + const disconnecting = await relationship.disconnect(); + await relationship.retry(); + await relationship.restartDaemon(); + await relationship.retry(); + + expect(disconnecting.state).toBe("disconnecting"); + expect(relationship.revocationAttempts()).toBe(4); + expect(relationship.socketAttempts()).toBe(1); + expect(relationship.relationshipFile()).toBeNull(); + }); + + test("successful disconnect clears a transient revocation error", async () => { + relationship = await HubRelationshipHarness.start(); + await relationship.beginConnect().result; + relationship.failRevocations(1); + + const disconnecting = await relationship.disconnect(); + await relationship.retry(); + + expect(disconnecting).toMatchObject({ state: "disconnecting", error: expect.any(String) }); + expect(await relationship.status()).toMatchObject({ state: "not_connected", error: null }); + }); + + test("force disconnect removes local authority and reports the remote warning", async () => { + relationship = await HubRelationshipHarness.start(); + await relationship.beginConnect().result; + relationship.failRevocations(1); + await relationship.disconnect(); + + const forced = await relationship.disconnect(true); + + expect(forced.state).toBe("not_connected"); + expect(forced.warning).toContain("remote revocation"); + expect(relationship.relationshipFile()).toBeNull(); + }); + + test("disconnect leaves no intent artifacts and shutdown closes the owned agent", async () => { + relationship = await HubRelationshipHarness.start(); + await relationship.beginConnect().result; + relationship.connectLatestSocket(); + relationship.beginOwnedCreate("orphan-create", "execution-orphan", { prompt: "sleep 30" }); + const created = await relationship.ownedCreateResult("orphan-create"); + expect(created).toMatchObject({ payload: { agent: { status: "running" } } }); + + await relationship.disconnect(true); + expect(await relationship.hubExecutionIntentFiles()).toEqual([]); + await relationship.restartDaemon(); + + expect(await relationship.storedOwnedStatus(created.payload.agentId!)).toBe("closed"); + }); +}); diff --git a/packages/server/src/server/hub/relationship-controller.ts b/packages/server/src/server/hub/relationship-controller.ts new file mode 100644 index 000000000..457a5b701 --- /dev/null +++ b/packages/server/src/server/hub/relationship-controller.ts @@ -0,0 +1,556 @@ +import { createHash, randomBytes, randomUUID } from "node:crypto"; +import { existsSync, readFileSync, renameSync, rmSync } from "node:fs"; +import path from "node:path"; +import type pino from "pino"; +import { z } from "zod"; +import { ensurePrivateFile, writePrivateFileAtomicSync } from "../private-files.js"; +import type { WebSocketLike } from "../websocket-server.js"; +import type { HubExecutionAgents } from "./daemon-executions.js"; +import type { + HubRelationshipRemote, + HubSocketConnection, + HubSocketEvents, +} from "./relationship-remote.js"; +import { HubEnrollmentRejectedError } from "./relationship-remote.js"; +import { BoundedExponentialHubRetryPolicy } from "./relationship-retry.js"; + +const FILE_NAME = "hub-relationship.json"; +const HUB_EXECUTION_SCOPE = "hub.execution.*"; +const SCOPES = [HUB_EXECUTION_SCOPE] as const; +const HubOriginSchema = z + .string() + .url() + .superRefine((value, context) => { + try { + normalizeHubUrl(value); + } catch (error) { + context.addIssue({ + code: "custom", + message: error instanceof Error ? error.message : "Invalid Hub URL", + }); + } + }); + +const RelationshipSchema = z.object({ + daemonId: z.string().min(1), + idempotencyKey: z.string().min(1), + hubOrigin: HubOriginSchema, + createdAt: z.string(), + scopes: z.tuple([z.literal(HUB_EXECUTION_SCOPE)]), +}); +const SanitizedRelationshipSchema = RelationshipSchema.omit({ idempotencyKey: true }); +const CredentialSchema = z.object({ secret: z.string().min(1) }); +const TransportSchema = z.object({ + kind: z.literal("direct_websocket"), + webSocketUrl: z + .string() + .url() + .refine((value) => ["ws:", "wss:"].includes(new URL(value).protocol)) + .refine((value) => new URL(value).hash === ""), +}); +const PendingSchema = z.object({ + version: z.literal(1), + state: z.literal("pending"), + relationship: RelationshipSchema, + credential: CredentialSchema, + enrollment: z.object({ token: z.string().min(1) }), + identity: z.object({ serverId: z.string().min(1), daemonPublicKey: z.string().min(1) }), +}); +const ActiveSchema = z.object({ + version: z.literal(1), + state: z.literal("active"), + relationship: RelationshipSchema, + credential: CredentialSchema, + transport: TransportSchema, +}); +const DisconnectingSchema = z.object({ + version: z.literal(1), + state: z.literal("disconnecting"), + relationship: RelationshipSchema, + credential: CredentialSchema, + transport: TransportSchema.optional(), +}); +const RevokedSchema = z.object({ + version: z.literal(1), + state: z.literal("revoked"), + relationship: SanitizedRelationshipSchema, + transport: TransportSchema.optional(), + reason: z.string().optional(), +}); +const RecordSchema = z + .discriminatedUnion("state", [PendingSchema, ActiveSchema, DisconnectingSchema, RevokedSchema]) + .superRefine((record, context) => { + if (!("transport" in record) || !record.transport) return; + const hub = new URL(record.relationship.hubOrigin); + const socket = new URL(record.transport.webSocketUrl); + const expectedProtocol = hub.protocol === "https:" ? "wss:" : "ws:"; + if (socket.protocol === expectedProtocol && socket.host === hub.host) return; + context.addIssue({ + code: "custom", + path: ["transport", "webSocketUrl"], + message: "Hub WebSocket URL must match the Hub origin", + }); + }); +type PendingRecord = z.infer; +type ActiveRecord = z.infer; +type DisconnectingRecord = z.infer; +type RevokedRecord = z.infer; +type HubRelationshipRecord = z.infer; + +export type HubConnectionState = + | "not_connected" + | "connecting" + | "connected" + | "reconnecting" + | "disconnecting" + | "revoked"; + +export interface HubRelationshipStatus { + state: HubConnectionState; + daemonId: string | null; + hubOrigin: string | null; + scopes: string[]; + connectedAt: string | null; + lastError: string | null; +} + +export interface HubRelationshipManagement { + connect(input: { hubUrl: string; token: string }): Promise; + status(): HubRelationshipStatus; + disconnect(input: { + force: boolean; + }): Promise<{ status: HubRelationshipStatus; warning?: string }>; +} + +export interface ScheduledRelationshipTask { + cancel(): void; +} + +export interface HubRelationshipClock { + now(): Date; + schedule(delayMs: number, task: () => void): ScheduledRelationshipTask; +} + +export interface HubRelationshipRetryPolicy { + delay(attempt: number): number; +} + +export interface HubRelationshipControllerOptions { + paseoHome: string; + serverId: string; + daemonPublicKey: string; + logger: pino.Logger; + remote: HubRelationshipRemote; + clock?: HubRelationshipClock; + retryPolicy?: HubRelationshipRetryPolicy; + createDaemonId?: () => string; + attachSocket: ( + socket: WebSocketLike, + options: { daemonId: string; scopes: readonly string[]; agents: HubExecutionAgents }, + ) => Promise; + createExecutionAgents: (daemonId: string) => HubExecutionAgents; +} + +const systemClock: HubRelationshipClock = { + now: () => new Date(), + schedule(delayMs, task) { + const timer = setTimeout(task, delayMs); + timer.unref?.(); + return { cancel: () => clearTimeout(timer) }; + }, +}; + +function normalizeHubUrl(value: string): string { + const url = new URL(value); + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new Error("Hub URL must use HTTP or HTTPS"); + } + if (url.username || url.password) { + throw new Error("Hub URL cannot include credentials"); + } + if (url.search || url.hash) { + throw new Error("Hub URL cannot include a query or fragment"); + } + url.pathname = url.pathname.replace(/\/$/, ""); + return url.toString().replace(/\/$/, ""); +} + +export class HubRelationshipController implements HubRelationshipManagement { + private readonly filePath: string; + private readonly clock: HubRelationshipClock; + private readonly retryPolicy: HubRelationshipRetryPolicy; + private record: HubRelationshipRecord | null; + private state: HubConnectionState = "not_connected"; + private connectedAt: string | null = null; + private lastError: string | null = null; + private socket: HubSocketConnection | null = null; + private retry: ScheduledRelationshipTask | null = null; + private generation = 0; + private enrollmentGeneration = 0; + private retryAttempt = 0; + private readonly inFlightEnrollments = new Set>(); + private executionAgents: { daemonId: string; value: HubExecutionAgents } | null = null; + + constructor(private readonly options: HubRelationshipControllerOptions) { + this.filePath = path.join(options.paseoHome, FILE_NAME); + this.clock = options.clock ?? systemClock; + this.retryPolicy = options.retryPolicy ?? new BoundedExponentialHubRetryPolicy(); + this.record = this.load(); + if (this.record?.state === "revoked") { + this.state = "revoked"; + this.lastError = this.record.reason ?? null; + } else if (this.record?.state === "disconnecting") this.state = "disconnecting"; + else if (this.record) this.state = "connecting"; + } + + async start(): Promise { + if (this.record?.state === "active") this.openSocket(this.record, false); + if (this.record?.state === "pending") { + const enrollmentGeneration = this.beginEnrollmentAttempt(); + try { + await this.tryEnrollment(this.record, enrollmentGeneration); + } catch (error) { + if (!(error instanceof HubEnrollmentRejectedError)) throw error; + this.options.logger.warn( + { statusCode: error.statusCode }, + "Discarded rejected pending Hub enrollment during startup", + ); + } + } + if (this.record?.state === "disconnecting") await this.tryRevocation(this.record); + } + + async stop(): Promise { + const pendingExecutionCleanup = this.retireExecutionAgents(); + this.cancelLifecycle(); + this.socket?.close(); + this.socket = null; + await pendingExecutionCleanup; + } + + status(): HubRelationshipStatus { + return { + state: this.state, + daemonId: this.record?.relationship.daemonId ?? null, + hubOrigin: this.record?.relationship.hubOrigin ?? null, + scopes: this.record?.relationship.scopes.slice() ?? [], + connectedAt: this.connectedAt, + lastError: this.lastError, + }; + } + + async connect(input: { hubUrl: string; token: string }): Promise { + if (this.record?.state === "pending") { + if (normalizeHubUrl(input.hubUrl) !== this.record.relationship.hubOrigin) { + throw new Error("A pending Hub enrollment already exists for a different Hub"); + } + this.record = { ...this.record, enrollment: { token: input.token } }; + const enrollmentGeneration = this.beginEnrollmentAttempt(); + this.persist(this.record); + this.state = "connecting"; + this.lastError = null; + await this.tryEnrollment(this.record, enrollmentGeneration); + return this.status(); + } + if (this.record && this.record.state !== "revoked") { + throw new Error("This daemon already has a Hub relationship"); + } + const pending: PendingRecord = { + version: 1, + state: "pending", + relationship: { + daemonId: this.options.createDaemonId?.() ?? randomUUID(), + idempotencyKey: randomUUID(), + hubOrigin: normalizeHubUrl(input.hubUrl), + createdAt: this.clock.now().toISOString(), + scopes: [...SCOPES], + }, + credential: { secret: randomBytes(32).toString("base64url") }, + enrollment: { token: input.token }, + identity: { serverId: this.options.serverId, daemonPublicKey: this.options.daemonPublicKey }, + }; + this.persist(pending); + this.record = pending; + this.state = "connecting"; + this.lastError = null; + await this.tryEnrollment(pending, this.beginEnrollmentAttempt()); + return this.status(); + } + + async disconnect(input: { + force: boolean; + }): Promise<{ status: HubRelationshipStatus; warning?: string }> { + const waitForEnrollment = this.record?.state === "pending"; + const pendingCreateCleanup = this.retireExecutionAgents(); + this.cancelLifecycle(); + this.socket?.close(); + this.socket = null; + if (!this.record || this.record.state === "revoked") { + this.remove(); + await pendingCreateCleanup; + return { status: this.status() }; + } + if (input.force) { + this.remove(); + await pendingCreateCleanup; + return { + status: this.status(), + warning: "Local Hub credential removed; remote revocation may remain pending.", + }; + } + const disconnecting: DisconnectingRecord = { + version: 1, + state: "disconnecting", + relationship: this.record.relationship, + credential: this.record.credential, + ...(this.record.state === "active" ? { transport: this.record.transport } : {}), + }; + this.persist(disconnecting); + this.record = disconnecting; + this.state = "disconnecting"; + if (waitForEnrollment) { + await Promise.all(this.inFlightEnrollments); + } + await this.tryRevocation(disconnecting); + await pendingCreateCleanup; + return { status: this.status() }; + } + + private async tryEnrollment(pending: PendingRecord, enrollmentGeneration: number): Promise { + if (enrollmentGeneration !== this.enrollmentGeneration) return; + const verifier = createHash("sha256").update(pending.credential.secret).digest("base64url"); + const request = this.options.remote.enroll({ + daemonId: pending.relationship.daemonId, + idempotencyKey: pending.relationship.idempotencyKey, + hubOrigin: pending.relationship.hubOrigin, + token: pending.enrollment.token, + serverId: pending.identity.serverId, + daemonPublicKey: pending.identity.daemonPublicKey, + credentialVerifier: verifier, + scopes: pending.relationship.scopes, + }); + const settled = request.then( + () => undefined, + () => undefined, + ); + this.inFlightEnrollments.add(settled); + try { + const enrollment = await request; + if (enrollmentGeneration !== this.enrollmentGeneration) return; + if ( + enrollment.daemonId !== pending.relationship.daemonId || + !enrollment.scopes.includes(HUB_EXECUTION_SCOPE) + ) { + throw new Error("Hub enrollment response did not match the pending relationship"); + } + const active: ActiveRecord = { + version: 1, + state: "active", + relationship: pending.relationship, + credential: pending.credential, + transport: { kind: "direct_websocket", webSocketUrl: enrollment.webSocketUrl }, + }; + this.persist(active); + this.record = active; + this.retry = null; + this.retryAttempt = 0; + this.openSocket(active, false); + } catch (error) { + if (enrollmentGeneration !== this.enrollmentGeneration) return; + if (error instanceof HubEnrollmentRejectedError) { + this.remove(); + throw error; + } + this.lastError = error instanceof Error ? error.message : String(error); + this.scheduleEnrollment(pending, enrollmentGeneration); + } finally { + this.inFlightEnrollments.delete(settled); + } + } + + private openSocket(record: ActiveRecord, reconnecting: boolean): void { + const generation = ++this.generation; + this.state = reconnecting ? "reconnecting" : "connecting"; + const events: HubSocketEvents = { + connected: (socket) => this.socketConnected(generation, record, socket), + rejected: (statusCode) => this.socketRejected(generation, statusCode), + closed: (code) => this.socketClosed(generation, record, code), + failed: (error) => this.socketFailed(generation, record, error), + }; + this.socket = this.options.remote.openSocket( + { + daemonId: record.relationship.daemonId, + webSocketUrl: record.transport.webSocketUrl, + credential: record.credential.secret, + }, + events, + ); + } + + private socketConnected(generation: number, record: ActiveRecord, socket: WebSocketLike): void { + if (generation !== this.generation) { + socket.close(); + return; + } + this.retryAttempt = 0; + this.state = "connected"; + this.connectedAt = this.clock.now().toISOString(); + this.lastError = null; + void this.options.attachSocket(socket, { + daemonId: record.relationship.daemonId, + scopes: record.relationship.scopes, + agents: this.executionAgentsFor(record.relationship.daemonId), + }); + } + + private executionAgentsFor(daemonId: string): HubExecutionAgents { + if (this.executionAgents?.daemonId === daemonId) return this.executionAgents.value; + const value = this.options.createExecutionAgents(daemonId); + this.executionAgents = { daemonId, value }; + return value; + } + + private retireExecutionAgents(): Promise { + const executionAgents = this.executionAgents; + this.executionAgents = null; + return executionAgents?.value.invalidateAuthority() ?? Promise.resolve(); + } + + private socketRejected(generation: number, statusCode: 401 | 403): void { + if (generation !== this.generation) return; + this.revoke(`Hub rejected socket authentication (${statusCode})`); + } + + private socketClosed(generation: number, record: ActiveRecord, code: number): void { + if (generation !== this.generation) return; + if (code === 4403) { + this.revoke("Hub revoked this relationship"); + return; + } + if (this.record?.state === "active") this.scheduleSocket(record); + } + + private socketFailed(generation: number, record: ActiveRecord, error: Error): void { + if (generation !== this.generation) return; + this.lastError = error.message; + if (this.record?.state === "active") this.scheduleSocket(record); + } + + private scheduleSocket(record: ActiveRecord): void { + this.state = "reconnecting"; + this.schedule(() => this.openSocket(record, true)); + } + + private scheduleEnrollment(record: PendingRecord, enrollmentGeneration: number): void { + if (enrollmentGeneration !== this.enrollmentGeneration) return; + this.state = "reconnecting"; + this.retry?.cancel(); + const delay = this.retryPolicy.delay(this.retryAttempt++); + this.retry = this.clock.schedule(delay, () => { + if (enrollmentGeneration !== this.enrollmentGeneration) return; + void this.tryEnrollment(record, enrollmentGeneration).catch((error: unknown) => { + if (error instanceof HubEnrollmentRejectedError) return; + this.options.logger.error({ err: error }, "Scheduled Hub enrollment retry failed"); + }); + }); + } + + private async tryRevocation(record: DisconnectingRecord): Promise { + const generation = this.generation; + try { + await this.options.remote.revoke({ + daemonId: record.relationship.daemonId, + hubOrigin: record.relationship.hubOrigin, + credential: record.credential.secret, + }); + if (generation !== this.generation) return; + this.remove(); + } catch (error) { + if (generation !== this.generation) return; + this.lastError = error instanceof Error ? error.message : String(error); + this.state = "disconnecting"; + this.schedule(() => void this.tryRevocation(record)); + } + } + + private schedule(task: () => void): void { + this.retry?.cancel(); + const generation = this.generation; + const delay = this.retryPolicy.delay(this.retryAttempt++); + this.retry = this.clock.schedule(delay, () => { + if (generation === this.generation) task(); + }); + } + + private revoke(reason: string): void { + void this.retireExecutionAgents(); + this.cancelLifecycle(); + if (!this.record) return; + const revoked: RevokedRecord = { + version: 1, + state: "revoked", + relationship: { + daemonId: this.record.relationship.daemonId, + hubOrigin: this.record.relationship.hubOrigin, + createdAt: this.record.relationship.createdAt, + scopes: this.record.relationship.scopes, + }, + transport: "transport" in this.record ? this.record.transport : undefined, + reason, + }; + this.persist(revoked); + this.record = revoked; + this.state = "revoked"; + this.lastError = reason; + } + + private cancelLifecycle(): void { + ++this.generation; + ++this.enrollmentGeneration; + this.retry?.cancel(); + this.retry = null; + } + + private beginEnrollmentAttempt(): number { + this.retry?.cancel(); + this.retry = null; + this.retryAttempt = 0; + return ++this.enrollmentGeneration; + } + + private persist(record: HubRelationshipRecord): void { + writePrivateFileAtomicSync(this.filePath, `${JSON.stringify(record, null, 2)}\n`); + } + + private remove(): void { + void this.retireExecutionAgents(); + this.cancelLifecycle(); + rmSync(this.filePath, { force: true }); + this.record = null; + this.state = "not_connected"; + this.connectedAt = null; + this.lastError = null; + } + + private load(): HubRelationshipRecord | null { + if (!existsSync(this.filePath)) return null; + let record: HubRelationshipRecord; + try { + record = RecordSchema.parse(JSON.parse(readFileSync(this.filePath, "utf8"))); + } catch (error) { + const quarantinePath = path.join( + path.dirname(this.filePath), + `hub-relationship.invalid-${this.clock.now().getTime()}-${randomUUID()}.json`, + ); + renameSync(this.filePath, quarantinePath); + ensurePrivateFile(quarantinePath); + this.options.logger.error( + { err: error, quarantinePath }, + "Quarantined invalid Hub relationship authority", + ); + return null; + } + ensurePrivateFile(this.filePath); + return record; + } +} diff --git a/packages/server/src/server/hub/relationship-remote.test.ts b/packages/server/src/server/hub/relationship-remote.test.ts new file mode 100644 index 000000000..e4950adab --- /dev/null +++ b/packages/server/src/server/hub/relationship-remote.test.ts @@ -0,0 +1,551 @@ +import { createServer } from "node:http"; +import { mkdtemp, rm } from "node:fs/promises"; +import type { AddressInfo, Socket } from "node:net"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import pino from "pino"; +import { afterEach, expect, test } from "vitest"; +import { WebSocket } from "ws"; +import type { HubExecutionAgents } from "./daemon-executions.js"; +import { + HubRelationshipController, + type HubRelationshipClock, + type HubRelationshipRetryPolicy, + type ScheduledRelationshipTask, +} from "./relationship-controller.js"; +import { + DirectHubRelationshipRemote, + HubEnrollmentRejectedError, + type HubSocketConnection, + type HubSocketEvents, +} from "./relationship-remote.js"; + +const openServers: ReturnType[] = []; +const openUpgradeHubs: UpgradeRejectingHub[] = []; +const openPaseoHomes: string[] = []; + +afterEach(async () => { + for (const hub of openUpgradeHubs.splice(0)) hub.destroyConnections(); + await Promise.all(openServers.splice(0).map((server) => closeServer(server))); + await Promise.all(openPaseoHomes.splice(0).map((home) => rm(home, { recursive: true }))); +}); + +test.each([401, 403, 404])( + "revocation status %s clears already-invalid local authority", + async (status) => { + const hubOrigin = await startHubReturning(status); + const remote = new DirectHubRelationshipRemote(); + + await expect( + remote.revoke({ daemonId: "daemon-1", hubOrigin, credential: "invalid" }), + ).resolves.toBeUndefined(); + }, +); + +test("transient revocation failures remain retryable", async () => { + const hubOrigin = await startHubReturning(503); + const remote = new DirectHubRelationshipRemote(); + + await expect( + remote.revoke({ daemonId: "daemon-1", hubOrigin, credential: "credential" }), + ).rejects.toThrow("Hub revocation failed (503)"); +}); + +test.each([408, 429])("transient enrollment status %s remains retryable", async (status) => { + const hubOrigin = await startHubReturning(status); + const remote = new DirectHubRelationshipRemote(); + + const error = await remote + .enroll({ + daemonId: "daemon-1", + idempotencyKey: "ceremony-1", + hubOrigin, + token: "token", + serverId: "server-1", + daemonPublicKey: "public-key", + credentialVerifier: "verifier", + scopes: ["hub.execution.*"], + }) + .catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(Error); + expect(error).not.toBeInstanceOf(HubEnrollmentRejectedError); + expect((error as Error).message).toBe(`Hub enrollment failed (${status})`); +}); + +test("enrollment rejects a transport URL that cannot open a WebSocket", async () => { + const hubOrigin = await startEnrollmentHub("ftp://hub.test/daemon"); + const remote = new DirectHubRelationshipRemote(); + + await expect( + remote.enroll({ + daemonId: "daemon-1", + idempotencyKey: "ceremony-1", + hubOrigin, + token: "token", + serverId: "server-1", + daemonPublicKey: "public-key", + credentialVerifier: "verifier", + scopes: ["hub.execution.*"], + }), + ).rejects.toThrow("Hub WebSocket URL must use ws or wss"); +}); + +test("enrollment rejects a WebSocket URL with a fragment", async () => { + const hubOrigin = await startEnrollmentHub("ws://hub.test/daemon#fragment"); + const remote = new DirectHubRelationshipRemote(); + + await expect( + remote.enroll({ + daemonId: "daemon-1", + idempotencyKey: "ceremony-1", + hubOrigin, + token: "token", + serverId: "server-1", + daemonPublicKey: "public-key", + credentialVerifier: "verifier", + scopes: ["hub.execution.*"], + }), + ).rejects.toThrow("Hub WebSocket URL cannot include a fragment"); +}); + +test("enrollment rejects a WebSocket outside the enrolled Hub authority", async () => { + const hubOrigin = await startEnrollmentHub("ws://other-hub.test/daemon"); + const remote = new DirectHubRelationshipRemote(); + + await expect( + remote.enroll({ + daemonId: "daemon-1", + idempotencyKey: "ceremony-1", + hubOrigin, + token: "token", + serverId: "server-1", + daemonPublicKey: "public-key", + credentialVerifier: "verifier", + scopes: ["hub.execution.*"], + }), + ).rejects.toThrow("Hub WebSocket URL must match the Hub origin"); +}); + +test.each(["enrollment", "revocation"])("%s HTTP calls are bounded", async (operation) => { + const hubOrigin = await startStalledHub(); + const remote = new DirectHubRelationshipRemote({ requestTimeoutMs: 25 }); + + const request = + operation === "enrollment" + ? remote.enroll({ + daemonId: "daemon-1", + idempotencyKey: "ceremony-1", + hubOrigin, + token: "token", + serverId: "server-1", + daemonPublicKey: "public-key", + credentialVerifier: "verifier", + scopes: ["hub.execution.*"], + }) + : remote.revoke({ + daemonId: "daemon-1", + hubOrigin, + credential: "credential", + }); + + await expect(request).rejects.toThrow("Hub request timed out"); +}); + +test.each([401, 403] as const)( + "socket authentication status %s is terminal and releases the failed upgrade", + async (status) => { + const hub = await UpgradeRejectingHub.start([status]); + const outcomes = new SocketOutcomes(); + const remote = new DirectHubRelationshipRemote(); + + const socket = remote.openSocket( + { + daemonId: "daemon-1", + webSocketUrl: hub.webSocketUrl, + credential: "invalid", + }, + outcomes.events, + ) as HubSocketConnection & WebSocket; + + expect(await outcomes.next()).toEqual({ type: "rejected", status }); + await hub.expectAttemptReleased(1); + expect(await outcomes.afterCleanup()).toEqual([{ type: "rejected", status }]); + expect(socket.readyState).toBe(WebSocket.CLOSED); + expect(hub.openConnections()).toBe(0); + }, +); + +test("a transient socket response releases the failed upgrade and reports one connection loss", async () => { + const hub = await UpgradeRejectingHub.start([503]); + const outcomes = new SocketOutcomes(); + const remote = new DirectHubRelationshipRemote(); + + const socket = remote.openSocket( + { + daemonId: "daemon-1", + webSocketUrl: hub.webSocketUrl, + credential: "credential", + }, + outcomes.events, + ) as HubSocketConnection & WebSocket; + + expect(await outcomes.next()).toEqual({ type: "closed", code: 1006 }); + await hub.expectAttemptReleased(1); + expect(await outcomes.afterCleanup()).toEqual([{ type: "closed", code: 1006 }]); + expect(socket.readyState).toBe(WebSocket.CLOSED); + expect(hub.openConnections()).toBe(0); +}); + +test("a stalled socket opening handshake times out and releases the connection", async () => { + const hub = await UpgradeRejectingHub.start(["stall"]); + const outcomes = new SocketOutcomes(); + const remote = new DirectHubRelationshipRemote({ requestTimeoutMs: 25 }); + + const socket = remote.openSocket( + { + daemonId: "daemon-1", + webSocketUrl: hub.webSocketUrl, + credential: "credential", + }, + outcomes.events, + ) as HubSocketConnection & WebSocket; + + expect(await outcomes.next()).toEqual({ + type: "failed", + message: "Opening handshake has timed out", + }); + expect(await outcomes.afterCleanup()).toEqual([ + { type: "failed", message: "Opening handshake has timed out" }, + ]); + expect(socket.readyState).toBe(WebSocket.CLOSED); +}); + +test.each([401, 403] as const)( + "controller treats socket authentication status %s as terminal without redialing", + async (status) => { + const hub = await UpgradeRejectingHub.start([status]); + const clock = new ManualRelationshipClock(); + const controller = await connectController(hub, clock); + + await hub.expectAttemptReleased(1); + + expect(controller.status()).toMatchObject({ + state: "revoked", + lastError: `Hub rejected socket authentication (${status})`, + }); + expect(hub.attemptCount()).toBe(1); + expect(clock.pendingTasks()).toBe(0); + expect(hub.openConnections()).toBe(0); + }, +); + +test("controller redials after a transient socket response and releases both failed upgrades", async () => { + const hub = await UpgradeRejectingHub.start([503, 401]); + const clock = new ManualRelationshipClock(); + const controller = await connectController(hub, clock); + + await hub.expectAttemptReleased(1); + expect(controller.status().state).toBe("reconnecting"); + expect(clock.pendingTasks()).toBe(1); + + clock.runNext(); + await hub.expectAttemptReleased(2); + + expect(hub.attemptCount()).toBe(2); + expect(controller.status().state).toBe("revoked"); + expect(clock.pendingTasks()).toBe(0); + expect(hub.openConnections()).toBe(0); +}); + +test("controller redials once after a failed upgrade without also handling its close", async () => { + const hub = await UpgradeRejectingHub.start([0, 401]); + const clock = new ManualRelationshipClock(); + const controller = await connectController(hub, clock); + + await hub.expectAttemptReleased(1); + + expect(controller.status().state).toBe("reconnecting"); + expect(clock.scheduledAttempts()).toEqual([0]); + expect(clock.pendingTasks()).toBe(1); + + clock.runNext(); + await hub.expectAttemptReleased(2); + + expect(hub.attemptCount()).toBe(2); + expect(controller.status().state).toBe("revoked"); + expect(clock.pendingTasks()).toBe(0); +}); + +async function startHubReturning(status: number): Promise { + const server = createServer((_request, response) => { + response.writeHead(status).end(); + }); + openServers.push(server); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address() as AddressInfo; + return `http://127.0.0.1:${address.port}`; +} + +async function startEnrollmentHub(webSocketUrl: string): Promise { + const server = createServer(async (request, response) => { + let body = ""; + for await (const chunk of request) body += chunk; + const enrollment = JSON.parse(body) as { daemonId: string }; + response.writeHead(200, { "content-type": "application/json" }).end( + JSON.stringify({ + daemonId: enrollment.daemonId, + scopes: ["hub.execution.*"], + webSocketUrl, + }), + ); + }); + openServers.push(server); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address() as AddressInfo; + return `http://127.0.0.1:${address.port}`; +} + +async function startStalledHub(): Promise { + const server = createServer(() => undefined); + openServers.push(server); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address() as AddressInfo; + return `http://127.0.0.1:${address.port}`; +} + +function closeServer(server: ReturnType): Promise { + return new Promise((resolve, reject) => { + server.close((error) => { + if (error) { + reject(error); + return; + } + resolve(); + }); + }); +} + +type SocketOutcome = + | { type: "connected" } + | { type: "rejected"; status: 401 | 403 } + | { type: "closed"; code: number } + | { type: "failed"; message: string }; + +interface Deferred { + promise: Promise; + resolve(value: T): void; +} + +function deferred(): Deferred { + let resolve!: (value: T) => void; + const promise = new Promise((accept) => { + resolve = accept; + }); + return { promise, resolve }; +} + +class SocketOutcomes { + private readonly outcomes: SocketOutcome[] = []; + private nextOutcome = deferred(); + + readonly events: HubSocketEvents = { + connected: () => this.record({ type: "connected" }), + rejected: (status) => this.record({ type: "rejected", status }), + closed: (code) => this.record({ type: "closed", code }), + failed: (error) => this.record({ type: "failed", message: error.message }), + }; + + next(): Promise { + return withDeadline(this.nextOutcome.promise, "Hub socket produced no outcome"); + } + + async afterCleanup(): Promise { + await new Promise((resolve) => setImmediate(resolve)); + return this.outcomes.slice(); + } + + private record(outcome: SocketOutcome): void { + this.outcomes.push(outcome); + this.nextOutcome.resolve(outcome); + this.nextOutcome = deferred(); + } +} + +class UpgradeRejectingHub { + private readonly server = createServer(async (request, response) => { + let body = ""; + for await (const chunk of request) body += chunk; + const enrollment = JSON.parse(body) as { daemonId: string }; + response.writeHead(200, { "content-type": "application/json" }).end( + JSON.stringify({ + daemonId: enrollment.daemonId, + scopes: ["hub.execution.*"], + webSocketUrl: this.webSocketUrl, + }), + ); + }); + private readonly sockets = new Set(); + private readonly releasedAttempts = new Map>(); + private attemptObserved = deferred(); + private attempts = 0; + + private constructor(private readonly statuses: Array) { + this.server.on("upgrade", (_request, socket) => { + const attempt = ++this.attempts; + this.attemptObserved.resolve(); + this.attemptObserved = deferred(); + const released = deferred(); + this.releasedAttempts.set(attempt, released); + this.sockets.add(socket); + socket.once("close", () => { + this.sockets.delete(socket); + released.resolve(); + }); + const status = this.statuses[attempt - 1] ?? this.statuses.at(-1) ?? 503; + if (status === "stall") { + socket.resume(); + return; + } + if (status === 0) { + socket.destroy(); + return; + } + socket.end( + `HTTP/1.1 ${status} Rejected\r\nContent-Length: 4\r\nConnection: close\r\n\r\ndeny`, + ); + }); + } + + static async start(statuses: Array): Promise { + const hub = new UpgradeRejectingHub(statuses); + openUpgradeHubs.push(hub); + openServers.push(hub.server); + await new Promise((resolve, reject) => { + hub.server.once("error", reject); + hub.server.listen(0, "127.0.0.1", resolve); + }); + return hub; + } + + get webSocketUrl(): string { + const address = this.server.address() as AddressInfo; + return `ws://127.0.0.1:${address.port}/daemon`; + } + + get origin(): string { + const address = this.server.address() as AddressInfo; + return `http://127.0.0.1:${address.port}`; + } + + attemptCount(): number { + return this.attempts; + } + + openConnections(): number { + return this.sockets.size; + } + + destroyConnections(): void { + for (const socket of this.sockets) socket.destroy(); + } + + async expectAttemptReleased(attempt: number): Promise { + while (this.attempts < attempt) { + await withDeadline( + this.attemptObserved.promise, + `Hub did not receive socket attempt ${attempt}`, + ); + } + const released = this.releasedAttempts.get(attempt); + if (!released) throw new Error(`Hub did not receive socket attempt ${attempt}`); + await withDeadline(released.promise, `Hub socket attempt ${attempt} remained open`); + await new Promise((resolve) => setImmediate(resolve)); + } +} + +class ManualRelationshipClock implements HubRelationshipClock, HubRelationshipRetryPolicy { + private readonly tasks: Array<{ cancelled: boolean; task: () => void }> = []; + private readonly attempts: number[] = []; + + now(): Date { + return new Date("2026-07-13T00:00:00.000Z"); + } + + delay(attempt: number): number { + this.attempts.push(attempt); + return 0; + } + + schedule(_delayMs: number, task: () => void): ScheduledRelationshipTask { + const scheduled = { cancelled: false, task }; + this.tasks.push(scheduled); + return { cancel: () => (scheduled.cancelled = true) }; + } + + pendingTasks(): number { + return this.tasks.filter((task) => !task.cancelled).length; + } + + scheduledAttempts(): number[] { + return this.attempts.slice(); + } + + runNext(): void { + const scheduled = this.tasks.shift(); + if (!scheduled || scheduled.cancelled) throw new Error("No Hub reconnect is scheduled"); + scheduled.task(); + } +} + +const unusedExecutionAgents: HubExecutionAgents = { + create: async () => { + throw new Error("Unexpected Hub agent create"); + }, + subscribe: () => () => undefined, + invalidateAuthority: async () => undefined, +}; + +async function connectController( + hub: UpgradeRejectingHub, + clock: ManualRelationshipClock, +): Promise { + const paseoHome = await mkdtemp(path.join(tmpdir(), "paseo-hub-socket-")); + openPaseoHomes.push(paseoHome); + const controller = new HubRelationshipController({ + paseoHome, + serverId: "server-1", + daemonPublicKey: "daemon-public-key", + logger: pino({ level: "silent" }), + remote: new DirectHubRelationshipRemote(), + clock, + retryPolicy: clock, + attachSocket: async () => undefined, + createExecutionAgents: () => unusedExecutionAgents, + }); + await controller.connect({ hubUrl: hub.origin, token: "enrollment-token" }); + return controller; +} + +async function withDeadline(promise: Promise, message: string): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + promise, + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error(message)), 2_000); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} diff --git a/packages/server/src/server/hub/relationship-remote.ts b/packages/server/src/server/hub/relationship-remote.ts new file mode 100644 index 000000000..82172a411 --- /dev/null +++ b/packages/server/src/server/hub/relationship-remote.ts @@ -0,0 +1,187 @@ +import { WebSocket } from "ws"; +import { z } from "zod"; +import type { WebSocketLike } from "../websocket-server.js"; + +export interface HubEnrollment { + daemonId: string; + idempotencyKey: string; + hubOrigin: string; + token: string; + serverId: string; + daemonPublicKey: string; + credentialVerifier: string; + scopes: string[]; +} + +export interface HubEnrollmentResult { + daemonId: string; + scopes: string[]; + webSocketUrl: string; +} + +export interface HubRevocation { + daemonId: string; + hubOrigin: string; + credential: string; +} + +export interface HubSocketCredentials { + daemonId: string; + webSocketUrl: string; + credential: string; +} + +export interface HubSocketEvents { + connected(socket: WebSocketLike): void; + rejected(statusCode: 401 | 403): void; + closed(code: number): void; + failed(error: Error): void; +} + +export interface HubSocketConnection { + close(): void; +} + +export interface HubRelationshipRemote { + enroll(input: HubEnrollment): Promise; + revoke(input: HubRevocation): Promise; + openSocket(input: HubSocketCredentials, events: HubSocketEvents): HubSocketConnection; +} + +export class HubEnrollmentRejectedError extends Error { + constructor(readonly statusCode: number) { + super(`Hub enrollment failed (${statusCode})`); + this.name = "HubEnrollmentRejectedError"; + } +} + +const EnrollmentResultSchema = z.object({ + daemonId: z.string(), + scopes: z.array(z.string()), + webSocketUrl: z + .string() + .url() + .refine((value) => ["ws:", "wss:"].includes(new URL(value).protocol), { + message: "Hub WebSocket URL must use ws or wss", + }) + .refine((value) => new URL(value).hash === "", { + message: "Hub WebSocket URL cannot include a fragment", + }), +}); + +function ensureWebSocketMatchesHubOrigin(hubOrigin: string, webSocketUrl: string): void { + const hub = new URL(hubOrigin); + const socket = new URL(webSocketUrl); + const expectedProtocol = hub.protocol === "https:" ? "wss:" : "ws:"; + if (socket.protocol !== expectedProtocol || socket.host !== hub.host) { + throw new Error("Hub WebSocket URL must match the Hub origin"); + } +} + +export class DirectHubRelationshipRemote implements HubRelationshipRemote { + private readonly requestTimeoutMs: number; + + constructor(options: { requestTimeoutMs?: number } = {}) { + this.requestTimeoutMs = options.requestTimeoutMs ?? 15_000; + } + + async enroll(input: HubEnrollment): Promise { + return this.withRequestTimeout(async (signal) => { + const response = await fetch(`${input.hubOrigin}/api/daemons/enroll`, { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${input.token}`, + }, + body: JSON.stringify({ + daemonId: input.daemonId, + idempotencyKey: input.idempotencyKey, + serverId: input.serverId, + daemonPublicKey: input.daemonPublicKey, + credentialVerifier: input.credentialVerifier, + scopes: input.scopes, + }), + signal, + }); + if (!response.ok) { + if (response.status === 401 || response.status === 403) { + throw new HubEnrollmentRejectedError(response.status); + } + throw new Error(`Hub enrollment failed (${response.status})`); + } + const enrollment = EnrollmentResultSchema.parse(await response.json()); + ensureWebSocketMatchesHubOrigin(input.hubOrigin, enrollment.webSocketUrl); + return enrollment; + }); + } + + async revoke(input: HubRevocation): Promise { + await this.withRequestTimeout(async (signal) => { + const response = await fetch( + `${input.hubOrigin}/api/daemons/${encodeURIComponent(input.daemonId)}`, + { + method: "DELETE", + headers: { authorization: `Bearer ${input.credential}` }, + signal, + }, + ); + if (!response.ok && ![401, 403, 404].includes(response.status)) { + throw new Error(`Hub revocation failed (${response.status})`); + } + }); + } + + openSocket(input: HubSocketCredentials, events: HubSocketEvents): HubSocketConnection { + const socket = new WebSocket(input.webSocketUrl, { + handshakeTimeout: this.requestTimeoutMs, + headers: { + authorization: `Bearer ${input.credential}`, + "x-paseo-daemon-id": input.daemonId, + }, + }); + let settled = false; + socket.once("open", () => { + if (!settled) events.connected(socket as WebSocketLike); + }); + socket.once("unexpected-response", (_request, response) => { + if (settled) { + response.destroy(); + return; + } + settled = true; + response.destroy(); + socket.terminate(); + if (response.statusCode === 401 || response.statusCode === 403) { + events.rejected(response.statusCode); + return; + } + events.closed(1006); + }); + socket.once("close", (code) => { + if (settled) return; + settled = true; + events.closed(code); + }); + socket.once("error", (error) => { + if (settled) return; + settled = true; + socket.terminate(); + events.failed(error); + }); + return socket; + } + + private async withRequestTimeout(operation: (signal: AbortSignal) => Promise): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), this.requestTimeoutMs); + timeout.unref?.(); + try { + return await operation(controller.signal); + } catch (error) { + if (controller.signal.aborted) throw new Error("Hub request timed out", { cause: error }); + throw error; + } finally { + clearTimeout(timeout); + } + } +} diff --git a/packages/server/src/server/hub/relationship-retry.test.ts b/packages/server/src/server/hub/relationship-retry.test.ts new file mode 100644 index 000000000..8331df1d5 --- /dev/null +++ b/packages/server/src/server/hub/relationship-retry.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, test } from "vitest"; +import { BoundedExponentialHubRetryPolicy } from "./relationship-retry.js"; + +describe("Hub relationship retry policy", () => { + test("keeps jitter within the lower and upper bounds", () => { + const lower = new BoundedExponentialHubRetryPolicy(() => 0); + const upper = new BoundedExponentialHubRetryPolicy(() => 1); + + expect(lower.delay(0)).toBe(375); + expect(upper.delay(0)).toBe(625); + }); + + test("grows exponentially and caps the base delay", () => { + const retry = new BoundedExponentialHubRetryPolicy(() => 0.5); + + expect([0, 1, 2, 3, 4, 5, 6, 20].map((attempt) => retry.delay(attempt))).toEqual([ + 500, 1_000, 2_000, 4_000, 8_000, 16_000, 30_000, 30_000, + ]); + }); + + test("never schedules a zero-delay tight loop", () => { + const retry = new BoundedExponentialHubRetryPolicy(() => 0); + + expect([0, 1, 2].map((attempt) => retry.delay(attempt))).toEqual([375, 750, 1_500]); + }); +}); diff --git a/packages/server/src/server/hub/relationship-retry.ts b/packages/server/src/server/hub/relationship-retry.ts new file mode 100644 index 000000000..005aff89f --- /dev/null +++ b/packages/server/src/server/hub/relationship-retry.ts @@ -0,0 +1,16 @@ +import type { HubRelationshipRetryPolicy } from "./relationship-controller.js"; + +const INITIAL_DELAY_MS = 500; +const MAX_BASE_DELAY_MS = 30_000; +const MIN_JITTER_FACTOR = 0.75; +const JITTER_RANGE = 0.5; + +export class BoundedExponentialHubRetryPolicy implements HubRelationshipRetryPolicy { + constructor(private readonly random: () => number = Math.random) {} + + delay(attempt: number): number { + const base = Math.min(MAX_BASE_DELAY_MS, INITIAL_DELAY_MS * 2 ** attempt); + const jitter = MIN_JITTER_FACTOR + this.random() * JITTER_RANGE; + return Math.max(1, Math.round(base * jitter)); + } +} diff --git a/packages/server/src/server/hub/test-utils/relationship-harness.ts b/packages/server/src/server/hub/test-utils/relationship-harness.ts new file mode 100644 index 000000000..6603bff87 --- /dev/null +++ b/packages/server/src/server/hub/test-utils/relationship-harness.ts @@ -0,0 +1,1409 @@ +import { execFile } from "node:child_process"; +import { execFileSync } from "node:child_process"; +import { EventEmitter } from "node:events"; +import { existsSync, readFileSync, realpathSync, statSync, watch } from "node:fs"; +import { mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; +import { networkInterfaces, platform, tmpdir } from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { Writable } from "node:stream"; +import pino from "pino"; +import { WebSocket } from "ws"; +import type { + AgentSnapshotPayload, + HubExecutionAgentCreateResponse, + HubExecutionAgentStream, + HubExecutionAgentUpdate, + RpcErrorMessage, + CreateAgentWorktreeTarget, + SessionOutboundMessage, +} from "../../messages.js"; +import { createPaseoDaemon, type PaseoDaemon, type PaseoDaemonConfig } from "../../bootstrap.js"; +import type { WebSocketLike } from "../../websocket-server.js"; +import type { + AgentClient, + AgentCreateSessionOptions, + AgentLaunchContext, + AgentPromptInput, + AgentPersistenceHandle, + AgentSession, + AgentSessionConfig, + FetchCatalogOptions, + ProviderCatalog, +} from "../../agent/agent-sdk-types.js"; +import { createTestAgentClients } from "../../test-utils/fake-agent-client.js"; +import { DaemonClient } from "../../test-utils/daemon-client.js"; +import { AgentStorage } from "../../agent/agent-storage.js"; +import { AgentManager } from "../../agent/agent-manager.js"; +import { DaemonExecutions } from "../daemon-executions.js"; +import { + createAgentCommand, + type CreateAgentCommandDependencies, +} from "../../agent/create-agent/create.js"; +import type { + HubRelationshipClock, + HubRelationshipRetryPolicy, + ScheduledRelationshipTask, +} from "../relationship-controller.js"; +import type { + HubEnrollment, + HubEnrollmentResult, + HubRelationshipRemote, + HubRevocation, + HubSocketConnection, + HubSocketCredentials, + HubSocketEvents, +} from "../relationship-remote.js"; +import { HubEnrollmentRejectedError } from "../relationship-remote.js"; + +const execFileAsync = promisify(execFile); +const HUB_ORIGIN = "https://hub.test"; +const SOCKET_URL = "wss://hub.test/daemon"; +const REPOSITORY_ROOT = path.resolve(import.meta.dirname, "../../../../../.."); + +export interface ArchiveWatcher { + close(): void; + onError(listener: (error: Error) => void): void; +} + +export interface ArchiveWatchFiles { + watchDirectory(path: string, onChange: () => void): ArchiveWatcher; +} + +const nodeArchiveWatchFiles: ArchiveWatchFiles = { + watchDirectory(directory, onChange) { + const watcher = watch(directory, onChange); + return { + close: () => watcher.close(), + onError: (listener) => watcher.on("error", listener), + }; + }, +}; + +export class SetupFailingArchiveWatchFiles implements ArchiveWatchFiles { + private attempts = 0; + private readonly openDirectories = new Set(); + + constructor(private readonly failOnAttempt: number) {} + + watchDirectory(directory: string): ArchiveWatcher { + this.attempts++; + if (this.attempts === this.failOnAttempt) { + throw new Error(`Cannot watch ${directory}`); + } + this.openDirectories.add(directory); + let open = true; + return { + close: () => { + if (!open) return; + open = false; + this.openDirectories.delete(directory); + }, + onError: () => {}, + }; + } + + activeDirectories(): string[] { + return [...this.openDirectories]; + } +} + +interface PersistedRelationship { + version: number; + state: string; + reason?: string; + relationship: { + daemonId: string; + idempotencyKey?: string; + hubOrigin: string; + scopes: string[]; + }; + credential?: { secret: string }; + enrollment?: { token: string }; + identity?: { serverId: string; daemonPublicKey: string }; +} + +export interface RelationshipInvocationSnapshot { + mode: number; + record: PersistedRelationship; +} + +interface Deferred { + promise: Promise; + resolve(value: T): void; + reject(error: Error): void; +} + +function deferred(): Deferred { + let resolve!: (value: T) => void; + let reject!: (error: Error) => void; + const promise = new Promise((accept, decline) => { + resolve = accept; + reject = decline; + }); + return { promise, resolve, reject }; +} + +class TestRelationshipClock implements HubRelationshipClock, HubRelationshipRetryPolicy { + private current = new Date("2026-07-13T00:00:00.000Z"); + private tasks: Array<{ cancelled: boolean; task: () => void }> = []; + + now(): Date { + return this.current; + } + + delay(attempt: number): number { + return Math.min(8_000, 1_000 * 2 ** attempt); + } + + schedule(_delayMs: number, task: () => void): ScheduledRelationshipTask { + const scheduled = { cancelled: false, task }; + this.tasks.push(scheduled); + return { cancel: () => (scheduled.cancelled = true) }; + } + + async runNext(): Promise { + while (true) { + const scheduled = this.tasks.shift(); + if (!scheduled) throw new Error("No relationship retry is scheduled"); + if (!scheduled.cancelled) { + scheduled.task(); + await Promise.resolve(); + return; + } + } + } + + pendingTasks(): number { + return this.tasks.filter((task) => !task.cancelled).length; + } +} + +class MemoryHubSocket extends EventEmitter implements WebSocketLike, HubSocketConnection { + readyState = 1; + bufferedAmount = 0; + sent: SessionOutboundMessage[] = []; + closed = false; + closeCode: number | null = null; + private messageObserved = deferred(); + + send(data: string | Uint8Array | ArrayBuffer): void { + if (typeof data !== "string") return; + const frame = JSON.parse(data) as { type: "session"; message: SessionOutboundMessage }; + this.sent.push(frame.message); + this.messageObserved.resolve(); + } + + close(code = 1000): void { + this.closed = true; + this.readyState = 3; + this.closeCode = code; + this.emit("close", code); + } + + receive(message: unknown): void { + this.emit("message", JSON.stringify({ type: "session", message }), false); + } + + receiveEnvelope(message: unknown): void { + this.emit("message", JSON.stringify(message), false); + } + + receiveBinary(): void { + this.emit("message", new Uint8Array([1, 2, 3]), true); + } + + async messageFor(requestId: string): Promise { + while (true) { + const message = this.sent.find( + (candidate) => + "payload" in candidate && + "requestId" in candidate.payload && + candidate.payload.requestId === requestId, + ); + if (message) return message; + await this.messageObserved.promise; + this.messageObserved = deferred(); + } + } + + async messageMatching( + predicate: (message: SessionOutboundMessage) => message is TMessage, + ): Promise { + while (true) { + const message = this.sent.find(predicate); + if (message) return message; + await this.messageObserved.promise; + this.messageObserved = deferred(); + } + } +} + +class ControlledAgentClient implements AgentClient { + readonly provider; + readonly capabilities; + private gate: Deferred | null = null; + private creationObserved = deferred(); + creations = 0; + resumes = 0; + createdConfigs: AgentSessionConfig[] = []; + + constructor(private readonly client: AgentClient) { + this.provider = client.provider; + this.capabilities = client.capabilities; + } + + holdCreation(): void { + this.gate = deferred(); + } + + async creationAt(count: number): Promise { + while (this.creations < count) { + await this.creationObserved.promise; + this.creationObserved = deferred(); + } + } + + finishCreation(): void { + if (!this.gate) throw new Error("Agent creation is not held"); + this.gate.resolve(); + this.gate = null; + } + + async createSession( + config: AgentSessionConfig, + launchContext?: AgentLaunchContext, + options?: AgentCreateSessionOptions, + ): Promise { + this.creations++; + this.createdConfigs.push({ ...config }); + this.creationObserved.resolve(); + await this.gate?.promise; + return this.client.createSession(config, launchContext, options); + } + + resumeSession( + handle: AgentPersistenceHandle, + overrides?: Partial, + launchContext?: AgentLaunchContext, + ): Promise { + this.resumes++; + return this.client.resumeSession(handle, overrides, launchContext); + } + + fetchCatalog(options: FetchCatalogOptions): Promise { + return this.client.fetchCatalog(options); + } + + isAvailable(): Promise { + return this.client.isAvailable(); + } +} + +interface SocketAttempt { + input: HubSocketCredentials; + events: HubSocketEvents; + socket: MemoryHubSocket; +} + +class InMemoryHubRelationships implements HubRelationshipRemote { + enrollments: HubEnrollment[] = []; + revocations: HubRevocation[] = []; + sockets: SocketAttempt[] = []; + private readonly enrollmentGates: Array> = []; + private readonly heldEnrollments: Array<{ + input: HubEnrollment; + gate: Deferred; + }> = []; + private enrollmentObserved = deferred(); + private socketObserved = deferred(); + private enrollmentRejection: 401 | 403 | null = null; + private enrollmentScopes = ["hub.execution.*"]; + private revokeFailures = 0; + private readonly relationships = new Set(); + readonly enrollmentSnapshots: RelationshipInvocationSnapshot[] = []; + readonly socketSnapshots: RelationshipInvocationSnapshot[] = []; + + constructor(private readonly captureRelationship: () => RelationshipInvocationSnapshot) {} + + holdEnrollment(): void { + this.enrollmentGates.push(deferred()); + } + + rejectNextEnrollment(statusCode: 401 | 403): void { + this.enrollmentRejection = statusCode; + } + + returnEnrollmentScopes(scopes: string[]): void { + this.enrollmentScopes = scopes.slice(); + } + + async enroll(input: HubEnrollment): Promise { + this.enrollmentSnapshots.push(this.captureRelationship()); + this.enrollments.push({ ...input, scopes: input.scopes.slice() }); + this.relationships.add(input.idempotencyKey); + this.enrollmentObserved.resolve(); + if (this.enrollmentRejection) { + const statusCode = this.enrollmentRejection; + this.enrollmentRejection = null; + throw new HubEnrollmentRejectedError(statusCode); + } + const gate = this.enrollmentGates.shift(); + if (gate) { + this.heldEnrollments.push({ input, gate }); + return gate.promise; + } + return this.enrollmentResult(input); + } + + completeEnrollment(): void { + const held = this.heldEnrollments.at(-1); + if (!held) throw new Error("No enrollment is waiting"); + held.gate.resolve(this.enrollmentResult(held.input)); + this.heldEnrollments.splice(this.heldEnrollments.indexOf(held), 1); + } + + loseEnrollmentResponse(): void { + const held = this.heldEnrollments.at(-1); + if (!held) throw new Error("No enrollment is waiting"); + held.gate.reject(new Error("Enrollment response was lost")); + this.heldEnrollments.splice(this.heldEnrollments.indexOf(held), 1); + } + + rejectEnrollment(index: number, statusCode: 401 | 403): void { + const held = this.heldEnrollments[index]; + if (!held) throw new Error(`Enrollment ${index} is not waiting`); + held.gate.reject(new HubEnrollmentRejectedError(statusCode)); + this.heldEnrollments.splice(index, 1); + } + + async enrollmentAt(index: number): Promise { + while (!this.enrollments[index]) { + await this.enrollmentObserved.promise; + this.enrollmentObserved = deferred(); + } + return this.enrollments[index]; + } + + failRevocations(count: number): void { + this.revokeFailures = count; + } + + async revoke(input: HubRevocation): Promise { + this.revocations.push({ ...input }); + if (this.revokeFailures > 0) { + this.revokeFailures--; + throw new Error("Hub is offline"); + } + } + + openSocket(input: HubSocketCredentials, events: HubSocketEvents): HubSocketConnection { + this.socketSnapshots.push(this.captureRelationship()); + const attempt = { input: { ...input }, events, socket: new MemoryHubSocket() }; + this.sockets.push(attempt); + this.socketObserved.resolve(); + return attempt.socket; + } + + async socketAt(index: number): Promise { + while (!this.sockets[index]) { + await this.socketObserved.promise; + this.socketObserved = deferred(); + } + return this.sockets[index]; + } + + relationshipCount(): number { + return this.relationships.size; + } + + private enrollmentResult(input: HubEnrollment): HubEnrollmentResult { + return { + daemonId: input.daemonId, + scopes: this.enrollmentScopes.slice(), + webSocketUrl: SOCKET_URL, + }; + } +} + +interface CliProcess { + result: Promise>; +} + +type AcceptedCreate = Omit< + HubExecutionAgentCreateResponse["payload"], + "agentId" | "agent" | "success" +> & { + agentId: string; + agent: AgentSnapshotPayload; + success: true; +}; + +const providerCatalog = { + async resolveCreateConfig(input) { + return { modeId: input.requestedMode, featureValues: input.featureValues }; + }, +} satisfies CreateAgentCommandDependencies["providerSnapshotManager"]; + +export class HubRelationshipHarness { + private readonly clock = new TestRelationshipClock(); + private readonly remote = new InMemoryHubRelationships(() => this.captureRelationship()); + private daemon: PaseoDaemon | null = null; + private config!: PaseoDaemonConfig; + private root = ""; + private paseoHome = ""; + private host = ""; + private readonly logs: string[] = []; + private readonly providerPrompts: AgentPromptInput[] = []; + private readonly cliProcesses = new Set>(); + private readonly claimedCliSockets = new Set(); + private readonly promptsToFail = new Set(); + private failNextSessionClose = false; + private observedEnrollments = 0; + private observedSockets = 0; + private readonly codex = new ControlledAgentClient( + createTestAgentClients({ + closeSession: async () => { + if (!this.failNextSessionClose) return; + this.failNextSessionClose = false; + throw new Error("Requested provider session close failure"); + }, + onStartTurn: (prompt) => { + const promptText = typeof prompt === "string" ? prompt : JSON.stringify(prompt); + if (this.promptsToFail.delete(promptText)) { + throw new Error("Requested provider prompt startup failure"); + } + this.providerPrompts.push(prompt); + }, + }).codex, + ); + + private constructor(private readonly archiveWatchFiles: ArchiveWatchFiles) {} + + static async start( + archiveWatchFiles: ArchiveWatchFiles = nodeArchiveWatchFiles, + ): Promise { + const harness = new HubRelationshipHarness(archiveWatchFiles); + await harness.createHome(); + await harness.startDaemon(); + return harness; + } + + holdEnrollment(): void { + this.remote.holdEnrollment(); + } + + beginConnect(token = "ceremony-token", hubUrl = HUB_ORIGIN): CliProcess { + return { result: this.runCli(["hub", "connect", hubUrl, "--token", token]) }; + } + + async status(): Promise> { + return this.runCli(["hub", "status"]); + } + + async disconnect(force = false): Promise> { + return this.runCli(["hub", "disconnect", ...(force ? ["--force"] : [])]); + } + + beginDisconnect(force = false): CliProcess { + return { result: this.runCli(["hub", "disconnect", ...(force ? ["--force"] : [])]) }; + } + + async relationshipStateBecomes(expected: string | null): Promise { + const observed = deferred(); + const watcher = watch(this.paseoHome, () => { + if ((this.relationshipFile()?.state ?? null) === expected) observed.resolve(); + }); + if ((this.relationshipFile()?.state ?? null) === expected) observed.resolve(); + try { + await observed.promise; + } finally { + watcher.close(); + } + } + + async manageRelationshipFromExternalSocket(): Promise { + const address = Object.values(networkInterfaces()) + .flat() + .find((candidate) => candidate?.family === "IPv4" && !candidate.internal)?.address; + if (!address) + throw new Error("No non-loopback IPv4 address is available for the external test"); + const target = this.daemon?.getListenTarget(); + if (!target || target.type !== "tcp") throw new Error("Daemon did not bind TCP"); + const socket = await this.openClaimedCliSocket(`ws://${address}:${target.port}/ws`); + const messages = [ + { + type: "hub.management.daemon.connect.request", + requestId: "external-hub-connect", + hubUrl: HUB_ORIGIN, + token: "external-token", + }, + { type: "hub.management.daemon.get_status.request", requestId: "external-hub-status" }, + { + type: "hub.management.daemon.disconnect.request", + requestId: "external-hub-disconnect", + force: false, + }, + ]; + const responses: SessionOutboundMessage[] = []; + for (const message of messages) { + socket.send(JSON.stringify({ type: "session", message })); + responses.push((await this.nextSocketEnvelope(socket)).message); + } + await this.closeClaimedCliSocket(socket); + return responses; + } + + async connectFromBrowserSocket(): Promise { + const target = this.daemon?.getListenTarget(); + if (!target || target.type !== "tcp") throw new Error("Daemon did not bind TCP"); + const socket = await this.openClaimedCliSocket(`ws://127.0.0.1:${target.port}/ws`, { + origin: `http://127.0.0.1:${target.port}`, + }); + socket.send( + JSON.stringify({ + type: "session", + message: { + type: "hub.management.daemon.connect.request", + requestId: "browser-hub-connect", + hubUrl: HUB_ORIGIN, + token: "browser-token", + }, + }), + ); + const response = (await this.nextSocketEnvelope(socket)).message; + await this.closeClaimedCliSocket(socket); + return response; + } + + async enrollmentBegins(): Promise { + return this.remote.enrollmentAt(this.observedEnrollments++); + } + + completeEnrollment(): void { + this.remote.completeEnrollment(); + } + + returnEnrollmentScopes(scopes: string[]): void { + this.remote.returnEnrollmentScopes(scopes); + } + + loseEnrollmentResponse(): void { + this.remote.loseEnrollmentResponse(); + } + + rejectEnrollment(index: number, statusCode: 401 | 403): void { + this.remote.rejectEnrollment(index, statusCode); + } + + pendingRelationshipRetries(): number { + return this.clock.pendingTasks(); + } + + rejectNextEnrollment(statusCode: 401 | 403): void { + this.remote.rejectNextEnrollment(statusCode); + } + + failRevocations(count: number): void { + this.remote.failRevocations(count); + } + + failProviderPromptStart(prompt = "Create through the Hub"): void { + this.promptsToFail.add(prompt); + } + + failNextProviderSessionClose(): void { + this.failNextSessionClose = true; + } + + async socketDialed(): Promise { + await this.remote.socketAt(this.observedSockets++); + } + + connectLatestSocket(): void { + const socket = this.latestSocket(); + socket.events.connected(socket.socket); + } + + rejectLatestSocket(statusCode: 401 | 403): void { + this.latestSocket().events.rejected(statusCode); + } + + rejectRelationship(code: 4403 | 401 | 403): void { + if (code === 4403) { + this.closeLatestSocket(code); + return; + } + this.rejectLatestSocket(code); + } + + closeLatestSocket(code: number): void { + const socket = this.latestSocket(); + socket.events.closed(code); + socket.socket.close(code); + } + + connectSocket(index: number): void { + const socket = this.remote.sockets[index]; + if (!socket) throw new Error(`Socket ${index} does not exist`); + socket.events.connected(socket.socket); + } + + closeSocket(index: number, code: number): void { + const socket = this.remote.sockets[index]; + if (!socket) throw new Error(`Socket ${index} does not exist`); + socket.events.closed(code); + socket.socket.close(code); + } + + sendHubRequestOnLatest(message: unknown): SessionOutboundMessage[] { + const socket = this.latestSocket().socket; + socket.receive(message); + return socket.sent.slice(); + } + + holdAgentCreation(): void { + this.codex.holdCreation(); + } + + beginOwnedCreate( + requestId: string, + executionId = "execution-race", + options: { + worktree?: CreateAgentWorktreeTarget; + autoArchive?: boolean; + prompt?: string; + modeId?: string; + } = {}, + ): void { + const { prompt = "Create through the Hub", ...requestOptions } = options; + this.latestSocket().socket.receive({ + type: "hub.execution.agent.create.request", + requestId, + executionId, + provider: "codex", + cwd: this.root, + workspaceId: "hub-workspace", + prompt, + ...requestOptions, + }); + } + + async agentCreationAttempts(count: number): Promise { + await this.codex.creationAt(count); + } + + finishAgentCreation(): void { + this.codex.finishCreation(); + } + + async ownedCreateResult(requestId: string): Promise { + return this.latestSocket().socket.messageFor(requestId); + } + + async durableOwnedAgentIds(): Promise { + return (await this.daemon!.agentStorage.list()) + .filter((record) => record.owner?.kind === "daemon") + .map((record) => record.id); + } + + async durableOwnedAgentIdsOnDisk(): Promise { + const storage = new AgentStorage( + path.join(this.paseoHome, "agents"), + pino({ level: "silent" }), + ); + return (await storage.list()) + .filter((record) => record.owner?.kind === "daemon") + .map((record) => record.id); + } + + activeOwnedAgentIds(): string[] { + return this.daemon!.agentManager.listAgents() + .filter((agent) => agent.owner?.kind === "daemon") + .map((agent) => agent.id); + } + + agentSubscriptionCount(): number { + return this.daemon!.agentManager.subscriptionCount(); + } + + async hubExecutionIntentFiles(): Promise { + const directory = path.join(this.paseoHome, "hub-executions"); + return existsSync(directory) ? readdir(directory) : []; + } + + socketDeliveredResponse(socketIndex: number, requestId: string): boolean { + const socket = this.remote.sockets[socketIndex]; + if (!socket) throw new Error(`Socket ${socketIndex} does not exist`); + return socket.socket.sent.some( + (message) => + "payload" in message && + "requestId" in message.payload && + message.payload.requestId === requestId, + ); + } + + providerCreations(): number { + return this.codex.creations; + } + + providerResumes(): number { + return this.codex.resumes; + } + + providerPromptTexts(): string[] { + return this.providerPrompts.map((prompt) => + typeof prompt === "string" ? prompt : JSON.stringify(prompt), + ); + } + + latestCreatedCwd(): string | null { + return this.codex.createdConfigs.at(-1)?.cwd ?? null; + } + + repoRoot(): string { + return this.root; + } + + async waitForOwnedArchiveCompletion( + agentId: string, + ): Promise<{ agentArchivedAt: string; workspaceArchivedAt: string }> { + const created = await this.daemon!.agentStorage.get(agentId); + if (!created) throw new Error(`Owned agent ${agentId} does not exist`); + if (!created.workspaceId) throw new Error(`Owned agent ${agentId} has no workspace`); + return new Promise<{ agentArchivedAt: string; workspaceArchivedAt: string }>( + (resolve, reject) => { + const watchers: ArchiveWatcher[] = []; + let settled = false; + const timeout = setTimeout( + () => finish(new Error(`Timed out waiting for owned agent ${agentId} to archive`)), + 15_000, + ); + timeout.unref?.(); + const closeWatchers = () => { + clearTimeout(timeout); + for (const watcher of watchers) watcher.close(); + }; + const finish = ( + result: Error | { agentArchivedAt: string; workspaceArchivedAt: string }, + ) => { + if (settled) return; + settled = true; + closeWatchers(); + if (result instanceof Error) reject(result); + else resolve(result); + }; + const observeCompletion = async () => { + try { + const archived = await this.daemon!.agentStorage.get(agentId); + const workspaceArchivedAt = this.workspaceArchivedAt(created.workspaceId!); + if (!archived?.archivedAt || !workspaceArchivedAt || existsSync(created.cwd)) return; + finish({ agentArchivedAt: archived.archivedAt, workspaceArchivedAt }); + } catch (error) { + finish(error instanceof Error ? error : new Error(String(error))); + } + }; + try { + const projectsWatcher = this.archiveWatchFiles.watchDirectory( + path.join(this.paseoHome, "projects"), + observeCompletion, + ); + watchers.push(projectsWatcher); + projectsWatcher.onError(finish); + const worktreeWatcher = this.archiveWatchFiles.watchDirectory( + path.dirname(created.cwd), + observeCompletion, + ); + watchers.push(worktreeWatcher); + worktreeWatcher.onError(finish); + void observeCompletion(); + } catch (error) { + finish(error instanceof Error ? error : new Error(String(error))); + } + }, + ); + } + + private workspaceArchivedAt(workspaceId: string): string | null { + const records = JSON.parse( + readFileSync(path.join(this.paseoHome, "projects", "workspaces.json"), "utf8"), + ) as Array<{ workspaceId: string; archivedAt?: string | null }>; + return records.find((workspace) => workspace.workspaceId === workspaceId)?.archivedAt ?? null; + } + + async worktreeState(worktreePath: string): Promise<{ exists: boolean; listed: boolean }> { + const { stdout } = await execFileAsync("git", [ + "-C", + this.root, + "worktree", + "list", + "--porcelain", + ]); + const listedPaths = stdout + .split("\n") + .filter((line) => line.startsWith("worktree ")) + .map((line) => this.comparablePath(line.slice("worktree ".length))); + return { + exists: existsSync(worktreePath), + listed: listedPaths.includes(this.comparablePath(worktreePath)), + }; + } + + async createBranch(branch: string): Promise { + await execFileAsync("git", ["-C", this.root, "branch", branch]); + } + + async currentBranch(cwd: string): Promise { + const { stdout } = await execFileAsync("git", ["-C", cwd, "branch", "--show-current"]); + return stdout.trim(); + } + + async ownedPermissionRequest(agentId: string) { + const message = await this.latestSocket().socket.messageMatching( + (candidate): candidate is HubExecutionAgentStream => + candidate.type === "hub.execution.agent.stream" && + candidate.payload.agentId === agentId && + candidate.payload.event.type === "permission_requested", + ); + if (message.payload.event.type !== "permission_requested") { + throw new Error(`Owned agent ${agentId} did not request permission`); + } + return message.payload.event.request; + } + + async allowOwnedPermission(agentId: string, requestId: string): Promise { + await this.daemon!.agentManager.respondToPermission(agentId, requestId, { behavior: "allow" }); + } + + async listedWorktrees(): Promise { + const { stdout } = await execFileAsync("git", [ + "-C", + this.root, + "worktree", + "list", + "--porcelain", + ]); + return stdout + .split("\n") + .filter((line) => line.startsWith("worktree ")) + .map((line) => line.slice("worktree ".length)); + } + + async createOwnedConcurrently(executionId = "execution-1"): Promise<{ + first: AcceptedCreate; + duplicate: AcceptedCreate; + }> { + this.beginOwnedCreate("create-1", executionId); + this.beginOwnedCreate("create-2", executionId); + const first = await this.acceptedCreate("create-1"); + const duplicate = await this.acceptedCreate("create-2"); + return { first, duplicate }; + } + + async ownedUpdate(agentId: string): Promise { + const message = await this.latestSocket().socket.messageMatching( + (candidate): candidate is HubExecutionAgentUpdate => + candidate.type === "hub.execution.agent.update" && candidate.payload.agentId === agentId, + ); + return message.payload; + } + + async ownedRunningUpdate(agentId: string): Promise { + const message = await this.latestSocket().socket.messageMatching( + (candidate): candidate is HubExecutionAgentUpdate => + candidate.type === "hub.execution.agent.update" && + candidate.payload.agentId === agentId && + candidate.payload.agent.status === "running", + ); + return message.payload; + } + + async ownedTurnCompletion(agentId: string): Promise { + const socket = this.latestSocket().socket; + const completed = await socket.messageMatching( + (candidate): candidate is HubExecutionAgentStream => + candidate.type === "hub.execution.agent.stream" && + candidate.payload.agentId === agentId && + candidate.payload.event.type === "turn_completed", + ); + return completed.payload; + } + + async ownedTurnFailure(agentId: string): Promise { + const failed = await this.latestSocket().socket.messageMatching( + (candidate): candidate is HubExecutionAgentStream => + candidate.type === "hub.execution.agent.stream" && + candidate.payload.agentId === agentId && + candidate.payload.event.type === "turn_failed", + ); + return failed.payload; + } + + latestOwnedTurnCompletions(agentId: string): number { + return this.latestSocket().socket.sent.filter( + (message) => + message.type === "hub.execution.agent.stream" && + message.payload.agentId === agentId && + message.payload.event.type === "turn_completed", + ).length; + } + + async ownedStream(agentId: string): Promise { + const message = await this.latestSocket().socket.messageMatching( + (candidate): candidate is HubExecutionAgentStream => + candidate.type === "hub.execution.agent.stream" && candidate.payload.agentId === agentId, + ); + return message.payload; + } + + async createUnrelatedLocalAgent(): Promise { + const agent = await this.daemon!.agentManager.createAgent( + { provider: "codex", cwd: this.root }, + undefined, + { workspaceId: "local-workspace" }, + ); + return agent.id; + } + + async deniedSteering(agentId: string): Promise { + const requestId = "denied-steer"; + this.latestSocket().socket.receive({ + type: "send_agent_message_request", + requestId, + agentId, + text: "This must not run", + }); + return ((await this.latestSocket().socket.messageFor(requestId)) as RpcErrorMessage).payload; + } + + async deniedBrowserDispatch(): Promise { + const requestId = "browser-1"; + this.latestSocket().socket.receive({ + type: "browser.automation.execute.response", + payload: { + requestId, + ok: false, + error: { code: "browser_denied", message: "denied", retryable: false }, + }, + }); + return ((await this.latestSocket().socket.messageFor(requestId)) as RpcErrorMessage).payload; + } + + observedAgentIds(): string[] { + return this.remote.sockets.flatMap(({ socket }) => + socket.sent.flatMap((message) => { + if (message.type === "hub.execution.agent.update") return [message.payload.agentId]; + if (message.type === "hub.execution.agent.stream") return [message.payload.agentId]; + return []; + }), + ); + } + + observedTrustedLifecycleMessages(): string[] { + return this.remote.sockets.flatMap(({ socket }) => + socket.sent + .filter((message) => message.type === "status") + .map((message) => message.payload.status), + ); + } + + probeTrustedHello(): number | null { + const socket = this.latestSocket().socket; + socket.receiveEnvelope({ + type: "hello", + clientId: "hub-must-not-resume", + clientType: "browser", + protocolVersion: 1, + capabilities: { voice: true, pushNotifications: true }, + }); + return socket.closeCode; + } + + probeBinaryFrame(): number | null { + const socket = this.latestSocket().socket; + socket.receiveBinary(); + return socket.closeCode; + } + + async trustedBroadcastCount(): Promise { + const before = this.daemonConfigBroadcasts(); + const client = await this.trustedClient(); + try { + await client.patchDaemonConfig({ appendSystemPrompt: "hub-broadcast-isolation" }); + this.beginOwnedCreate("broadcast-barrier", "broadcast-barrier"); + await this.acceptedCreate("broadcast-barrier"); + return this.daemonConfigBroadcasts() - before; + } finally { + await client.close(); + } + } + + async trustedDaemonStatus() { + const client = await this.trustedClient(); + try { + return await client.getDaemonStatus(); + } finally { + await client.close(); + } + } + + async reconnectAndRetry(executionId = "execution-1") { + this.closeLatestSocket(1006); + await this.retry(); + this.connectLatestSocket(); + this.beginOwnedCreate("reconnect-retry", executionId); + return this.acceptedCreate("reconnect-retry"); + } + + async reconstructAndReplay(executionId = "execution-1") { + const storage = new AgentStorage( + path.join(this.paseoHome, "agents"), + pino({ level: "silent" }), + ); + const manager = new AgentManager({ + clients: createTestAgentClients(), + registry: storage, + logger: pino({ level: "silent" }), + }); + const executions = this.executionsForReconstruction(manager, storage); + const replay = await executions.create(this.ownedCreateInput(executionId)); + const durableAgentCount = (await storage.list()).filter( + (record) => record.owner?.kind === "daemon", + ).length; + return { replay, durableAgentCount }; + } + + async removeOwnedAgent(agentId: string) { + await this.daemon!.agentStorage.remove(agentId); + const storage = new AgentStorage( + path.join(this.paseoHome, "agents"), + pino({ level: "silent" }), + ); + return { + durableAgentCount: (await storage.list()).filter((record) => record.owner?.kind === "daemon") + .length, + }; + } + + socketAttempts(): number { + return this.remote.sockets.length; + } + + revocationAttempts(): number { + return this.remote.revocations.length; + } + + latestRevocation(): HubRevocation | null { + return this.remote.revocations.at(-1) ?? null; + } + + enrollmentAttempts(): HubEnrollment[] { + return this.remote.enrollments.map((input) => ({ ...input, scopes: input.scopes.slice() })); + } + + enrollmentInvocation(index = 0): RelationshipInvocationSnapshot { + const snapshot = this.remote.enrollmentSnapshots[index]; + if (!snapshot) throw new Error(`Enrollment invocation ${index} does not exist`); + return snapshot; + } + + socketInvocation(index = 0): RelationshipInvocationSnapshot { + const snapshot = this.remote.socketSnapshots[index]; + if (!snapshot) throw new Error(`Socket invocation ${index} does not exist`); + return snapshot; + } + + enrolledRelationships(): number { + return this.remote.relationshipCount(); + } + + loggableValues(status: Record): string { + return JSON.stringify({ status, logs: this.logs }); + } + + relationshipFile(): PersistedRelationship | null { + const file = path.join(this.paseoHome, "hub-relationship.json"); + if (!existsSync(file)) return null; + return JSON.parse(readFileSync(file, "utf8")) as PersistedRelationship; + } + + relationshipFileMode(): number { + return statSync(path.join(this.paseoHome, "hub-relationship.json")).mode & 0o777; + } + + async corruptRelationshipFile(contents = "{not-json"): Promise { + await this.stopDaemon(); + await writeFile(path.join(this.paseoHome, "hub-relationship.json"), contents, "utf8"); + } + + async quarantinedRelationshipFiles(): Promise { + return (await readdir(this.paseoHome)).filter((file) => + file.startsWith("hub-relationship.invalid-"), + ); + } + + async startStoppedDaemon(): Promise { + await this.startDaemon(); + } + + async storedOwnedStatus(agentId: string): Promise { + return (await this.daemon!.agentStorage.get(agentId))?.lastStatus ?? null; + } + + private captureRelationship(): RelationshipInvocationSnapshot { + const record = this.relationshipFile(); + if (!record) throw new Error("Relationship authority was not persisted before invocation"); + return { mode: this.relationshipFileMode(), record }; + } + + async retry(): Promise { + await this.clock.runNext(); + } + + async restartDaemon(): Promise { + await this.stopDaemon(); + await this.startDaemon(); + } + + async shutdownDaemon(): Promise { + await this.stopDaemon(); + } + + async close(): Promise { + await this.stopDaemon(); + await Promise.allSettled(this.cliProcesses); + await Promise.all( + [...this.claimedCliSockets].map((socket) => this.closeClaimedCliSocket(socket)), + ); + await this.removeRoot(); + } + + private async createHome(): Promise { + this.root = await mkdtemp(path.join(tmpdir(), "paseo-hub-relationship-")); + this.paseoHome = path.join(this.root, ".paseo"); + const staticDir = path.join(this.root, "static"); + await Promise.all([mkdir(this.paseoHome, { recursive: true }), mkdir(staticDir)]); + execFileSync("git", ["init", "-b", "main", this.root], { stdio: "ignore" }); + execFileSync("git", ["-C", this.root, "config", "user.email", "hub@test.invalid"]); + execFileSync("git", ["-C", this.root, "config", "user.name", "Hub Test"]); + execFileSync("git", ["-C", this.root, "commit", "--allow-empty", "-m", "initial"], { + stdio: "ignore", + }); + this.config = { + listen: "0.0.0.0:0", + paseoHome: this.paseoHome, + corsAllowedOrigins: [], + hostnames: true, + mcpEnabled: false, + staticDir, + mcpDebug: false, + agentClients: { + ...createTestAgentClients(), + codex: this.codex, + }, + agentStoragePath: path.join(this.paseoHome, "agents"), + relayEnabled: false, + relayEndpoint: "relay.paseo.sh:443", + appBaseUrl: "https://app.paseo.sh", + }; + } + + private async startDaemon(): Promise { + const destination = new Writable({ + write: (chunk, _encoding, done) => { + this.logs.push(String(chunk)); + done(); + }, + }); + this.daemon = await createPaseoDaemon(this.config, pino({ level: "trace" }, destination), { + hubRelationshipRemote: this.remote, + hubRelationshipClock: this.clock, + hubRelationshipRetryPolicy: this.clock, + createHubDaemonId: () => "daemon-test", + }); + await this.daemon.start(); + const target = this.daemon.getListenTarget(); + if (!target || target.type !== "tcp") throw new Error("Daemon did not bind TCP"); + this.host = `127.0.0.1:${target.port}`; + } + + private async stopDaemon(): Promise { + await this.daemon?.stop(); + this.daemon = null; + } + + private runCli(args: string[]): Promise> { + return this.trackCli(this.executeCli(args)); + } + + private async executeCli(args: string[]): Promise> { + const entrypoint = path.join(import.meta.dirname, "../../test-utils/hub-cli-entry.ts"); + const { stdout } = await execFileAsync( + process.execPath, + [ + "--conditions=source", + "--import", + "tsx", + entrypoint, + ...args, + "--host", + this.host, + "--json", + ], + { cwd: REPOSITORY_ROOT, env: { ...process.env, NO_COLOR: "1" } }, + ); + const parsed = JSON.parse(stdout) as unknown; + if (Array.isArray(parsed)) return parsed[0] as Record; + return parsed as Record; + } + + private trackCli(process: Promise): Promise { + this.cliProcesses.add(process); + void process.then( + () => this.cliProcesses.delete(process), + () => this.cliProcesses.delete(process), + ); + return process; + } + + private nextSocketEnvelope(socket: WebSocket): Promise<{ message: SessionOutboundMessage }> { + return new Promise((resolve, reject) => { + socket.once("message", (data) => { + try { + resolve(JSON.parse(data.toString()) as { message: SessionOutboundMessage }); + } catch (error) { + reject(error); + } + }); + socket.once("error", reject); + }); + } + + private async openClaimedCliSocket( + url: string, + options: { origin?: string } = {}, + ): Promise { + const socket = new WebSocket(url, options); + this.claimedCliSockets.add(socket); + await new Promise((resolve, reject) => { + socket.once("open", resolve); + socket.once("error", reject); + }); + socket.send( + JSON.stringify({ + type: "hello", + clientId: "external-claimed-cli", + clientType: "cli", + protocolVersion: 1, + }), + ); + await this.nextSocketEnvelope(socket); + return socket; + } + + private async closeClaimedCliSocket(socket: WebSocket): Promise { + this.claimedCliSockets.delete(socket); + if (socket.readyState === WebSocket.CLOSED) return; + const closed = new Promise((resolve) => socket.once("close", () => resolve())); + socket.terminate(); + await closed; + } + + private comparablePath(value: string): string { + const resolved = path.resolve(value); + let canonical = resolved; + try { + canonical = realpathSync.native(resolved); + } catch { + // An archived worktree no longer exists, so compare its normalized target path. + } + const normalized = path.normalize(canonical); + return platform() === "win32" ? normalized.toLowerCase() : normalized; + } + + private async removeRoot(): Promise { + let retryableCode: string | null = null; + if (platform() === "win32") retryableCode = "EBUSY"; + if (platform() === "darwin") retryableCode = "ENOTEMPTY"; + const attempts = retryableCode ? 10 : 1; + for (let attempt = 1; attempt <= attempts; attempt++) { + try { + await rm(this.root, { recursive: true, force: true }); + return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== retryableCode || attempt === attempts) { + throw error; + } + await new Promise((resolve) => setTimeout(resolve, 100 * attempt)); + } + } + } + + private latestSocket(): SocketAttempt { + const socket = this.remote.sockets.at(-1); + if (!socket) throw new Error("No Hub socket has been opened"); + return socket; + } + + private async acceptedCreate(requestId: string): Promise { + const message = (await this.latestSocket().socket.messageFor( + requestId, + )) as HubExecutionAgentCreateResponse; + if (!message.payload.success || !message.payload.agentId || !message.payload.agent) { + throw new Error(message.payload.error ?? "Hub agent creation failed"); + } + return { + ...message.payload, + success: true, + agentId: message.payload.agentId, + agent: message.payload.agent, + }; + } + + private ownedCreateInput(executionId: string) { + return { + executionId, + provider: "codex", + cwd: this.root, + workspaceId: "hub-workspace", + prompt: "Create through the Hub", + }; + } + + private executionsForReconstruction(manager: AgentManager, storage: AgentStorage) { + return new DaemonExecutions({ + daemonId: this.relationshipFile()!.relationship.daemonId, + agentManager: manager, + agentStorage: storage, + createAgent: (input) => + createAgentCommand( + { + agentManager: manager, + agentStorage: storage, + logger: pino({ level: "silent" }), + providerSnapshotManager: providerCatalog, + }, + input, + ), + }); + } + + private async trustedClient(): Promise { + const client = new DaemonClient({ + url: `ws://${this.host}/ws`, + appVersion: "0.1.106", + }); + await client.connect(); + await client.fetchAgents({ subscribe: { subscriptionId: "hub-relationship-trusted" } }); + return client; + } + + private daemonConfigBroadcasts(): number { + return this.remote.sockets + .flatMap(({ socket }) => socket.sent) + .filter( + (message) => + message.type === "status" && message.payload.status === "daemon_config_changed", + ).length; + } +} diff --git a/packages/server/src/server/persistence-hooks.ts b/packages/server/src/server/persistence-hooks.ts index 5fc511372..ba220eefa 100644 --- a/packages/server/src/server/persistence-hooks.ts +++ b/packages/server/src/server/persistence-hooks.ts @@ -110,6 +110,7 @@ export function extractTimestamps(record: StoredAgentRecord): { lastUserMessageAt: Date | null; labels?: Record; workspaceId?: string; + owner?: StoredAgentRecord["owner"]; } { return { createdAt: new Date(record.createdAt), @@ -117,6 +118,7 @@ export function extractTimestamps(record: StoredAgentRecord): { lastUserMessageAt: record.lastUserMessageAt ? new Date(record.lastUserMessageAt) : null, labels: record.labels, workspaceId: record.workspaceId, + owner: record.owner, }; } diff --git a/packages/server/src/server/session.test.ts b/packages/server/src/server/session.test.ts index 683d2821e..4db2bc7a6 100644 --- a/packages/server/src/server/session.test.ts +++ b/packages/server/src/server/session.test.ts @@ -18,7 +18,7 @@ import { FileTransferOpcode, type FileTransferFrame, } from "@getpaseo/protocol/binary-frames/index"; -import { Session } from "./session.js"; +import { isSessionRpcAllowed, Session } from "./session.js"; import { DownloadTokenStore } from "./file-download/token-store.js"; import { StructuredAgentFallbackError } from "./agent/agent-response-loop.js"; import type { StoredAgentRecord } from "./agent/agent-storage.js"; @@ -278,6 +278,7 @@ vi.mock("./worktree-bootstrap.js", async (importOriginal) => { }); interface SessionForTestOptions { + scopes?: readonly string[]; agentManager?: { [K in keyof SessionOptions["agentManager"]]?: unknown }; agentStorage?: { [K in keyof SessionOptions["agentStorage"]]?: unknown }; github?: Partial; @@ -349,7 +350,7 @@ function createSessionForTest(options: SessionForTestOptions = {}): Session { }; const messages = options.messages ?? []; - return new Session({ + const sessionOptions: SessionOptions = { clientId: "test-client", onMessage: (message) => messages.push(message), ...(options.targetedMessages @@ -413,9 +414,85 @@ function createSessionForTest(options: SessionForTestOptions = {}): Session { serverId: options.serverId, daemonVersion: options.daemonVersion, daemonRuntimeConfig: options.daemonRuntimeConfig, - }); + scopes: options.scopes ?? ["*"], + }; + return new Session(sessionOptions); } +describe("session authorization scopes", () => { + test("rejects an RPC outside an exact grant with the generic RPC error", async () => { + const messages: SessionOutboundMessage[] = []; + const session = createSessionForTest({ + scopes: ["hub.execution.agent.create.request"], + messages, + }); + + await session.handleMessage({ type: "ping", requestId: "restricted-ping", clientSentAt: 42 }); + + expect(messages).toEqual([ + { + type: "rpc_error", + payload: { + requestId: "restricted-ping", + requestType: "ping", + error: "Session is not authorized for ping", + code: "access_denied", + }, + }, + ]); + }); + + test.each([ + ["*", "ping"], + ["hub.execution.*", "hub.execution.agent.create.request"], + ["hub.execution.agent.create.request", "hub.execution.agent.create.request"], + ])("scope %s authorizes %s", (scope, requestType) => { + expect(isSessionRpcAllowed([scope], requestType)).toBe(true); + }); + + test.each([ + ["hub.execution.*", "hub.management.daemon.get_status.request"], + ["hub.execution.agent.create.request", "hub.execution.agent.update"], + ["hub.execution.*", "hub.executions.agent.create.request"], + ])("scope %s rejects %s", (scope, requestType) => { + expect(isSessionRpcAllowed([scope], requestType)).toBe(false); + }); + + test("replaces a session's scopes without reconstructing the session", async () => { + const messages: SessionOutboundMessage[] = []; + const session = createSessionForTest({ scopes: ["hub.execution.*"], messages }); + + await session.handleMessage({ + type: "ping", + requestId: "before-scope-change", + clientSentAt: 1, + }); + session.setScopes(["*"]); + await session.handleMessage({ type: "ping", requestId: "after-scope-change", clientSentAt: 2 }); + + expect(messages).toEqual([ + { + type: "rpc_error", + payload: { + requestId: "before-scope-change", + requestType: "ping", + error: "Session is not authorized for ping", + code: "access_denied", + }, + }, + { + type: "pong", + payload: { + requestId: "after-scope-change", + clientSentAt: 2, + serverReceivedAt: expect.any(Number), + serverSentAt: expect.any(Number), + }, + }, + ]); + }); +}); + describe("project command-center RPCs", () => { test("returns normalized repositories from the host GitHub service", async () => { const messages: SessionOutboundMessage[] = []; diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 70fbb9b13..8476973c4 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -152,6 +152,9 @@ import { AgentConfigSession } from "./session/agent-config/agent-config-session. import { ProjectConfigSession } from "./session/project-config/project-config-session.js"; import { DaemonSession, type DaemonRuntimeConfig } from "./session/daemon/daemon-session.js"; import type { DaemonWebSocketRuntimeDiagnosticSnapshot } from "./session/daemon/diagnostics.js"; +import type { HubRelationshipManagement } from "./hub/relationship-controller.js"; +import { HubExecutionController } from "./hub/execution-controller.js"; +import type { HubExecutionAgents } from "./hub/daemon-executions.js"; import { DownloadTokenStore } from "./file-download/token-store.js"; import { PushTokenStore } from "./push/token-store.js"; import { @@ -381,6 +384,7 @@ type AgentMcpTransportFactory = () => Promise; export interface SessionOptions { clientId: string; + scopes: readonly string[]; appVersion?: string | null; clientCapabilities?: Record | null; onMessage: (msg: SessionOutboundMessage) => void; @@ -418,6 +422,8 @@ export interface SessionOptions { terminalManager: TerminalManager | null; providerSnapshotManager: ProviderSnapshotManager; providerUsageService: ProviderUsageService; + hubExecutionAgents?: HubExecutionAgents; + hubRelationships?: HubRelationshipManagement; serviceProxy?: ServiceProxySubsystem; scriptRuntimeStore?: WorkspaceScriptRuntimeStore; workspaceSetupSnapshots?: Map; @@ -481,6 +487,34 @@ function parseClientCapabilities( return new Set(result); } +export function isSessionRpcAllowed(scopes: readonly string[], rpcName: string): boolean { + return scopes.some((scope) => { + if (scope === "*" || scope === rpcName) { + return true; + } + if (!scope.endsWith(".*")) { + return false; + } + return rpcName.startsWith(scope.slice(0, -1)); + }); +} + +function sessionRequestId(message: SessionInboundMessage): string | null { + if ("requestId" in message && typeof message.requestId === "string") { + return message.requestId; + } + if ( + "payload" in message && + typeof message.payload === "object" && + message.payload !== null && + "requestId" in message.payload && + typeof message.payload.requestId === "string" + ) { + return message.payload.requestId; + } + return null; +} + interface AgentTimelineProjectionSelection { timeline: AgentTimelineFetchResult; entries: TimelineProjectionEntry[]; @@ -510,6 +544,7 @@ function describeRegistryTransition(record: ArchivedRecordSnapshot | null): Regi */ export class Session { private readonly clientId: string; + private scopes: readonly string[]; private appVersion: string | null; private clientCapabilities: ReadonlySet; private readonly sessionId: string; @@ -579,12 +614,14 @@ export class Session { private readonly agentConfigSession: AgentConfigSession; private readonly projectConfigSession: ProjectConfigSession; private readonly daemonSession: DaemonSession; + private readonly hubExecutionController: HubExecutionController | null; private readonly workspaceScripts: WorkspaceScriptsService; private readonly createAgentLifecycleDispatch: CreateAgentLifecycleDispatch; constructor(options: SessionOptions) { const { clientId, + scopes, appVersion, clientCapabilities, onMessage, @@ -635,6 +672,7 @@ export class Session { getWebSocketRuntimeMetrics, } = options; this.clientId = clientId; + this.scopes = [...scopes]; this.appVersion = appVersion ?? null; this.clientCapabilities = parseClientCapabilities(clientCapabilities); this.sessionId = uuidv4(); @@ -800,7 +838,14 @@ export class Session { listProjects: () => this.projectRegistry.list(), listWorkspaces: () => this.workspaceRegistry.list(), logger: this.sessionLogger, + hubRelationships: options.hubRelationships, }); + this.hubExecutionController = options.hubExecutionAgents + ? new HubExecutionController({ + agents: options.hubExecutionAgents, + send: (message) => this.emit(message), + }) + : null; this.daemonConfigStore = daemonConfigStore; this.terminalManager = terminalManager; this.terminalController = new TerminalSessionController({ @@ -1506,6 +1551,21 @@ export class Session { }, "agent.session.inbound", ); + if (!isSessionRpcAllowed(this.scopes, msg.type)) { + const requestId = sessionRequestId(msg); + if (requestId) { + this.emit({ + type: "rpc_error", + payload: { + requestId, + requestType: msg.type, + error: `Session is not authorized for ${msg.type}`, + code: "access_denied", + }, + }); + } + return; + } try { await this.dispatchInboundMessage(msg, source); } catch (error) { @@ -1545,12 +1605,17 @@ export class Session { } } + public setScopes(scopes: readonly string[]): void { + this.scopes = [...scopes]; + } + private async dispatchInboundMessage(msg: SessionInboundMessage, source?: object): Promise { const promise = this.dispatchVoiceAndControlMessage(msg) ?? this.dispatchAgentRewindMessage(msg) ?? this.dispatchAgentRelationshipMessage(msg) ?? this.dispatchAgentTimelineMessage(msg, source) ?? + this.dispatchHubExecutionMessage(msg) ?? this.dispatchAgentLifecycleMessage(msg) ?? this.dispatchAgentConfigMessage(msg) ?? this.dispatchCheckoutMessage(msg) ?? @@ -1667,6 +1732,12 @@ export class Session { } } + private dispatchHubExecutionMessage(msg: SessionInboundMessage): Promise | undefined { + return msg.type === "hub.execution.agent.create.request" + ? this.hubExecutionController?.createAgent(msg) + : undefined; + } + private dispatchAgentLifecycleMessage(msg: SessionInboundMessage): Promise | undefined { switch (msg.type) { case "fetch_agents_request": @@ -1730,6 +1801,10 @@ export class Session { return this.daemonSession.handleGetStatusRequest(msg); case "daemon.get_pairing_offer.request": return this.daemonSession.handleGetPairingOfferRequest(msg); + case "hub.management.daemon.connect.request": + case "hub.management.daemon.get_status.request": + case "hub.management.daemon.disconnect.request": + return this.daemonSession.handleHubRelationshipRequest(msg); case "diagnostics.request": return this.daemonSession.handleDiagnosticsRequest(msg); case "daemon.update.request": @@ -2731,6 +2806,10 @@ export class Session { let createdWorktreeForCleanup: CreatePaseoWorktreeWorkflowResult | null = null; let createdAgentId: string | null = null; try { + const requestedCwd = resolve(config.cwd); + if (!(await this.filesystem.isDirectory(requestedCwd))) { + throw new Error(`Working directory does not exist or is not a directory: ${requestedCwd}`); + } const trimmedPrompt = initialPrompt?.trim(); const { provisionalTitle } = resolveCreateAgentTitles({ configTitle: config.title, @@ -6108,6 +6187,9 @@ export class Session { * Emit a message to the client */ private emit(msg: SessionOutboundMessage): void { + if (msg.type !== "rpc_error" && !isSessionRpcAllowed(this.scopes, msg.type)) { + return; + } // JSON.stringify(msg) is only computed when trace is enabled — it runs for // every outbound message otherwise, and trace is disabled by default. // Optional-chained because test logger stubs don't implement isLevelEnabled. @@ -6145,6 +6227,7 @@ export class Session { this.unsubscribeAgentEvents = null; } this.agentUpdates.dispose(); + await this.hubExecutionController?.cleanup(); if (this.unsubscribeTerminalWorkspaceContributionEvents) { this.unsubscribeTerminalWorkspaceContributionEvents(); this.unsubscribeTerminalWorkspaceContributionEvents = null; diff --git a/packages/server/src/server/session.workspace-git-watch.test.ts b/packages/server/src/server/session.workspace-git-watch.test.ts index 64e4baa3c..06daa4726 100644 --- a/packages/server/src/server/session.workspace-git-watch.test.ts +++ b/packages/server/src/server/session.workspace-git-watch.test.ts @@ -182,6 +182,7 @@ function createSessionForWorkspaceGitWatchTests(options?: { const session = new Session({ clientId: "test-client", + scopes: ["*"], onMessage: (message) => emitted.push(message as { type: string; payload: unknown }), logger: createStub(logger), downloadTokenStore: createStub({}), diff --git a/packages/server/src/server/session.workspace-resolution-invariants.test.ts b/packages/server/src/server/session.workspace-resolution-invariants.test.ts index 42dcd593b..277320e54 100644 --- a/packages/server/src/server/session.workspace-resolution-invariants.test.ts +++ b/packages/server/src/server/session.workspace-resolution-invariants.test.ts @@ -88,6 +88,7 @@ function createHarness(input: { const session = new Session({ clientId: "test", + scopes: ["*"], appVersion: null, onMessage: (m) => emitted.push(m), logger: createStub(logger), diff --git a/packages/server/src/server/session.workspaces.test.ts b/packages/server/src/server/session.workspaces.test.ts index 05ba4d591..cd2b94e17 100644 --- a/packages/server/src/server/session.workspaces.test.ts +++ b/packages/server/src/server/session.workspaces.test.ts @@ -615,6 +615,7 @@ function createSessionForWorkspaceTests( const session = asTestSession( new Session({ clientId: "test-client", + scopes: ["*"], appVersion: options.appVersion ?? null, onMessage: options.onMessage ?? vi.fn(), onWorkspaceRecovered: options.onWorkspaceRecovered, @@ -815,6 +816,7 @@ test("create_agent_request keeps requested child cwd when grouped under an exist const session = asTestSession( new Session({ clientId: "test-client", + scopes: ["*"], appVersion: null, onMessage: (message) => emitted.push(message), logger: asSessionLogger(logger), @@ -957,6 +959,7 @@ test("create_agent_request launches from an exact subdirectory in a created work const emitted: SessionOutboundMessage[] = []; const session = new Session({ clientId: "test-client", + scopes: ["*"], appVersion: null, onMessage: (message) => emitted.push(message), logger: asSessionLogger(logger), @@ -1095,6 +1098,7 @@ test("create_agent_request does not title an existing workspace from the agent p const session = asTestSession( new Session({ clientId: "test-client", + scopes: ["*"], appVersion: null, onMessage: vi.fn(), logger: asSessionLogger(logger), @@ -1418,6 +1422,7 @@ test("archive emits an authoritative agent_update upsert for subscribed clients" const session = asTestSession( new Session({ clientId: "test-client", + scopes: ["*"], onMessage: (message) => emitted.push(message), logger: asSessionLogger(logger), downloadTokenStore: asDownloadTokenStore(), @@ -1784,6 +1789,7 @@ test("close_items_request archives agents and kills terminals in one batch", asy const session = asTestSession( new Session({ clientId: "test-client", + scopes: ["*"], onMessage: (message) => emitted.push(message), logger: asSessionLogger(sessionLogger), downloadTokenStore: asDownloadTokenStore(), @@ -1953,6 +1959,7 @@ test("close_items_request archives stored agents that are not currently loaded", const session = asTestSession( new Session({ clientId: "test-client", + scopes: ["*"], onMessage: (message) => emitted.push(message), logger: asSessionLogger(sessionLogger), downloadTokenStore: asDownloadTokenStore(), @@ -2113,6 +2120,7 @@ test("close_items_request continues after an archive failure", async () => { const session = asTestSession( new Session({ clientId: "test-client", + scopes: ["*"], onMessage: (message) => emitted.push(message), logger: asSessionLogger(sessionLogger), downloadTokenStore: asDownloadTokenStore(), @@ -3212,6 +3220,7 @@ test("workspace update stream keeps persisted workspace visible after agents sto const session = asTestSession( new Session({ clientId: "test-client", + scopes: ["*"], onMessage: (message) => emitted.push(message), logger: asSessionLogger(logger), downloadTokenStore: asDownloadTokenStore(), diff --git a/packages/server/src/server/session/daemon/daemon-session.test.ts b/packages/server/src/server/session/daemon/daemon-session.test.ts index 9417a17ed..661439bad 100644 --- a/packages/server/src/server/session/daemon/daemon-session.test.ts +++ b/packages/server/src/server/session/daemon/daemon-session.test.ts @@ -10,6 +10,7 @@ import { } from "./daemon-session.js"; import type { DaemonWebSocketRuntimeDiagnosticSnapshot } from "./diagnostics.js"; import type { ProviderAvailability } from "../../agent/agent-manager.js"; +import type { HubRelationshipManagement } from "../../hub/relationship-controller.js"; import type { SessionOutboundMessage } from "../../messages.js"; const tempDirs: string[] = []; @@ -40,6 +41,7 @@ function makeSubsystem(overrides: { daemonRuntimeConfig?: DaemonRuntimeConfig; listProviderAvailability?: () => Promise; getWebSocketRuntimeMetrics?: () => DaemonWebSocketRuntimeDiagnosticSnapshot | null; + hubRelationships?: HubRelationshipManagement; }) { const emitted: SessionOutboundMessage[] = []; const restartIntents: Parameters[0][] = []; @@ -60,12 +62,67 @@ function makeSubsystem(overrides: { listWorkspaces: async () => [], listProviderAvailability: overrides.listProviderAvailability ?? (async () => []), getWebSocketRuntimeMetrics: overrides.getWebSocketRuntimeMetrics, + hubRelationships: overrides.hubRelationships, logger: pino({ level: "silent" }), }); return { subsystem, emitted, paseoHome, restartIntents }; } describe("DaemonSession", () => { + test("Hub relationship command failures return correlated RPC errors", async () => { + const { subsystem, emitted } = makeSubsystem({ + hubRelationships: { + connect: async () => { + throw new Error("Hub rejected enrollment (401)"); + }, + status: () => ({ + state: "not_connected", + daemonId: null, + hubOrigin: null, + scopes: [], + connectedAt: null, + lastError: null, + }), + disconnect: async () => { + throw new Error("Hub revocation failed (503)"); + }, + }, + }); + + await subsystem.handleHubRelationshipRequest({ + type: "hub.management.daemon.connect.request", + requestId: "connect-1", + hubUrl: "https://hub.test", + token: "token", + }); + await subsystem.handleHubRelationshipRequest({ + type: "hub.management.daemon.disconnect.request", + requestId: "disconnect-1", + force: false, + }); + + expect(emitted).toEqual([ + { + type: "rpc_error", + payload: { + requestId: "connect-1", + requestType: "hub.management.daemon.connect.request", + error: "Hub rejected enrollment (401)", + code: "handler_error", + }, + }, + { + type: "rpc_error", + payload: { + requestId: "disconnect-1", + requestType: "hub.management.daemon.disconnect.request", + error: "Hub revocation failed (503)", + code: "handler_error", + }, + }, + ]); + }); + test("status reports identity, runtime config, and providers with errors normalized to null", async () => { const { subsystem, emitted } = makeSubsystem({ serverId: "srv-1", diff --git a/packages/server/src/server/session/daemon/daemon-session.ts b/packages/server/src/server/session/daemon/daemon-session.ts index beb525118..7142ce8f8 100644 --- a/packages/server/src/server/session/daemon/daemon-session.ts +++ b/packages/server/src/server/session/daemon/daemon-session.ts @@ -10,6 +10,7 @@ import { import { DaemonSelfUpdateSessionController } from "./daemon-self-update-session-controller.js"; import type { ManagedAgent } from "../../agent/agent-manager.js"; import type { PersistedProjectRecord, PersistedWorkspaceRecord } from "../../workspace-registry.js"; +import type { HubRelationshipManagement } from "../../hub/relationship-controller.js"; export interface DaemonRuntimeConfig { listen: string | null; @@ -48,6 +49,7 @@ export interface DaemonSessionOptions { listProviderAvailability: () => Promise; getWebSocketRuntimeMetrics?: () => DaemonWebSocketRuntimeDiagnosticSnapshot | null; logger: pino.Logger; + hubRelationships?: HubRelationshipManagement; } /** @@ -71,6 +73,7 @@ export class DaemonSession { private readonly getWebSocketRuntimeMetrics: () => DaemonWebSocketRuntimeDiagnosticSnapshot | null; private readonly logger: pino.Logger; private readonly selfUpdate: DaemonSelfUpdateSessionController; + private readonly hubRelationships: HubRelationshipManagement | null; constructor(options: DaemonSessionOptions) { this.host = options.host; @@ -85,6 +88,7 @@ export class DaemonSession { this.listProviderAvailability = options.listProviderAvailability; this.getWebSocketRuntimeMetrics = options.getWebSocketRuntimeMetrics ?? (() => null); this.logger = options.logger; + this.hubRelationships = options.hubRelationships ?? null; this.selfUpdate = new DaemonSelfUpdateSessionController({ clientId: this.clientId, daemonVersion: this.daemonVersion ?? null, @@ -95,6 +99,56 @@ export class DaemonSession { }); } + async handleHubRelationshipRequest( + msg: Extract< + SessionInboundMessage, + { + type: + | "hub.management.daemon.connect.request" + | "hub.management.daemon.get_status.request" + | "hub.management.daemon.disconnect.request"; + } + >, + ): Promise { + try { + if (!this.hubRelationships) throw new Error("Hub relationship management is unavailable"); + if (msg.type === "hub.management.daemon.connect.request") { + const status = await this.hubRelationships.connect({ + hubUrl: msg.hubUrl, + token: msg.token, + }); + this.host.emit({ + type: "hub.management.daemon.connect.response", + payload: { requestId: msg.requestId, status }, + }); + return; + } + if (msg.type === "hub.management.daemon.disconnect.request") { + const result = await this.hubRelationships.disconnect({ force: msg.force ?? false }); + this.host.emit({ + type: "hub.management.daemon.disconnect.response", + payload: { requestId: msg.requestId, ...result }, + }); + return; + } + this.host.emit({ + type: "hub.management.daemon.get_status.response", + payload: { requestId: msg.requestId, status: this.hubRelationships.status() }, + }); + } catch (error) { + this.logger.error({ err: error }, "Failed to handle Hub relationship request"); + this.host.emit({ + type: "rpc_error", + payload: { + requestId: msg.requestId, + requestType: msg.type, + error: error instanceof Error ? error.message : String(error), + code: "handler_error", + }, + }); + } + } + async handleGetStatusRequest( msg: Extract, ): Promise { diff --git a/packages/server/src/server/snapshot-mutation-ownership.test.ts b/packages/server/src/server/snapshot-mutation-ownership.test.ts index 0f23c2a40..87308a3de 100644 --- a/packages/server/src/server/snapshot-mutation-ownership.test.ts +++ b/packages/server/src/server/snapshot-mutation-ownership.test.ts @@ -99,6 +99,7 @@ describe("snapshot mutation ownership boundary", () => { const session = asInternals( new Session({ clientId: "test-client", + scopes: ["*"], onMessage, logger: createStub(logger), downloadTokenStore: createStub({}), diff --git a/packages/server/src/server/test-utils/fake-agent-client.ts b/packages/server/src/server/test-utils/fake-agent-client.ts index 6468f9fb1..e025634b3 100644 --- a/packages/server/src/server/test-utils/fake-agent-client.ts +++ b/packages/server/src/server/test-utils/fake-agent-client.ts @@ -57,10 +57,12 @@ interface FakeAgentSessionOptions { sessionId?: string; memoryMarker?: string | null; closeSession?: () => Promise; + onStartTurn?: (prompt: AgentPromptInput) => void; } export interface TestAgentClientOptions { closeSession?: () => Promise; + onStartTurn?: (prompt: AgentPromptInput) => void; } function createDeferred(): Deferred { @@ -329,6 +331,7 @@ class FakeAgentSession implements AgentSession { private activeForegroundTurnId: string | null = null; private readonly closeSession: (() => Promise) | undefined; + private readonly onStartTurn: ((prompt: AgentPromptInput) => void) | undefined; constructor(options: FakeAgentSessionOptions) { this.providerName = options.providerName; @@ -336,6 +339,7 @@ class FakeAgentSession implements AgentSession { this.id = options.sessionId ?? randomUUID(); this.memoryMarker = options.memoryMarker ?? null; this.closeSession = options.closeSession; + this.onStartTurn = options.onStartTurn; this.historyPath = path.join( tmpdir(), "paseo-fake-provider-history", @@ -426,6 +430,7 @@ class FakeAgentSession implements AgentSession { const turnId = `fake-turn-${this.nextTurnOrdinal++}`; this.activeForegroundTurnId = turnId; + this.onStartTurn?.(prompt); void this.emitTurnEvents(prompt); @@ -1180,6 +1185,7 @@ class FakeAgentClient implements AgentClient { providerName: this.provider, config: { ...config }, closeSession: this.options.closeSession, + onStartTurn: this.options.onStartTurn, }); } @@ -1203,6 +1209,7 @@ class FakeAgentClient implements AgentClient { sessionId: handle.sessionId, memoryMarker: typeof marker === "string" ? marker : null, closeSession: this.options.closeSession, + onStartTurn: this.options.onStartTurn, }); } diff --git a/packages/server/src/server/test-utils/hub-cli-entry.ts b/packages/server/src/server/test-utils/hub-cli-entry.ts new file mode 100644 index 000000000..3e64439c3 --- /dev/null +++ b/packages/server/src/server/test-utils/hub-cli-entry.ts @@ -0,0 +1,4 @@ +import { createHubCommand } from "../../../../cli/src/commands/hub/index.js"; + +const argv = [process.argv[0] ?? "node", process.argv[1] ?? "paseo", ...process.argv.slice(3)]; +await createHubCommand().parseAsync(argv, { from: "node" }); diff --git a/packages/server/src/server/websocket-server.notifications.test.ts b/packages/server/src/server/websocket-server.notifications.test.ts index 0772e7bd5..fb517efe5 100644 --- a/packages/server/src/server/websocket-server.notifications.test.ts +++ b/packages/server/src/server/websocket-server.notifications.test.ts @@ -190,6 +190,7 @@ function connectClient( ) { const ws = createOpenSocket(); asInternals(server).sessions.set(ws, { + kind: "trusted", session: createSessionWithActivity(activity), clientId: "client-test", appVersion: null, diff --git a/packages/server/src/server/websocket-server.terminal-notifications.test.ts b/packages/server/src/server/websocket-server.terminal-notifications.test.ts index 3862f5b4f..7d3cba294 100644 --- a/packages/server/src/server/websocket-server.terminal-notifications.test.ts +++ b/packages/server/src/server/websocket-server.terminal-notifications.test.ts @@ -198,6 +198,7 @@ function createOpenSocket() { function connectClient(server: VoiceAssistantWebSocketServer) { const ws = createOpenSocket(); asInternals<{ sessions: Map }>(server).sessions.set(ws, { + kind: "trusted", session: { getClientActivity: vi.fn(() => null), }, diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index 649b54152..73e6f80d1 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -18,6 +18,7 @@ import type { CheckoutDiffManager, CheckoutDiffMetrics } from "./checkout-diff-m import type { DaemonConfigStore, MutableDaemonConfig } from "./daemon-config-store.js"; import { type ServerInfoStatusPayload, + type SessionOutboundMessage, type WorkspaceSetupSnapshot, type WSHelloMessage, type WSInboundMessage, @@ -32,6 +33,8 @@ import type { TerminalActivity } from "@getpaseo/protocol/terminal-activity"; import type { HostnamesConfig } from "./hostnames.js"; import { isHostnameAllowed } from "./hostnames.js"; import { Session, type SessionLifecycleIntent, type SessionRuntimeMetrics } from "./session.js"; +import type { HubRelationshipManagement } from "./hub/relationship-controller.js"; +import type { HubExecutionAgents } from "./hub/daemon-executions.js"; import type { AgentProvider } from "./agent/agent-sdk-types.js"; import { ProviderSnapshotManager } from "./agent/provider-snapshot-manager.js"; import type { WorkspaceGitRuntimeSnapshot, WorkspaceGitService } from "./workspace-git-service.js"; @@ -98,6 +101,8 @@ interface PendingConnection { interface WebSocketConnectionIdentity { connectionId: string; transport: "direct" | "relay"; + peer: "loopback" | "local_ipc" | "external"; + browserOrigin: boolean; host?: string; origin?: string; userAgent?: string; @@ -330,7 +335,7 @@ function getBrowserHostCapability( return parsed.success ? parsed.data : null; } -interface WebSocketLike { +export interface WebSocketLike { readyState: number; bufferedAmount?: number; send: (data: string | Uint8Array | ArrayBuffer) => void; @@ -339,7 +344,8 @@ interface WebSocketLike { once: (event: "close" | "error", listener: (...args: unknown[]) => void) => void; } -interface SessionConnection { +interface TrustedSessionConnection { + kind: "trusted"; session: Session; clientId: string; appVersion: string | null; @@ -349,11 +355,46 @@ interface SessionConnection { externalDisconnectCleanupTimeout: ReturnType | null; } +interface HubConnection { + kind: "hub"; + session: Session; + daemonId: string; + connectionLogger: pino.Logger; + socket: WebSocketLike; +} + +type SessionConnection = TrustedSessionConnection | HubConnection; + +type TrustedLifecycleKey = + | "clientId" + | "appVersion" + | "clientCapabilities" + | "sockets" + | "externalDisconnectCleanupTimeout"; +type HubLifecycleOverlap = Extract; +const HUB_HAS_NO_TRUSTED_LIFECYCLE_STATE: HubLifecycleOverlap extends never ? true : never = true; +void HUB_HAS_NO_TRUSTED_LIFECYCLE_STATE; + interface BrowserToolsRegistration { capabilitySignature: string; unregister: () => void; } +interface SocketSessionOptions { + clientId: string; + appVersion: string | null; + clientCapabilities: Record | null; + scopes: readonly string[]; + connectionLogger: pino.Logger; + onMessage: (message: SessionOutboundMessage) => void; + onMessageToSource?: (source: object, message: SessionOutboundMessage) => void; + onBinaryMessage?: (frame: Uint8Array) => void; + getTransportBufferedAmount?: () => number | null; + onLifecycleIntent?: (intent: SessionLifecycleIntent) => void; + hubExecutionAgents?: HubExecutionAgents; + hubRelationships?: HubRelationshipManagement; +} + const SLOW_REQUEST_THRESHOLD_MS = 500; const EXTERNAL_SESSION_DISCONNECT_GRACE_MS = 90_000; const HELLO_TIMEOUT_MS = 15_000; @@ -409,7 +450,7 @@ export class VoiceAssistantWebSocketServer { private readonly pendingConnections: Map = new Map(); private readonly sessions: Map = new Map(); private readonly socketIdentities: Map = new Map(); - private readonly externalSessionsByKey: Map = new Map(); + private readonly externalSessionsByKey: Map = new Map(); private readonly serverId: string; private readonly daemonVersion: string; private readonly daemonRuntimeConfig: DaemonRuntimeConfig | undefined; @@ -460,6 +501,7 @@ export class VoiceAssistantWebSocketServer { private readonly providerUsageService: ProviderUsageService; private unsubscribeTerminalActivity: (() => void) | null = null; private readonly browserToolsBroker: BrowserToolsBroker | null; + private readonly hubRelationships: HubRelationshipManagement | null; private readonly browserToolsRegistrations = new Map(); private acceptingConnections = true; @@ -506,6 +548,7 @@ export class VoiceAssistantWebSocketServer { daemonRuntimeConfig?: DaemonRuntimeConfig, serviceProxyPublicBaseUrl?: string | null, browserToolsBroker?: BrowserToolsBroker | null, + hubRelationships?: HubRelationshipManagement | null, ) { this.logger = logger.child({ module: "websocket-server" }); this.serverId = serverId; @@ -515,6 +558,7 @@ export class VoiceAssistantWebSocketServer { this.daemonVersion = daemonVersion.trim(); this.daemonRuntimeConfig = daemonRuntimeConfig; this.browserToolsBroker = browserToolsBroker ?? null; + this.hubRelationships = hubRelationships ?? null; this.agentManager = agentManager; this.agentStorage = agentStorage; this.projectRegistry = projectRegistry ?? createNoopProjectRegistry(); @@ -751,7 +795,10 @@ export class VoiceAssistantWebSocketServer { public broadcast(message: WSOutboundMessage): void { const payload = JSON.stringify(message); - for (const ws of this.sessions.keys()) { + for (const [ws, connection] of this.sessions) { + if (connection.kind !== "trusted") { + continue; + } // WebSocket.OPEN = 1 if (ws.readyState === 1) { ws.send(payload); @@ -760,18 +807,20 @@ export class VoiceAssistantWebSocketServer { } } - public listActiveSessions(): Session[] { + public listTrustedSessions(): Session[] { return Array.from( new Set( - [...this.sessions.values(), ...this.externalSessionsByKey.values()].map( - (connection) => connection.session, - ), + [...this.sessions.values(), ...this.externalSessionsByKey.values()] + .filter( + (connection): connection is TrustedSessionConnection => connection.kind === "trusted", + ) + .map((connection) => connection.session), ), ); } public publishProjectUpdate(update: ProjectUpdate): void { - for (const session of this.listActiveSessions()) session.emitProjectUpdate(update); + for (const session of this.listTrustedSessions()) session.emitProjectUpdate(update); } public publishSpeechReadiness(readiness: SpeechReadinessSnapshot | null): void { @@ -797,6 +846,44 @@ export class VoiceAssistantWebSocketServer { await this.attachSocket(ws, undefined, metadata); } + public async attachHubSocket( + ws: WebSocketLike, + options: { + daemonId: string; + scopes: readonly string[]; + agents: HubExecutionAgents; + }, + ): Promise { + if (!this.acceptingConnections) { + ws.close(WS_CLOSE_SERVER_SHUTDOWN, "Server shutting down"); + return; + } + + const connectionLogger = this.logger.child({ + connectionKind: "hub", + daemonId: options.daemonId, + }); + const session = this.createSocketSession({ + clientId: `hub:${options.daemonId}`, + appVersion: null, + clientCapabilities: null, + scopes: options.scopes, + connectionLogger, + onMessage: (message) => this.sendToClient(ws, wrapSessionMessage(message)), + hubExecutionAgents: options.agents, + }); + const connection: HubConnection = { + kind: "hub", + session, + daemonId: options.daemonId, + connectionLogger, + socket: ws, + }; + this.sessions.set(ws, connection); + this.bindSocketHandlers(ws); + connectionLogger.info("Hub session attached"); + } + public prepareForShutdown(): void { this.acceptingConnections = false; } @@ -832,13 +919,14 @@ export class VoiceAssistantWebSocketServer { const cleanupPromises: Promise[] = []; for (const connection of uniqueConnections) { - if (connection.externalDisconnectCleanupTimeout) { + if (connection.kind === "trusted" && connection.externalDisconnectCleanupTimeout) { clearTimeout(connection.externalDisconnectCleanupTimeout); connection.externalDisconnectCleanupTimeout = null; } - cleanupPromises.push(connection.session.cleanup()); - for (const ws of connection.sockets) { + cleanupPromises.push(Promise.resolve(connection.session.cleanup())); + const sockets = connection.kind === "trusted" ? connection.sockets : [connection.socket]; + for (const ws of sockets) { cleanupPromises.push( new Promise((resolve) => { // WebSocket.CLOSED = 3 @@ -908,12 +996,16 @@ export class VoiceAssistantWebSocketServer { } private sendToConnection(connection: SessionConnection, message: WSOutboundMessage): void { - for (const ws of connection.sockets) { + const sockets = connection.kind === "trusted" ? connection.sockets : [connection.socket]; + for (const ws of sockets) { this.sendToClient(ws, message); } } private sendBinaryToConnection(connection: SessionConnection, frame: Uint8Array): void { + if (connection.kind !== "trusted") { + return; + } for (const ws of connection.sockets) { this.sendBinaryToClient(ws, frame); } @@ -981,14 +1073,16 @@ export class VoiceAssistantWebSocketServer { appVersion: string | null; clientCapabilities: Record | null; connectionLogger: pino.Logger; - }): SessionConnection { + }): TrustedSessionConnection { const { ws, clientId, appVersion, clientCapabilities, connectionLogger } = params; - let connection: SessionConnection | null = null; + let connection: TrustedSessionConnection | null = null; - const session = new Session({ + const session = this.createSocketSession({ clientId, appVersion, clientCapabilities, + scopes: ["*"], + connectionLogger, onMessage: (msg) => { if (!connection) { return; @@ -1026,14 +1120,42 @@ export class VoiceAssistantWebSocketServer { onLifecycleIntent: (intent) => { this.onLifecycleIntent?.(intent); }, + hubRelationships: this.hubRelationships ?? undefined, + }); + + connection = { + kind: "trusted", + session, + clientId, + appVersion, + clientCapabilities, + connectionLogger, + sockets: new Set([ws]), + externalDisconnectCleanupTimeout: null, + }; + session.updateClientCapabilities(clientCapabilities, ws); + return connection; + } + + private createSocketSession(options: SocketSessionOptions): Session { + return new Session({ + clientId: options.clientId, + appVersion: options.appVersion, + clientCapabilities: options.clientCapabilities, + scopes: options.scopes, + onMessage: options.onMessage, + onMessageToSource: options.onMessageToSource, + onBinaryMessage: options.onBinaryMessage, + getTransportBufferedAmount: options.getTransportBufferedAmount, + onLifecycleIntent: options.onLifecycleIntent, + logger: options.connectionLogger.child({ module: "session" }), onWorkspaceRecovered: async (workspace) => { await Promise.all( - this.listActiveSessions().map((activeSession) => + this.listTrustedSessions().map((activeSession) => activeSession.refreshRecoveredWorkspaceForExternalMutation(workspace), ), ); }, - logger: connectionLogger.child({ module: "session" }), downloadTokenStore: this.downloadTokenStore, pushTokenStore: this.pushTokenStore, paseoHome: this.paseoHome, @@ -1057,6 +1179,8 @@ export class VoiceAssistantWebSocketServer { terminalManager: this.terminalManager, providerSnapshotManager: this.providerSnapshotManager, providerUsageService: this.providerUsageService, + hubExecutionAgents: options.hubExecutionAgents, + hubRelationships: options.hubRelationships, serviceProxy: this.serviceProxy ?? undefined, scriptRuntimeStore: this.scriptRuntimeStore ?? undefined, workspaceSetupSnapshots: this.workspaceSetupSnapshots, @@ -1096,18 +1220,6 @@ export class VoiceAssistantWebSocketServer { daemonRuntimeConfig: this.daemonRuntimeConfig, getWebSocketRuntimeMetrics: () => this.lastRuntimeMetricsSnapshot, }); - - connection = { - session, - clientId, - appVersion, - clientCapabilities, - connectionLogger, - sockets: new Set([ws]), - externalDisconnectCleanupTimeout: null, - }; - session.updateClientCapabilities(clientCapabilities, ws); - return connection; } private clearPendingConnection(ws: WebSocketLike): PendingConnection | null { @@ -1284,6 +1396,8 @@ export class VoiceAssistantWebSocketServer { providerSubagents: true, // COMPAT(workspacePinning): added in v0.1.107, remove gate after 2027-01-12. workspacePinning: true, + // COMPAT(hubRelationship): added in v0.1.X, drop the gate when floor >= v0.1.X. + hubRelationship: true, // COMPAT(projectGithubClone): added in v0.1.108, remove gate after 2027-01-15. projectGithubClone: true, // COMPAT(workspaceGithubRepositorySearch): added in v0.1.108, remove gate after 2027-01-15. @@ -1410,6 +1524,15 @@ export class VoiceAssistantWebSocketServer { } this.sessions.delete(ws); + if (connection.kind === "hub") { + this.socketIdentities.delete(ws); + connection.connectionLogger.info( + { code: details.code, reason: stringifyCloseReason(details.reason) }, + "Hub session disconnected", + ); + connection.session.cleanup(); + return; + } connection.sockets.delete(ws); connection.session.clearAgentTimelineSubscription(ws); this.socketIdentities.delete(ws); @@ -1459,7 +1582,7 @@ export class VoiceAssistantWebSocketServer { } private async cleanupConnection( - connection: SessionConnection, + connection: TrustedSessionConnection, logMessage: string, ): Promise { this.incrementRuntimeCounter("sessionCleanup"); @@ -1486,7 +1609,7 @@ export class VoiceAssistantWebSocketServer { await connection.session.cleanup(); } - private syncBrowserToolsClientRegistration(connection: SessionConnection): void { + private syncBrowserToolsClientRegistration(connection: TrustedSessionConnection): void { if (!this.browserToolsBroker) { return; } @@ -1562,7 +1685,7 @@ export class VoiceAssistantWebSocketServer { log.warn( { - clientId: activeConnection?.clientId, + clientId: activeConnection?.kind === "trusted" ? activeConnection.clientId : undefined, requestId: requestInfo?.requestId, requestType: requestInfo?.requestType, error: parsedMessage.error.message, @@ -1627,6 +1750,11 @@ export class VoiceAssistantWebSocketServer { } return true; } + if (activeConnection.kind === "hub") { + log.warn("Rejected binary frame on Hub session"); + ws.close(WS_CLOSE_INVALID_HELLO, "Binary frames are not supported on Hub sessions"); + return true; + } void Promise.resolve(activeConnection.session.handleBinaryFrame(decodedFrame)).catch( (error: unknown) => { this.handleRawMessageError({ @@ -1766,15 +1894,26 @@ export class VoiceAssistantWebSocketServer { const controlRpc = getControlRpcLogInfo(message.message); if (controlRpc) { const identity = this.socketIdentities.get(ws); + let connectionFields: Record; + if (identity) { + connectionFields = toConnectionLogFields(identity); + } else if (activeConnection.kind === "trusted") { + connectionFields = { clientId: activeConnection.clientId }; + } else { + connectionFields = { daemonId: activeConnection.daemonId }; + } activeConnection.connectionLogger.warn( { - ...(identity ? toConnectionLogFields(identity) : { clientId: activeConnection.clientId }), + ...connectionFields, ...controlRpc, }, "ws_control_rpc_received", ); } - if (message.message.type === "browser.automation.execute.response") { + if ( + activeConnection.kind === "trusted" && + message.message.type === "browser.automation.execute.response" + ) { this.browserToolsBroker?.receiveResponse(message.message as BrowserAutomationExecuteResponse); return; } @@ -1784,7 +1923,7 @@ export class VoiceAssistantWebSocketServer { const durationMs = performance.now() - startMs; this.recordRequestLatency(message.message.type, durationMs); - if (durationMs >= SLOW_REQUEST_THRESHOLD_MS) { + if (durationMs >= SLOW_REQUEST_THRESHOLD_MS && activeConnection.kind === "trusted") { activeConnection.connectionLogger.warn( { requestType: message.message.type, @@ -1896,7 +2035,9 @@ export class VoiceAssistantWebSocketServer { } private collectSessionRuntimeMetrics(): WebSocketRuntimeMetrics { - const uniqueConnections = new Set(this.externalSessionsByKey.values()); + const uniqueConnections = new Set( + this.externalSessionsByKey.values(), + ); let terminalDirectorySubscriptionCount = 0; let terminalSubscriptionCount = 0; let inflightRequests = 0; @@ -1997,6 +2138,9 @@ export class VoiceAssistantWebSocketServer { }> = []; for (const [ws, connection] of this.sessions) { + if (connection.kind !== "trusted") { + continue; + } clientEntries.push({ ws, state: this.getClientActivityState(connection.session), @@ -2079,6 +2223,9 @@ export class VoiceAssistantWebSocketServer { }> = []; for (const [ws, connection] of this.sessions) { + if (connection.kind !== "trusted") { + continue; + } clientEntries.push({ ws, state: this.getClientActivityState(connection.session), @@ -2153,6 +2300,8 @@ function createWebSocketConnectionIdentity( return { connectionId: `conn_${randomUUID().replaceAll("-", "")}`, transport: metadata?.transport === "relay" ? "relay" : "direct", + peer: resolveConnectionPeer(requestMetadata, metadata), + browserOrigin: requestMetadata.origin !== undefined, ...(requestMetadata.host ? { host: requestMetadata.host } : {}), ...(requestMetadata.origin ? { origin: requestMetadata.origin } : {}), ...(requestMetadata.userAgent ? { userAgent: requestMetadata.userAgent } : {}), @@ -2165,6 +2314,7 @@ function toConnectionLogFields(identity: WebSocketConnectionIdentity): Record messages.push(message), logger: pino({ level: "silent" }),