diff --git a/packages/server/src/server/plugins/compiler-build.ts b/packages/server/src/server/plugins/compiler-build.ts new file mode 100644 index 0000000000..5abd0df6ef --- /dev/null +++ b/packages/server/src/server/plugins/compiler-build.ts @@ -0,0 +1,431 @@ +import { stat } from "node:fs/promises"; +import { readPluginManifest } from "./manifest.js"; +import { assertPluginCompatibility } from "@getpaseo/protocol/plugin-requirements"; +import { existsSync, readFileSync, realpathSync } from "node:fs"; +import { isBuiltin } from "node:module"; +import path from "node:path"; +import { createPluginImportReader, type PluginImportKind } from "./compiler-imports.js"; +import type { Metafile, OnResolveResult, Plugin } from "esbuild"; +import { + isPluginClientOnlySdkSpecifier, + isPluginServerOnlySdkSpecifier, + PLUGIN_SDK_SPECIFIERS, +} from "./plugin-sdk-specifiers.js"; + +import { loadEsbuild, SERVER_HOST_MODULES } from "./compiler-esbuild.js"; + +type PluginBuildTarget = "client" | "server"; + +type PluginModuleLocation = PluginBuildTarget | "shared" | "invalid"; + +function directoryTarget(filePath: string, pluginDirectory: string): PluginModuleLocation | null { + const relative = path.relative(pluginDirectory, filePath); + if (relative.startsWith("..") || path.isAbsolute(relative)) return "invalid"; + const segments = relative.split(path.sep); + if (segments.includes("node_modules")) return null; + if (segments[0] === "client") return "client"; + if (segments[0] === "server") return "server"; + if (segments[0] === "shared") return "shared"; + if (/^index\.client\.tsx?$/.test(relative)) return "client"; + if (/^index\.server\.tsx?$/.test(relative)) return "server"; + return "invalid"; +} + +function containsPath(directory: string, filePath: string): boolean { + const relative = path.relative(directory, filePath); + return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); +} + +function dependencyName(specifier: string): string { + const segments = specifier.split("/"); + return specifier.startsWith("@") ? segments.slice(0, 2).join("/") : segments[0]; +} + +function findDependencyRoot( + resolvedPath: string, + specifier: string, + pluginDirectory: string, +): string | null { + const expectedName = dependencyName(specifier); + const ambientName = `@types/${expectedName.replace(/^@/, "").replace("/", "__")}`; + let directory = path.dirname(resolvedPath); + for (;;) { + const manifestPath = path.join(directory, "package.json"); + if (existsSync(manifestPath)) { + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as { name?: unknown }; + if (manifest.name === expectedName || manifest.name === ambientName) { + if (containsPath(pluginDirectory, directory) || containsPath(directory, pluginDirectory)) { + return null; + } + return directory; + } + } + const parent = path.dirname(directory); + if (parent === directory) return null; + directory = parent; + } +} + +function moduleBoundaryError( + moduleLocation: PluginModuleLocation | null, + target: PluginBuildTarget | "shared", + filePath: string, +): OnResolveResult | null { + if (moduleLocation === "invalid") { + return { + errors: [{ text: `Plugin modules belong in client/, server/, or shared/: ${filePath}` }], + }; + } + if (moduleLocation === null || moduleLocation === "shared" || moduleLocation === target) { + return null; + } + return { + errors: [ + { + text: `${moduleLocation}-only module cannot be imported into the plugin ${target} bundle: ${filePath}`, + }, + ], + }; +} + +function lexicalBoundaryError( + specifier: string, + resolveDirectory: string, + pluginDirectory: string, + target: PluginBuildTarget | "shared", +): OnResolveResult | null { + if (!specifier.startsWith(".") && !path.isAbsolute(specifier)) return null; + const lexicalPath = path.resolve(resolveDirectory, specifier); + if (containsPath(pluginDirectory, lexicalPath)) { + return moduleBoundaryError(directoryTarget(lexicalPath, pluginDirectory), target, lexicalPath); + } + // Normalize only a containing root alias. Resolving the whole import would erase + // an authored server/ or client/ location when the final file is a symlink. + for (let ancestor = path.dirname(lexicalPath); ; ancestor = path.dirname(ancestor)) { + if (existsSync(ancestor) && realpathSync.native(ancestor) === pluginDirectory) { + const ownedPath = path.join(pluginDirectory, path.relative(ancestor, lexicalPath)); + return moduleBoundaryError(directoryTarget(ownedPath, pluginDirectory), target, lexicalPath); + } + if (path.dirname(ancestor) === ancestor) return null; + } +} + +function createRuntimeBoundaryPlugin(target: PluginBuildTarget, pluginDirectory: string): Plugin { + const boundaryResolution = {}; + const linkedDependencyRoots = new Set(); + return { + name: `paseo-plugin-${target}-runtime-boundary`, + setup(buildContext) { + const checked = new Set(); + const imports = createPluginImportReader(pluginDirectory); + function resolvedBoundaryError( + resolvedFile: string, + specifier: string, + importer: string, + owner: PluginBuildTarget | "shared", + ) { + const file = realpathSync.native(resolvedFile); + const location = directoryTarget(file, pluginDirectory); + if (location === "invalid") { + if ( + [...linkedDependencyRoots].some( + (root) => containsPath(root, importer) && containsPath(root, file), + ) + ) + return null; + if (!specifier.startsWith(".") && !path.isAbsolute(specifier)) { + const root = findDependencyRoot(file, specifier, pluginDirectory); + if (root) { + linkedDependencyRoots.add(root); + return null; + } + } + } + return moduleBoundaryError(location, owner, file); + } + async function resolveImportFiles( + file: string, + specifier: string, + kind: PluginImportKind, + typeOnly: boolean, + ): Promise> { + const declaration = imports.resolve(specifier, file, kind); + const dependencyFiles = new Set(); + if (declaration && (typeOnly || /\.d\.[cm]?ts$/.test(declaration))) + dependencyFiles.add(declaration); + if (typeOnly && !declaration) { + throw new Error(`Could not resolve type dependency "${specifier}" imported by ${file}`); + } + if (!typeOnly && kind !== "type-reference") { + const resolution = await buildContext.resolve(specifier, { + importer: file, + resolveDir: path.dirname(file), + kind, + }); + // esbuild owns missing-runtime errors: erased imports and guarded optional + // requires are legal. Inspect every dependency that actually resolves. + if (!resolution.errors.length && !resolution.external && resolution.namespace === "file") + dependencyFiles.add(realpathSync.native(resolution.path)); + } + return dependencyFiles; + } + async function checkSourceImports( + sourcePath: string, + inheritedOwner: PluginBuildTarget | "shared", + ): Promise { + const file = realpathSync.native(sourcePath); + const owner = + directoryTarget(file, pluginDirectory) === "shared" ? "shared" : inheritedOwner; + const key = `${owner}:${file}`; + if (checked.has(key) || !/\.[cm]?[jt]sx?$/.test(file)) return null; + checked.add(key); + for (const { specifier, kind, typeOnly } of imports.read(file)) { + const packageSpecifier = + kind === "type-reference" + ? `@types/${specifier.replace(/^@/, "").replace("/", "__")}` + : specifier; + const error = + runtimeSpecifierError(specifier, owner, file) ?? + runtimeSpecifierError(packageSpecifier, owner, file) ?? + lexicalBoundaryError(specifier, path.dirname(file), pluginDirectory, owner); + if (error) return error; + // Protocol types are shared host contracts, also exposed by the SDK's peer + // dependency during development. Erased imports need no plugin-local install; + // runtime imports still go through normal dependency resolution below. + if (typeOnly && specifier.startsWith("@getpaseo/protocol/")) continue; + // Host modules have separately enforced SDK boundaries and need no local installation. + if ( + (PLUGIN_SDK_SPECIFIERS as readonly string[]).includes(specifier) || + /^(zod|react|react-native|@tanstack\/react-query)(\/|$)/.test(specifier) || + isBuiltin(specifier) || + packageSpecifier === "@types/node" + ) + continue; + const dependencyFiles = await resolveImportFiles(file, specifier, kind, typeOnly); + for (const dependencyFile of dependencyFiles) { + const boundaryError = resolvedBoundaryError(dependencyFile, specifier, file, owner); + if (boundaryError) return boundaryError; + const dependencyError = await checkSourceImports(dependencyFile, owner); + if (dependencyError) return dependencyError; + } + } + return null; + } + buildContext.onLoad({ filter: /\.[cm]?[jt]sx?$/ }, (args) => + checkSourceImports(args.path, target), + ); + buildContext.onResolve({ filter: /.*/ }, async (args) => { + if (args.kind === "entry-point") return null; + if (args.pluginData === boundaryResolution) return null; + const importer = + args.namespace === "file" ? realpathSync.native(args.importer) : args.importer; + const resolveDir = args.namespace === "file" ? path.dirname(importer) : args.resolveDir; + const owner = directoryTarget(importer, pluginDirectory) === "shared" ? "shared" : target; + const specifierError = runtimeSpecifierError(args.path, owner, importer); + if (specifierError) return specifierError; + const lexicalError = lexicalBoundaryError(args.path, resolveDir, pluginDirectory, owner); + if (lexicalError) return lexicalError; + const resolution = await buildContext.resolve(args.path, { + importer, + namespace: args.namespace, + resolveDir, + kind: args.kind, + pluginData: boundaryResolution, + with: args.with, + }); + if ( + resolution.errors.length > 0 || + resolution.external || + resolution.namespace !== "file" + ) { + return null; + } + return resolvedBoundaryError(resolution.path, args.path, importer, owner); + }); + }, + }; +} + +function wrapCommonJsBundle(code: string): string { + return `(function(require) {\nconst module = { exports: {} };\nconst exports = module.exports;\n${code}\nreturn module.exports;\n})`; +} + +function makeHermesInteropEager(code: string): string { + // Hermes evaluates esbuild's lazy CommonJS interop getters from a string with + // the final loop binding, so every named import can resolve to the last export. + // Plugin bundles execute once and do not need live bindings from host modules. + return code.replaceAll("get: () => from[key]", "value: from[key]"); +} + +function runtimeSpecifierError( + specifier: string, + target: PluginBuildTarget | "shared", + importer: string, +): OnResolveResult | null { + let kind: string | null = null; + if (specifier === "@getpaseo/plugin/client/host") kind = "host-private"; + else if ( + (specifier === "@getpaseo/plugin" || + specifier.startsWith("@getpaseo/plugin/") || + specifier === "@paseo/plugin" || + specifier.startsWith("@paseo/plugin/")) && + !(PLUGIN_SDK_SPECIFIERS as readonly string[]).includes(specifier) + ) + kind = "Unknown SDK"; + else if (target !== "server" && (isBuiltin(specifier) || specifier === "@types/node")) + kind = "Node"; + else if (target !== "server" && isPluginServerOnlySdkSpecifier(specifier)) kind = "server-only"; + else if ( + target !== "client" && + (isPluginClientOnlySdkSpecifier(specifier) || + /^((?:@types\/)?react(?:-dom|-native)?|use-sync-external-store|@tanstack\/react-query)(\/|$)/.test( + specifier, + )) + ) + kind = "client-only"; + return kind + ? { + errors: [ + { + text: `${kind} module cannot be imported into the plugin ${target} bundle: ${specifier} imported by ${importer}`, + }, + ], + } + : null; +} + +function checkSharedDependencies(inputs: Metafile["inputs"], pluginDirectory: string): void { + function inputLocation(file: string): PluginModuleLocation | null { + const absolutePath = path.resolve(file); + // Metafile keys must stay unchanged for graph traversal. Only filesystem + // inputs have canonical paths; data URLs and external specifiers do not. + return directoryTarget( + existsSync(absolutePath) ? realpathSync.native(absolutePath) : absolutePath, + pluginDirectory, + ); + } + const pending = Object.keys(inputs).filter((file) => inputLocation(file) === "shared"); + const visited = new Set(); + while (pending.length) { + const file = pending.pop()!; + if (visited.has(file)) continue; + visited.add(file); + for (const dependency of inputs[file]?.imports ?? []) { + const location = dependency.external ? null : inputLocation(dependency.path); + const error = + runtimeSpecifierError(dependency.original ?? dependency.path, "shared", file) ?? + (dependency.external + ? null + : moduleBoundaryError( + location === "invalid" ? null : location, + "shared", + dependency.path, + )); + if (error?.errors?.length) throw new Error(error.errors[0].text); + if (!dependency.external) pending.push(dependency.path); + } + } +} + +async function compileTarget(entryPath: string, target: PluginBuildTarget): Promise { + const { build } = loadEsbuild(); + // Use native canonical paths throughout: TypeScript expands Windows short names + // when resolving type references, while the JS realpath implementation retains them. + const pluginDirectory = realpathSync.native(path.dirname(entryPath)); + const result = await build({ + entryPoints: [realpathSync.native(entryPath)], + bundle: true, + format: "cjs", + jsx: "automatic", + // `import icon from "./icon.svg"` yields the markup for an inline `{ svg }` icon. + loader: { ".svg": "text" }, + platform: target === "server" ? "node" : "neutral", + target: target === "server" ? "node20" : "es2020", + // The neutral platform reads no package.json entry fields, so packages without + // `exports` would not resolve. Read the same fields as esbuild's browser platform + // minus `browser`, since the bundle also runs in React Native. + mainFields: target === "client" ? ["module", "main"] : undefined, + // Metro lowers async syntax before Hermes sees app code. Plugin client bundles bypass Metro, + // so apply the same compatibility transform before the app evaluates them from source. + supported: target === "client" ? { "async-await": false } : undefined, + external: + target === "client" + ? [ + ...PLUGIN_SDK_SPECIFIERS, + "@tanstack/react-query", + "react", + "react/jsx-runtime", + "react-native", + "zod", + ] + : SERVER_HOST_MODULES, + plugins: [createRuntimeBoundaryPlugin(target, pluginDirectory)], + metafile: true, + logLevel: "silent", + treeShaking: true, + write: false, + }); + checkSharedDependencies(result.metafile.inputs, pluginDirectory); + const output = result.outputFiles[0]?.text; + if (!output) throw new Error(`Plugin ${target} compilation produced no output`); + return wrapCommonJsBundle(makeHermesInteropEager(output)); +} + +export async function compilePlugin(entryPaths: { + client: string | null; + server: string | null; +}): Promise<{ + clientBundle: string | null; + serverBundle: string | null; +}> { + const [clientBundle, serverBundle] = await Promise.all([ + entryPaths.client ? compileTarget(entryPaths.client, "client") : null, + entryPaths.server ? compileTarget(entryPaths.server, "server") : null, + ]); + return { clientBundle, serverBundle }; +} + +const CLIENT_ENTRY_FILENAMES = ["index.client.ts", "index.client.tsx"] as const; +const SERVER_ENTRY_FILENAMES = ["index.server.ts", "index.server.tsx"] as const; +async function findEntry(directory: string, filenames: readonly string[]): Promise { + for (const filename of filenames) { + const filePath = path.join(directory, filename); + const info = await stat(filePath).catch(() => null); + if (info?.isFile()) return filePath; + } + return null; +} + +async function resolveEntryPaths(directory: string): Promise<{ + client: string | null; + server: string | null; +}> { + const [client, server] = await Promise.all([ + findEntry(directory, CLIENT_ENTRY_FILENAMES), + findEntry(directory, SERVER_ENTRY_FILENAMES), + ]); + if (client || server) return { client, server }; + const legacyEntry = await findEntry(directory, ["index.ts", "index.tsx"]); + if (legacyEntry) { + throw new Error( + "This plugin was made for an older version of Paseo and cannot run on Paseo v0.8. Ask its author to update it. Plugin authors can follow the migration guide: https://paseo.sh/docs/plugins/migration", + ); + } + throw new Error( + `Plugin entry points are missing: expected ${CLIENT_ENTRY_FILENAMES.join(" or ")} and/or ${SERVER_ENTRY_FILENAMES.join(" or ")} in ${directory}`, + ); +} + +export async function compileDirectoryPlugin( + directory: string, + version: string, + builtinId?: string, +) { + const manifest = await readPluginManifest(directory); + if (builtinId !== undefined && manifest.id !== builtinId) { + throw new Error(`Built-in plugin ${builtinId} has manifest ID ${manifest.id}`); + } + assertPluginCompatibility({ ...manifest, version, runtime: "daemon" }); + const bundles = await compilePlugin(await resolveEntryPaths(directory)); + return { ...bundles, manifest }; +} diff --git a/packages/server/src/server/plugins/compiler-esbuild.ts b/packages/server/src/server/plugins/compiler-esbuild.ts new file mode 100644 index 0000000000..c36c9c22f9 --- /dev/null +++ b/packages/server/src/server/plugins/compiler-esbuild.ts @@ -0,0 +1,64 @@ +import { existsSync } from "node:fs"; +import { createRequire } from "node:module"; +import path from "node:path"; +import { PLUGIN_SDK_SPECIFIERS } from "./plugin-sdk-specifiers.js"; +const nodeRequire = createRequire(import.meta.url); +export const SERVER_HOST_MODULES = [...PLUGIN_SDK_SPECIFIERS, "zod"]; +const ESBUILD_BINARY_PATH = "ESBUILD_BINARY_PATH"; + +// esbuild resolves its own platform binary via require.resolve() the first time its +// module is evaluated. Inside the packaged desktop app that resolves to a path under +// app.asar even though electron-builder unpacks the real binary to app.asar.unpacked. +// child_process.spawn bypasses Electron's asar fs shim, so the OS rejects that path +// with ENOTDIR. Point esbuild at the real unpacked binary before its module loads. +export function unpackedEsbuildBinaryFromPackageDir( + esbuildPackageDir: string, + platform: NodeJS.Platform, + arch: string, +): string | null { + const asarSegment = `${path.sep}app.asar${path.sep}`; + const asarIndex = esbuildPackageDir.indexOf(asarSegment); + if (asarIndex === -1) return null; + return path.join( + esbuildPackageDir.slice(0, asarIndex), + "app.asar.unpacked", + "node_modules", + `@esbuild/${platform}-${arch}`, + ...(platform === "win32" ? ["esbuild.exe"] : ["bin", "esbuild"]), + ); +} + +export function resolveExistingAsarUnpackedEsbuildBinary( + esbuildPackageDir: string, + platform: NodeJS.Platform = process.platform, + arch: string = process.arch, + exists: (file: string) => boolean = existsSync, +): string | null { + const binaryPath = unpackedEsbuildBinaryFromPackageDir(esbuildPackageDir, platform, arch); + return binaryPath && exists(binaryPath) ? binaryPath : null; +} + +function resolveAsarUnpackedEsbuildBinary(): string | null { + let esbuildDir: string; + try { + esbuildDir = path.dirname(nodeRequire.resolve("esbuild/package.json")); + } catch { + return null; + } + return resolveExistingAsarUnpackedEsbuildBinary(esbuildDir); +} + +export function loadEsbuild(): typeof import("esbuild") { + const previousBinaryPath = process.env[ESBUILD_BINARY_PATH]; + const unpackedBinary = resolveAsarUnpackedEsbuildBinary(); + if (unpackedBinary) process.env[ESBUILD_BINARY_PATH] = unpackedBinary; + + try { + // esbuild reads this variable while its CommonJS module is evaluated. Keep + // the compatibility bridge local so it cannot become an agent's environment. + return nodeRequire("esbuild") as typeof import("esbuild"); + } finally { + if (previousBinaryPath === undefined) delete process.env[ESBUILD_BINARY_PATH]; + else process.env[ESBUILD_BINARY_PATH] = previousBinaryPath; + } +} diff --git a/packages/server/src/server/plugins/compiler-worker.ts b/packages/server/src/server/plugins/compiler-worker.ts new file mode 100644 index 0000000000..ddf2544b8b --- /dev/null +++ b/packages/server/src/server/plugins/compiler-worker.ts @@ -0,0 +1,62 @@ +import { parentPort } from "node:worker_threads"; +import type { ZodIssue } from "zod"; +import { compilePlugin, compileDirectoryPlugin } from "./compiler-build.js"; +import { runPluginBuild } from "./preparation-build.js"; +import type { CompilerTask } from "./compiler.js"; + +export type CompilerReply = + | { kind: "log"; data: Record; message: string } + | { kind: "result"; value: unknown } + | { + kind: "error"; + error: Error; + properties: Record; + issues?: ZodIssue[]; + errors?: unknown[]; + warnings?: unknown[]; + }; + +const port = parentPort!; +// The compiler owner serializes requests; the port queues the first request +// until this entrypoint has loaded, including in source/tsx development. +port.on("message", (task: CompilerTask) => { + void run(task); +}); + +async function run(task: CompilerTask): Promise { + try { + let value: unknown; + switch (task.kind) { + case "entries": + value = await compilePlugin(task.entries); + break; + case "directory": + value = await compileDirectoryPlugin(task.directory, task.version, task.builtinId); + break; + case "prepare": + await runPluginBuild( + task.directory, + task.commands, + { + info: (data, message) => + port.postMessage({ kind: "log", data, message } satisfies CompilerReply), + }, + task.environment, + ); + break; + } + port.postMessage({ kind: "result", value } satisfies CompilerReply); + } catch (cause) { + const error = cause instanceof Error ? cause : new Error(String(cause)); + port.postMessage({ + kind: "error", + error, + // Error cloning drops own metadata such as filesystem code/path. Zod's + // readonly issues are reconstructed separately by the compiler owner. + properties: Object.fromEntries(Object.entries(error).filter(([key]) => key !== "issues")), + ...("issues" in error ? { issues: error.issues as ZodIssue[] } : {}), + ...("errors" in error ? { errors: error.errors as unknown[] } : {}), + ...("warnings" in error ? { warnings: error.warnings as unknown[] } : {}), + } satisfies CompilerReply); + } +} diff --git a/packages/server/src/server/plugins/compiler.test.ts b/packages/server/src/server/plugins/compiler.test.ts index 6f694fbaa3..67ca96e596 100644 --- a/packages/server/src/server/plugins/compiler.test.ts +++ b/packages/server/src/server/plugins/compiler.test.ts @@ -3,8 +3,10 @@ import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { ZodError } from "zod"; import { compilePlugin, + compileDirectoryPlugin, resolveExistingAsarUnpackedEsbuildBinary, unpackedEsbuildBinaryFromPackageDir, } from "./compiler.js"; @@ -975,3 +977,52 @@ export default function contribute() { void value; void secret; return () => und expect(serverBundle).toBeNull(); }); }); + +it("preserves esbuild diagnostics across the compiler boundary", async () => { + const { server } = await createSplitPlugin(); + await writeFile(server, "export default function contribute( {"); + const error = await compilePlugin({ client: null, server }).catch((cause: unknown) => cause); + expect(error).toBeInstanceOf(Error); + expect(error).toMatchObject({ + name: "Error", + message: expect.stringContaining("Expected identifier but found end of file"), + errors: [ + expect.objectContaining({ + text: "Expected identifier but found end of file", + location: expect.objectContaining({ + line: 1, + lineText: "export default function contribute( {", + }), + }), + ], + warnings: [], + }); + // A failed task does not poison the next build. + await writeFile(server, 'export default () => "recovered";'); + const bundles = await compilePlugin({ client: null, server }); + expect(bundles.serverBundle).toContain("recovered"); +}); + +it("preserves manifest validation errors across the compiler boundary", async () => { + const { directory } = await createSplitPlugin(); + await writeFile(path.join(directory, "paseo-plugin.json"), JSON.stringify({ id: 42 })); + const error = await compileDirectoryPlugin(directory, "0.13.2").catch((cause: unknown) => cause); + expect(error).toBeInstanceOf(ZodError); + expect(error).toMatchObject({ + name: "ZodError", + issues: [expect.objectContaining({ path: ["id"], code: "invalid_type" })], + }); + await writeFile(path.join(directory, "paseo-plugin.json"), "{"); + await expect(compileDirectoryPlugin(directory, "0.13.2")).rejects.toBeInstanceOf(SyntaxError); +}); + +it("preserves filesystem error metadata for a missing entry", async () => { + const { server } = await createSplitPlugin(); + await rm(server); + await expect(compilePlugin({ client: null, server })).rejects.toMatchObject({ + code: "ENOENT", + errno: expect.any(Number), + syscall: "realpath", + path: server, + }); +}); diff --git a/packages/server/src/server/plugins/compiler.ts b/packages/server/src/server/plugins/compiler.ts index 4cf89a902c..b96c8ecd4e 100644 --- a/packages/server/src/server/plugins/compiler.ts +++ b/packages/server/src/server/plugins/compiler.ts @@ -1,442 +1,152 @@ -import { existsSync, readFileSync, realpathSync } from "node:fs"; -import { createRequire, isBuiltin } from "node:module"; -import path from "node:path"; -import { createPluginImportReader, type PluginImportKind } from "./compiler-imports.js"; -import type { Metafile, OnResolveResult, Plugin } from "esbuild"; -import { - isPluginClientOnlySdkSpecifier, - isPluginServerOnlySdkSpecifier, - PLUGIN_SDK_SPECIFIERS, -} from "./plugin-sdk-specifiers.js"; +import { createRequire } from "node:module"; +import { pathToFileURL } from "node:url"; +import { Worker } from "node:worker_threads"; +import pLimit from "p-limit"; +import { ZodError } from "zod"; +import type * as build from "./compiler-build.js"; +import type { PluginManifest } from "./manifest.js"; +import type { CompilerReply } from "./compiler-worker.js"; -const nodeRequire = createRequire(import.meta.url); -export const SERVER_HOST_MODULES = [...PLUGIN_SDK_SPECIFIERS, "zod"]; -const ESBUILD_BINARY_PATH = "ESBUILD_BINARY_PATH"; +export { + SERVER_HOST_MODULES, + resolveExistingAsarUnpackedEsbuildBinary, + unpackedEsbuildBinaryFromPackageDir, +} from "./compiler-esbuild.js"; -// esbuild resolves its own platform binary via require.resolve() the first time its -// module is evaluated. Inside the packaged desktop app that resolves to a path under -// app.asar even though electron-builder unpacks the real binary to app.asar.unpacked. -// child_process.spawn bypasses Electron's asar fs shim, so the OS rejects that path -// with ENOTDIR. Point esbuild at the real unpacked binary before its module loads. -export function unpackedEsbuildBinaryFromPackageDir( - esbuildPackageDir: string, - platform: NodeJS.Platform, - arch: string, -): string | null { - const asarSegment = `${path.sep}app.asar${path.sep}`; - const asarIndex = esbuildPackageDir.indexOf(asarSegment); - if (asarIndex === -1) return null; - return path.join( - esbuildPackageDir.slice(0, asarIndex), - "app.asar.unpacked", - "node_modules", - `@esbuild/${platform}-${arch}`, - ...(platform === "win32" ? ["esbuild.exe"] : ["bin", "esbuild"]), - ); -} - -export function resolveExistingAsarUnpackedEsbuildBinary( - esbuildPackageDir: string, - platform: NodeJS.Platform = process.platform, - arch: string = process.arch, - exists: (file: string) => boolean = existsSync, -): string | null { - const binaryPath = unpackedEsbuildBinaryFromPackageDir(esbuildPackageDir, platform, arch); - return binaryPath && exists(binaryPath) ? binaryPath : null; -} - -function resolveAsarUnpackedEsbuildBinary(): string | null { - let esbuildDir: string; - try { - esbuildDir = path.dirname(nodeRequire.resolve("esbuild/package.json")); - } catch { - return null; - } - return resolveExistingAsarUnpackedEsbuildBinary(esbuildDir); -} - -function loadEsbuild(): typeof import("esbuild") { - const previousBinaryPath = process.env[ESBUILD_BINARY_PATH]; - const unpackedBinary = resolveAsarUnpackedEsbuildBinary(); - if (unpackedBinary) process.env[ESBUILD_BINARY_PATH] = unpackedBinary; - - try { - // esbuild reads this variable while its CommonJS module is evaluated. Keep - // the compatibility bridge local so it cannot become an agent's environment. - return nodeRequire("esbuild") as typeof import("esbuild"); - } finally { - if (previousBinaryPath === undefined) delete process.env[ESBUILD_BINARY_PATH]; - else process.env[ESBUILD_BINARY_PATH] = previousBinaryPath; - } -} - -type PluginBuildTarget = "client" | "server"; - -type PluginModuleLocation = PluginBuildTarget | "shared" | "invalid"; - -function directoryTarget(filePath: string, pluginDirectory: string): PluginModuleLocation | null { - const relative = path.relative(pluginDirectory, filePath); - if (relative.startsWith("..") || path.isAbsolute(relative)) return "invalid"; - const segments = relative.split(path.sep); - if (segments.includes("node_modules")) return null; - if (segments[0] === "client") return "client"; - if (segments[0] === "server") return "server"; - if (segments[0] === "shared") return "shared"; - if (/^index\.client\.tsx?$/.test(relative)) return "client"; - if (/^index\.server\.tsx?$/.test(relative)) return "server"; - return "invalid"; -} - -function containsPath(directory: string, filePath: string): boolean { - const relative = path.relative(directory, filePath); - return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); -} - -function dependencyName(specifier: string): string { - const segments = specifier.split("/"); - return specifier.startsWith("@") ? segments.slice(0, 2).join("/") : segments[0]; -} - -function findDependencyRoot( - resolvedPath: string, - specifier: string, - pluginDirectory: string, -): string | null { - const expectedName = dependencyName(specifier); - const ambientName = `@types/${expectedName.replace(/^@/, "").replace("/", "__")}`; - let directory = path.dirname(resolvedPath); - for (;;) { - const manifestPath = path.join(directory, "package.json"); - if (existsSync(manifestPath)) { - const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as { name?: unknown }; - if (manifest.name === expectedName || manifest.name === ambientName) { - if (containsPath(pluginDirectory, directory) || containsPath(directory, pluginDirectory)) { - return null; - } - return directory; - } - } - const parent = path.dirname(directory); - if (parent === directory) return null; - directory = parent; - } -} - -function moduleBoundaryError( - moduleLocation: PluginModuleLocation | null, - target: PluginBuildTarget | "shared", - filePath: string, -): OnResolveResult | null { - if (moduleLocation === "invalid") { - return { - errors: [{ text: `Plugin modules belong in client/, server/, or shared/: ${filePath}` }], +export type CompilerTask = + | { kind: "entries"; entries: Parameters[0] } + | { kind: "directory"; directory: string; version: string; builtinId?: string } + | { + kind: "prepare"; + directory: string; + commands: PluginManifest["build"]; + environment: NodeJS.ProcessEnv; }; - } - if (moduleLocation === null || moduleLocation === "shared" || moduleLocation === target) { - return null; - } - return { - errors: [ - { - text: `${moduleLocation}-only module cannot be imported into the plugin ${target} bundle: ${filePath}`, - }, - ], - }; + +interface TaskResult { + entries: Awaited>; + directory: Awaited>; + prepare: void; } -function lexicalBoundaryError( - specifier: string, - resolveDirectory: string, - pluginDirectory: string, - target: PluginBuildTarget | "shared", -): OnResolveResult | null { - if (!specifier.startsWith(".") && !path.isAbsolute(specifier)) return null; - const lexicalPath = path.resolve(resolveDirectory, specifier); - if (containsPath(pluginDirectory, lexicalPath)) { - return moduleBoundaryError(directoryTarget(lexicalPath, pluginDirectory), target, lexicalPath); - } - // Normalize only a containing root alias. Resolving the whole import would erase - // an authored server/ or client/ location when the final file is a symlink. - for (let ancestor = path.dirname(lexicalPath); ; ancestor = path.dirname(ancestor)) { - if (existsSync(ancestor) && realpathSync.native(ancestor) === pluginDirectory) { - const ownedPath = path.join(pluginDirectory, path.relative(ancestor, lexicalPath)); - return moduleBoundaryError(directoryTarget(ownedPath, pluginDirectory), target, lexicalPath); - } - if (path.dirname(ancestor) === ancestor) return null; - } -} +const IDLE_TIMEOUT_MS = 1_000; +const source = import.meta.url.endsWith(".ts"); +const workerUrl = new URL( + source ? "./compiler-worker.ts" : "./compiler-worker.js", + import.meta.url, +); +// Resolve the development loader once at module initialization, never during a +// compile request. Packaged workers execute ordinary JavaScript without tsx. +const workerEntrypoint = source + ? `import(${JSON.stringify(pathToFileURL(createRequire(import.meta.url).resolve("tsx/esm/api")).href)}).then(({ register }) => { register(); return import(${JSON.stringify(workerUrl.href)}); });` + : workerUrl; -function createRuntimeBoundaryPlugin(target: PluginBuildTarget, pluginDirectory: string): Plugin { - const boundaryResolution = {}; - const linkedDependencyRoots = new Set(); - return { - name: `paseo-plugin-${target}-runtime-boundary`, - setup(buildContext) { - const checked = new Set(); - const imports = createPluginImportReader(pluginDirectory); - function resolvedBoundaryError( - resolvedFile: string, - specifier: string, - importer: string, - owner: PluginBuildTarget | "shared", - ) { - const file = realpathSync.native(resolvedFile); - const location = directoryTarget(file, pluginDirectory); - if (location === "invalid") { - if ( - [...linkedDependencyRoots].some( - (root) => containsPath(root, importer) && containsPath(root, file), - ) - ) - return null; - if (!specifier.startsWith(".") && !path.isAbsolute(specifier)) { - const root = findDependencyRoot(file, specifier, pluginDirectory); - if (root) { - linkedDependencyRoots.add(root); - return null; - } +// Reuse the isolate during a build batch (notably daemon bootstrap), then release +// TypeScript's heap when idle. Only this owner admits work or stops the worker. +class PluginCompiler { + private readonly queue = pLimit(1); + private worker: Worker | null = null; + private idleTimer: ReturnType | null = null; + private stopping: Promise = Promise.resolve(); + + run( + task: T, + onLog?: (data: Record, message: string) => void, + ): Promise { + return this.queue(async () => { + if (this.idleTimer) clearTimeout(this.idleTimer); + this.idleTimer = null; + await this.stopping; + const worker = this.worker ?? this.startWorker(); + worker.ref(); + try { + return await this.request(worker, task, onLog); + } finally { + if (this.worker === worker) { + worker.unref(); + this.idleTimer = setTimeout(() => this.stopWorker(worker), IDLE_TIMEOUT_MS); + this.idleTimer.unref(); + } else { + await this.stopping; + } + } + }); + } + + private startWorker(): Worker { + const worker = new Worker(workerEntrypoint, { eval: source, execArgv: [] }); + this.worker = worker; + // Keep error/exit ownership while idle too. The next caller gets a fresh + // isolate after a crash; author build commands are never retried. + worker.on("error", () => this.stopWorker(worker)); + worker.on("exit", () => { + if (this.worker === worker) this.stopWorker(worker); + }); + return worker; + } + + private stopWorker(worker: Worker): void { + if (this.worker !== worker) return; + this.worker = null; + if (this.idleTimer) clearTimeout(this.idleTimer); + this.idleTimer = null; + this.stopping = worker.terminate(); + } + + private async request( + worker: Worker, + task: T, + onLog?: (data: Record, message: string) => void, + ): Promise { + let onMessage: (reply: CompilerReply) => void; + let onError: (error: Error) => void; + let onExit: (code: number) => void; + try { + return await new Promise((resolve, reject) => { + onError = reject; + onExit = (code) => reject(new Error(`Plugin compiler worker exited (code ${code})`)); + onMessage = (reply) => { + if (reply.kind === "log") { + onLog?.(reply.data, reply.message); + } else if (reply.kind === "result") { + resolve(reply.value as TaskResult[T["kind"]]); + } else { + const error = reply.issues ? new ZodError(reply.issues) : reply.error; + Object.assign(error, reply.properties); + if (reply.errors) Object.assign(error, { errors: reply.errors }); + if (reply.warnings) Object.assign(error, { warnings: reply.warnings }); + // Promise.all can reject while the other target is still compiling. + // Retire the isolate before another request can enter it. + this.stopWorker(worker); + reject(error); } - } - return moduleBoundaryError(location, owner, file); - } - async function resolveImportFiles( - file: string, - specifier: string, - kind: PluginImportKind, - typeOnly: boolean, - ): Promise> { - const declaration = imports.resolve(specifier, file, kind); - const dependencyFiles = new Set(); - if (declaration && (typeOnly || /\.d\.[cm]?ts$/.test(declaration))) - dependencyFiles.add(declaration); - if (typeOnly && !declaration) { - throw new Error(`Could not resolve type dependency "${specifier}" imported by ${file}`); - } - if (!typeOnly && kind !== "type-reference") { - const resolution = await buildContext.resolve(specifier, { - importer: file, - resolveDir: path.dirname(file), - kind, - }); - // esbuild owns missing-runtime errors: erased imports and guarded optional - // requires are legal. Inspect every dependency that actually resolves. - if (!resolution.errors.length && !resolution.external && resolution.namespace === "file") - dependencyFiles.add(realpathSync.native(resolution.path)); - } - return dependencyFiles; - } - async function checkSourceImports( - sourcePath: string, - inheritedOwner: PluginBuildTarget | "shared", - ): Promise { - const file = realpathSync.native(sourcePath); - const owner = - directoryTarget(file, pluginDirectory) === "shared" ? "shared" : inheritedOwner; - const key = `${owner}:${file}`; - if (checked.has(key) || !/\.[cm]?[jt]sx?$/.test(file)) return null; - checked.add(key); - for (const { specifier, kind, typeOnly } of imports.read(file)) { - const packageSpecifier = - kind === "type-reference" - ? `@types/${specifier.replace(/^@/, "").replace("/", "__")}` - : specifier; - const error = - runtimeSpecifierError(specifier, owner, file) ?? - runtimeSpecifierError(packageSpecifier, owner, file) ?? - lexicalBoundaryError(specifier, path.dirname(file), pluginDirectory, owner); - if (error) return error; - // Protocol types are shared host contracts, also exposed by the SDK's peer - // dependency during development. Erased imports need no plugin-local install; - // runtime imports still go through normal dependency resolution below. - if (typeOnly && specifier.startsWith("@getpaseo/protocol/")) continue; - // Host modules have separately enforced SDK boundaries and need no local installation. - if ( - (PLUGIN_SDK_SPECIFIERS as readonly string[]).includes(specifier) || - /^(zod|react|react-native|@tanstack\/react-query)(\/|$)/.test(specifier) || - isBuiltin(specifier) || - packageSpecifier === "@types/node" - ) - continue; - const dependencyFiles = await resolveImportFiles(file, specifier, kind, typeOnly); - for (const dependencyFile of dependencyFiles) { - const boundaryError = resolvedBoundaryError(dependencyFile, specifier, file, owner); - if (boundaryError) return boundaryError; - const dependencyError = await checkSourceImports(dependencyFile, owner); - if (dependencyError) return dependencyError; - } - } - return null; - } - buildContext.onLoad({ filter: /\.[cm]?[jt]sx?$/ }, (args) => - checkSourceImports(args.path, target), - ); - buildContext.onResolve({ filter: /.*/ }, async (args) => { - if (args.kind === "entry-point") return null; - if (args.pluginData === boundaryResolution) return null; - const importer = - args.namespace === "file" ? realpathSync.native(args.importer) : args.importer; - const resolveDir = args.namespace === "file" ? path.dirname(importer) : args.resolveDir; - const owner = directoryTarget(importer, pluginDirectory) === "shared" ? "shared" : target; - const specifierError = runtimeSpecifierError(args.path, owner, importer); - if (specifierError) return specifierError; - const lexicalError = lexicalBoundaryError(args.path, resolveDir, pluginDirectory, owner); - if (lexicalError) return lexicalError; - const resolution = await buildContext.resolve(args.path, { - importer, - namespace: args.namespace, - resolveDir, - kind: args.kind, - pluginData: boundaryResolution, - with: args.with, - }); - if ( - resolution.errors.length > 0 || - resolution.external || - resolution.namespace !== "file" - ) { - return null; - } - return resolvedBoundaryError(resolution.path, args.path, importer, owner); + }; + worker.on("error", onError); + worker.on("exit", onExit); + worker.on("message", onMessage); + worker.postMessage(task, []); }); - }, - }; -} - -function wrapCommonJsBundle(code: string): string { - return `(function(require) {\nconst module = { exports: {} };\nconst exports = module.exports;\n${code}\nreturn module.exports;\n})`; -} - -function makeHermesInteropEager(code: string): string { - // Hermes evaluates esbuild's lazy CommonJS interop getters from a string with - // the final loop binding, so every named import can resolve to the last export. - // Plugin bundles execute once and do not need live bindings from host modules. - return code.replaceAll("get: () => from[key]", "value: from[key]"); -} - -function runtimeSpecifierError( - specifier: string, - target: PluginBuildTarget | "shared", - importer: string, -): OnResolveResult | null { - let kind: string | null = null; - if (specifier === "@getpaseo/plugin/client/host") kind = "host-private"; - else if ( - (specifier === "@getpaseo/plugin" || - specifier.startsWith("@getpaseo/plugin/") || - specifier === "@paseo/plugin" || - specifier.startsWith("@paseo/plugin/")) && - !(PLUGIN_SDK_SPECIFIERS as readonly string[]).includes(specifier) - ) - kind = "Unknown SDK"; - else if (target !== "server" && (isBuiltin(specifier) || specifier === "@types/node")) - kind = "Node"; - else if (target !== "server" && isPluginServerOnlySdkSpecifier(specifier)) kind = "server-only"; - else if ( - target !== "client" && - (isPluginClientOnlySdkSpecifier(specifier) || - /^((?:@types\/)?react(?:-dom|-native)?|use-sync-external-store|@tanstack\/react-query)(\/|$)/.test( - specifier, - )) - ) - kind = "client-only"; - return kind - ? { - errors: [ - { - text: `${kind} module cannot be imported into the plugin ${target} bundle: ${specifier} imported by ${importer}`, - }, - ], - } - : null; -} - -function checkSharedDependencies(inputs: Metafile["inputs"], pluginDirectory: string): void { - function inputLocation(file: string): PluginModuleLocation | null { - const absolutePath = path.resolve(file); - // Metafile keys must stay unchanged for graph traversal. Only filesystem - // inputs have canonical paths; data URLs and external specifiers do not. - return directoryTarget( - existsSync(absolutePath) ? realpathSync.native(absolutePath) : absolutePath, - pluginDirectory, - ); - } - const pending = Object.keys(inputs).filter((file) => inputLocation(file) === "shared"); - const visited = new Set(); - while (pending.length) { - const file = pending.pop()!; - if (visited.has(file)) continue; - visited.add(file); - for (const dependency of inputs[file]?.imports ?? []) { - const location = dependency.external ? null : inputLocation(dependency.path); - const error = - runtimeSpecifierError(dependency.original ?? dependency.path, "shared", file) ?? - (dependency.external - ? null - : moduleBoundaryError( - location === "invalid" ? null : location, - "shared", - dependency.path, - )); - if (error?.errors?.length) throw new Error(error.errors[0].text); - if (!dependency.external) pending.push(dependency.path); + } finally { + worker.off("message", onMessage!); + worker.off("error", onError!); + worker.off("exit", onExit!); } } } -async function compileTarget(entryPath: string, target: PluginBuildTarget): Promise { - const { build } = loadEsbuild(); - // Use native canonical paths throughout: TypeScript expands Windows short names - // when resolving type references, while the JS realpath implementation retains them. - const pluginDirectory = realpathSync.native(path.dirname(entryPath)); - const result = await build({ - entryPoints: [realpathSync.native(entryPath)], - bundle: true, - format: "cjs", - jsx: "automatic", - // `import icon from "./icon.svg"` yields the markup for an inline `{ svg }` icon. - loader: { ".svg": "text" }, - platform: target === "server" ? "node" : "neutral", - target: target === "server" ? "node20" : "es2020", - // The neutral platform reads no package.json entry fields, so packages without - // `exports` would not resolve. Read the same fields as esbuild's browser platform - // minus `browser`, since the bundle also runs in React Native. - mainFields: target === "client" ? ["module", "main"] : undefined, - // Metro lowers async syntax before Hermes sees app code. Plugin client bundles bypass Metro, - // so apply the same compatibility transform before the app evaluates them from source. - supported: target === "client" ? { "async-await": false } : undefined, - external: - target === "client" - ? [ - ...PLUGIN_SDK_SPECIFIERS, - "@tanstack/react-query", - "react", - "react/jsx-runtime", - "react-native", - "zod", - ] - : SERVER_HOST_MODULES, - plugins: [createRuntimeBoundaryPlugin(target, pluginDirectory)], - metafile: true, - logLevel: "silent", - treeShaking: true, - write: false, - }); - checkSharedDependencies(result.metafile.inputs, pluginDirectory); - const output = result.outputFiles[0]?.text; - if (!output) throw new Error(`Plugin ${target} compilation produced no output`); - return wrapCommonJsBundle(makeHermesInteropEager(output)); +const compiler = new PluginCompiler(); + +export function runCompilerTask( + task: T, + onLog?: (data: Record, message: string) => void, +): Promise { + return compiler.run(task, onLog); } -export async function compilePlugin(entryPaths: { - client: string | null; - server: string | null; -}): Promise<{ - clientBundle: string | null; - serverBundle: string | null; -}> { - const [clientBundle, serverBundle] = await Promise.all([ - entryPaths.client ? compileTarget(entryPaths.client, "client") : null, - entryPaths.server ? compileTarget(entryPaths.server, "server") : null, - ]); - return { clientBundle, serverBundle }; +export function compilePlugin(entries: Parameters[0]) { + return runCompilerTask({ kind: "entries", entries }); +} + +export function compileDirectoryPlugin(directory: string, version: string, builtinId?: string) { + return runCompilerTask({ kind: "directory", directory, version, builtinId }); } diff --git a/packages/server/src/server/plugins/plugin-paseo-api.e2e.test.ts b/packages/server/src/server/plugins/plugin-paseo-api.e2e.test.ts index 729b36fa1c..cad97ce743 100644 --- a/packages/server/src/server/plugins/plugin-paseo-api.e2e.test.ts +++ b/packages/server/src/server/plugins/plugin-paseo-api.e2e.test.ts @@ -1,3 +1,6 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { performance } from "node:perf_hooks"; import pino from "pino"; import { createPaseoDaemon } from "../bootstrap.js"; import { loadConfig } from "../config.js"; @@ -522,3 +525,24 @@ export default function contribute(server: PluginServerContext) { await daemon.close(); } }, 60_000); + +test("keeps reload and enable responsive while compiling a real plugin", async () => { + const directory = await mkdtemp(path.join(tmpdir(), "paseo-responsive-plugin-")); + roots.push(directory); + const { stdout, stderr } = await promisify(execFile)( + process.execPath, + [ + "--import", + "tsx", + fileURLToPath(new URL("./test-fixtures/compile-responsiveness.e2e.ts", import.meta.url)), + directory, + ], + { timeout: 20_000 }, + ); + process.stderr.write(stderr); + const results = z + .array(z.object({ blockingMs: z.number() })) + .length(2) + .parse(JSON.parse(stdout)); + for (const result of results) expect(result.blockingMs).toBeLessThan(20); +}); diff --git a/packages/server/src/server/plugins/preparation-build.ts b/packages/server/src/server/plugins/preparation-build.ts new file mode 100644 index 0000000000..cd42772617 --- /dev/null +++ b/packages/server/src/server/plugins/preparation-build.ts @@ -0,0 +1,76 @@ +import { spawnProcess } from "../../utils/spawn.js"; + +import type { PluginManifest } from "./manifest.js"; + +const OUTPUT_LIMIT = 64 * 1024; + +export async function runPluginBuild( + directory: string, + commands: PluginManifest["build"], + logger: { info(data: Record, message: string): void }, + environment: NodeJS.ProcessEnv, +): Promise { + for (const command of commands ?? []) { + logger.info({ directory, command }, "Running plugin build command"); + const result = await run(command, directory, environment); + if (result.stdout) + logger.info({ directory, command, output: result.stdout }, "Plugin build stdout"); + if (result.stderr) + logger.info({ directory, command, output: result.stderr }, "Plugin build stderr"); + if (result.error) { + throw new Error( + `Plugin build command failed to start: ${formatCommand(command)}\n${result.error.message}`, + ); + } + if (result.exitCode !== 0) { + throw new Error( + `Plugin build command failed (exit ${result.exitCode}): ${formatCommand(command)}${formatOutput(result)}`, + ); + } + } +} + +function run( + command: string[], + directory: string, + environment: NodeJS.ProcessEnv, +): Promise { + return new Promise((resolve) => { + const executable = command[0]!; + const arguments_ = command.slice(1); + const child = spawnProcess(executable, arguments_, { cwd: directory, env: environment }); + let stdout = ""; + let stderr = ""; + let error: Error | undefined; + child.stdout?.on("data", (chunk: Buffer | string) => { + stdout = appendOutput(stdout, chunk.toString()); + }); + child.stderr?.on("data", (chunk: Buffer | string) => { + stderr = appendOutput(stderr, chunk.toString()); + }); + child.once("error", (spawnError) => { + error = spawnError; + }); + child.once("close", (exitCode) => resolve({ exitCode, stdout, stderr, error })); + }); +} + +function appendOutput(current: string, next: string): string { + return `${current}${next}`.slice(-OUTPUT_LIMIT); +} + +function formatCommand(command: string[]): string { + return command.map((argument) => JSON.stringify(argument)).join(" "); +} + +function formatOutput(result: CommandResult): string { + const output = [result.stdout, result.stderr].filter(Boolean).join("\n").trim(); + return output ? `\n${output}` : ""; +} + +interface CommandResult { + exitCode: number | null; + stdout: string; + stderr: string; + error?: Error; +} diff --git a/packages/server/src/server/plugins/preparation.test.ts b/packages/server/src/server/plugins/preparation.test.ts index 0fdb0e34fc..d83fa012e3 100644 --- a/packages/server/src/server/plugins/preparation.test.ts +++ b/packages/server/src/server/plugins/preparation.test.ts @@ -1,6 +1,8 @@ import { chmod, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; +import { Writable } from "node:stream"; +import pino from "pino"; import { afterEach, describe, expect, it } from "vitest"; import { createTestLogger } from "../../test-utils/test-logger.js"; import { runPluginBuild } from "./preparation.js"; @@ -117,6 +119,52 @@ describe("runPluginBuild", () => { ).rejects.toThrow(/Plugin build command failed \(exit 2\)/); }); + it("forwards build progress and output from the worker to the caller's logger", async () => { + const directory = await mkdtemp(path.join(tmpdir(), "paseo-plugin-build-")); + directories.push(directory); + const records: Array<{ msg: string; output?: string }> = []; + const logger = pino( + new Writable({ + write(chunk, _encoding, callback) { + records.push(JSON.parse(chunk.toString())); + callback(); + }, + }), + ); + await runPluginBuild( + directory, + [[process.execPath, "-e", 'console.log("prepared"); console.error("diagnostic");']], + logger, + ); + expect(records).toMatchObject([ + { msg: "Running plugin build command" }, + { msg: "Plugin build stdout", output: "prepared\n" }, + { msg: "Plugin build stderr", output: expect.stringContaining("diagnostic\n") }, + ]); + }); + + it("uses the caller's current environment across consecutive builds", async () => { + const directory = await mkdtemp(path.join(tmpdir(), "paseo-plugin-build-")); + directories.push(directory); + const previous = process.env.PASEO_PLUGIN_TEST_BUILD_VALUE; + const command = [ + process.execPath, + "-e", + 'require("node:fs").writeFileSync(process.argv[1], process.env.PASEO_PLUGIN_TEST_BUILD_VALUE);', + ]; + try { + process.env.PASEO_PLUGIN_TEST_BUILD_VALUE = "first"; + await runPluginBuild(directory, [[...command, "first.txt"]], createTestLogger()); + process.env.PASEO_PLUGIN_TEST_BUILD_VALUE = "second"; + await runPluginBuild(directory, [[...command, "second.txt"]], createTestLogger()); + expect(await readFile(path.join(directory, "first.txt"), "utf8")).toBe("first"); + expect(await readFile(path.join(directory, "second.txt"), "utf8")).toBe("second"); + } finally { + if (previous === undefined) delete process.env.PASEO_PLUGIN_TEST_BUILD_VALUE; + else process.env.PASEO_PLUGIN_TEST_BUILD_VALUE = previous; + } + }); + it("handles empty or undefined build commands gracefully", async () => { const directory = await mkdtemp(path.join(tmpdir(), "paseo-plugin-build-")); directories.push(directory); diff --git a/packages/server/src/server/plugins/preparation.ts b/packages/server/src/server/plugins/preparation.ts index 267f9697e9..79aec65cdc 100644 --- a/packages/server/src/server/plugins/preparation.ts +++ b/packages/server/src/server/plugins/preparation.ts @@ -1,71 +1,15 @@ -import { spawnProcess } from "../../utils/spawn.js"; import type pino from "pino"; import type { PluginManifest } from "./manifest.js"; - -const OUTPUT_LIMIT = 64 * 1024; +import { runCompilerTask } from "./compiler.js"; export async function runPluginBuild( directory: string, commands: PluginManifest["build"], logger: pino.Logger, ): Promise { - for (const command of commands ?? []) { - logger.info({ directory, command }, "Running plugin build command"); - const result = await run(command, directory); - if (result.stdout) - logger.info({ directory, command, output: result.stdout }, "Plugin build stdout"); - if (result.stderr) - logger.info({ directory, command, output: result.stderr }, "Plugin build stderr"); - if (result.error) { - throw new Error( - `Plugin build command failed to start: ${formatCommand(command)}\n${result.error.message}`, - ); - } - if (result.exitCode !== 0) { - throw new Error( - `Plugin build command failed (exit ${result.exitCode}): ${formatCommand(command)}${formatOutput(result)}`, - ); - } - } -} - -function run(command: string[], directory: string): Promise { - return new Promise((resolve) => { - const executable = command[0]!; - const arguments_ = command.slice(1); - const child = spawnProcess(executable, arguments_, { cwd: directory }); - let stdout = ""; - let stderr = ""; - let error: Error | undefined; - child.stdout?.on("data", (chunk: Buffer | string) => { - stdout = appendOutput(stdout, chunk.toString()); - }); - child.stderr?.on("data", (chunk: Buffer | string) => { - stderr = appendOutput(stderr, chunk.toString()); - }); - child.once("error", (spawnError) => { - error = spawnError; - }); - child.once("close", (exitCode) => resolve({ exitCode, stdout, stderr, error })); - }); -} - -function appendOutput(current: string, next: string): string { - return `${current}${next}`.slice(-OUTPUT_LIMIT); -} - -function formatCommand(command: string[]): string { - return command.map((argument) => JSON.stringify(argument)).join(" "); -} - -function formatOutput(result: CommandResult): string { - const output = [result.stdout, result.stderr].filter(Boolean).join("\n").trim(); - return output ? `\n${output}` : ""; -} - -interface CommandResult { - exitCode: number | null; - stdout: string; - stderr: string; - error?: Error; + if (!commands?.length) return; + await runCompilerTask( + { kind: "prepare", directory, commands, environment: { ...process.env } }, + (data, message) => logger.info(data, message), + ); } diff --git a/packages/server/src/server/plugins/runtime.posix.test.ts b/packages/server/src/server/plugins/runtime.posix.test.ts index 8dd80ce499..30ada955a6 100644 --- a/packages/server/src/server/plugins/runtime.posix.test.ts +++ b/packages/server/src/server/plugins/runtime.posix.test.ts @@ -1,4 +1,4 @@ -import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; import { fork } from "node:child_process"; import { tmpdir } from "node:os"; import { PassThrough } from "node:stream"; @@ -1849,9 +1849,9 @@ export default function contribute(plugin: any) { const child = createReloadChild("blocked", events); const runtime = createTestRuntime({ spawnChild: () => child }); - await expect(runtime.startPlugin("blocked", directory, () => false)).rejects.toThrow( - "Plugin start cancelled: blocked", - ); + await expect( + runtime.startPlugin("blocked", directory, () => events.length === 0), + ).rejects.toThrow("Plugin start cancelled: blocked"); expect(runtime.catalog()).toEqual([]); expect(events).toEqual(["start:blocked", "shutdown:blocked", "exit:blocked"]); @@ -2055,3 +2055,50 @@ export default function contribute(server) { await runtime.stopAll(); } }); + +const examplesDirectory = fileURLToPath( + new URL("../../../../../plugin-examples/", import.meta.url), +); +// The Markdown example owns npm preparation in the real app fixture: +// packages/app/e2e/browser/plugin-file-panel-examples.spec.ts. +const examples = (await readdir(examplesDirectory, { withFileTypes: true })) + .filter((entry) => entry.isDirectory() && entry.name !== "markdown-panel") + .map((entry) => entry.name); + +it.each(examples)("builds and loads the %s plugin example", async (example) => { + const settingsDirectory = await mkdtemp(path.join(tmpdir(), "paseo-example-settings-")); + temporaryDirectories.push(settingsDirectory); + const runtime = createTestRuntime({ settingsDirectory }, undefined, "0.13.2"); + try { + await runtime.startPlugin(example, path.join(examplesDirectory, example)); + expect(runtime.catalog()).toMatchObject([{ id: example }]); + expect(pluginMessages(runtime, example)).toContain("[paseo] Plugin ready"); + } finally { + await runtime.stopAll(); + } +}); + +it("does not initialize a plugin cancelled during compilation", async () => { + const directory = await createPlugin("cancelled-build", "export default () => {};"); + const marker = path.join(directory, "initialized.txt"); + await writeFile( + path.join(directory, "index.server.ts"), + ` +import { writeFileSync } from "node:fs"; +export default function contribute() { + writeFileSync(${JSON.stringify(marker)}, "initialized"); + return () => {}; +}`, + ); + const runtime = createTestRuntime(); + let allowed = true; + try { + const loading = runtime.startPlugin("cancelled-build", directory, () => allowed); + allowed = false; + await expect(loading).rejects.toThrow("Plugin start cancelled: cancelled-build"); + expect(await readTextIfPresent(marker)).toBeNull(); + expect(runtime.catalog()).toEqual([]); + } finally { + await runtime.stopAll(); + } +}); diff --git a/packages/server/src/server/plugins/runtime.ts b/packages/server/src/server/plugins/runtime.ts index f5a644ef9d..bc06d5d991 100644 --- a/packages/server/src/server/plugins/runtime.ts +++ b/packages/server/src/server/plugins/runtime.ts @@ -13,7 +13,6 @@ import type { } from "@getpaseo/plugin/server"; import { validateBeforeRequest, validateBeforeResult } from "./lifecycle/index.js"; import { fork } from "node:child_process"; -import { stat } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { randomUUID } from "node:crypto"; @@ -28,10 +27,8 @@ import { type ProviderInput, } from "@getpaseo/plugin/server/provider"; import type { PluginLogEntry } from "@getpaseo/protocol/messages"; -import { compilePlugin } from "./compiler.js"; -import { readPluginManifest } from "./manifest.js"; +import { compileDirectoryPlugin } from "./compiler.js"; import type { PluginRequirements } from "@getpaseo/protocol/messages"; -import { assertPluginCompatibility } from "@getpaseo/protocol/plugin-requirements"; import type { PluginProcessMessage, PluginProcessRequest, @@ -43,8 +40,6 @@ import { PluginSessionSocket } from "./session-socket.js"; import { InternalPluginChild } from "./internal-child.js"; import { evaluateBundle } from "./bundle-evaluator.js"; -const CLIENT_ENTRY_FILENAMES = ["index.client.ts", "index.client.tsx"] as const; -const SERVER_ENTRY_FILENAMES = ["index.server.ts", "index.server.tsx"] as const; const REQUEST_TIMEOUT_MS = 30_000; const TOOL_CALL_TIMEOUT_MS = 10 * 60_000; const MAX_LOG_ENTRIES = 500; @@ -273,35 +268,6 @@ function send(child: PluginChild, message: PluginProcessRequest): Promise }); } -async function findEntry(directory: string, filenames: readonly string[]): Promise { - for (const filename of filenames) { - const filePath = path.join(directory, filename); - const info = await stat(filePath).catch(() => null); - if (info?.isFile()) return filePath; - } - return null; -} - -async function resolveEntryPaths(directory: string): Promise<{ - client: string | null; - server: string | null; -}> { - const [client, server] = await Promise.all([ - findEntry(directory, CLIENT_ENTRY_FILENAMES), - findEntry(directory, SERVER_ENTRY_FILENAMES), - ]); - if (client || server) return { client, server }; - const legacyEntry = await findEntry(directory, ["index.ts", "index.tsx"]); - if (legacyEntry) { - throw new Error( - "This plugin was made for an older version of Paseo and cannot run on Paseo v0.8. Ask its author to update it. Plugin authors can follow the migration guide: https://paseo.sh/docs/plugins/migration", - ); - } - throw new Error( - `Plugin entry points are missing: expected ${CLIENT_ENTRY_FILENAMES.join(" or ")} and/or ${SERVER_ENTRY_FILENAMES.join(" or ")} in ${directory}`, - ); -} - export class PluginRuntime { private readonly plugins = new Map(); private readonly pendingEvents = new Set>(); @@ -350,10 +316,16 @@ export class PluginRuntime { ): Promise { if (this.plugins.has(pluginId)) throw new Error(`Plugin is already running: ${pluginId}`); this.appendLog(pluginId, "stdout", "[paseo] Loading plugin"); - const loaded = await this.loadDirectoryPlugin(pluginId, configuredPath).catch((error) => { - this.appendLog(pluginId, "stderr", `[paseo] Plugin failed to load: ${describeError(error)}`); - throw error; - }); + const loaded = await this.loadDirectoryPlugin(pluginId, configuredPath, canPublish).catch( + (error) => { + this.appendLog( + pluginId, + "stderr", + `[paseo] Plugin failed to load: ${describeError(error)}`, + ); + throw error; + }, + ); if (!canPublish()) { await this.stopPlugin(loaded); throw new Error(`Plugin start cancelled: ${pluginId}`); @@ -367,12 +339,8 @@ export class PluginRuntime { if (this.plugins.has(input.id)) throw new Error(`Plugin is already running: ${input.id}`); this.appendLog(input.id, "stdout", "[paseo] Loading plugin"); const directory = path.resolve(input.directory); - const manifest = await readPluginManifest(directory); - if (manifest.id !== input.id) { - throw new Error(`Built-in plugin ${input.id} has manifest ID ${manifest.id}`); - } - assertPluginCompatibility({ ...manifest, version: this.daemonVersion, runtime: "daemon" }); - const bundles = await compilePlugin(await resolveEntryPaths(directory)); + const bundles = await compileDirectoryPlugin(directory, this.daemonVersion, input.id); + const { manifest } = bundles; if (!bundles.serverBundle) throw new Error(`Built-in plugin ${input.id} needs a server entry`); const loaded = await this.launchPlugin({ pluginId: input.id, @@ -388,11 +356,7 @@ export class PluginRuntime { } async validatePlugin(configuredPath: string): Promise { - const directory = path.resolve(configuredPath); - const manifest = await readPluginManifest(directory); - assertPluginCompatibility({ ...manifest, version: this.daemonVersion, runtime: "daemon" }); - const entryPaths = await resolveEntryPaths(directory); - await compilePlugin(entryPaths); + await compileDirectoryPlugin(path.resolve(configuredPath), this.daemonVersion); } async stopPluginById(pluginId: string): Promise { @@ -738,12 +702,12 @@ export class PluginRuntime { private async loadDirectoryPlugin( pluginId: string, configuredPath: string, + canPublish: () => boolean, ): Promise { const directory = path.resolve(configuredPath); - const manifest = await readPluginManifest(directory); - assertPluginCompatibility({ ...manifest, version: this.daemonVersion, runtime: "daemon" }); - const entryPaths = await resolveEntryPaths(directory); - const bundles = await compilePlugin(entryPaths); + const bundles = await compileDirectoryPlugin(directory, this.daemonVersion); + const { manifest } = bundles; + if (!canPublish()) throw new Error(`Plugin start cancelled: ${pluginId}`); const serverBundle = bundles.serverBundle; if (!serverBundle) { return { diff --git a/packages/server/src/server/plugins/test-fixtures/compile-responsiveness.e2e.ts b/packages/server/src/server/plugins/test-fixtures/compile-responsiveness.e2e.ts new file mode 100644 index 0000000000..229e388b34 --- /dev/null +++ b/packages/server/src/server/plugins/test-fixtures/compile-responsiveness.e2e.ts @@ -0,0 +1,187 @@ +import assert from "node:assert/strict"; +import { createHook, executionAsyncResource } from "node:async_hooks"; +import { open as openFile, writeFile } from "node:fs/promises"; +import { performance, PerformanceObserver } from "node:perf_hooks"; +import path from "node:path"; +import { MessagePort } from "node:worker_threads"; +import pino from "pino"; +import { DaemonConfigStore } from "../../daemon-config-store.js"; +import { PluginService } from "../index.js"; + +// A fresh process excludes the E2E harness's transform/bootstrap garbage from +// the compiler measurement. All work and dependencies below are real. +const directory = process.argv[2]; +assert.ok(directory, "Fixture directory is required"); +const results: { blockingMs: number }[] = []; +await writeFile( + path.join(directory, "paseo-plugin.json"), + JSON.stringify({ id: "responsive-build" }), +); +// Erased types stress TypeScript's AST walk without a multi-MiB runtime bundle. +// Payload transfer is a different cost from import validation. +const entryFile = await openFile(path.join(directory, "index.client.ts"), "w"); +try { + await entryFile.write("type FixtureValues = ["); + for (let offset = 0; offset < 350_000; offset += 10_000) { + await entryFile.write( + Array.from({ length: 10_000 }, (_, index) => offset + index).join(",") + ",", + ); + } + await entryFile.write( + ']; export default function contribute() { return () => "compiled fixture"; }', + ); +} finally { + await entryFile.close(); +} +const store = new DaemonConfigStore(directory, { + mcp: { injectIntoAgents: true }, + browserTools: { enabled: false }, + providers: {}, + metadataGeneration: { providers: [] }, + autoArchiveAfterMerge: false, + enableTerminalAgentHooks: false, + appendSystemPrompt: "", + pluginsEnabled: true, + plugins: { + "responsive-build": { source: "directory", path: directory, enabled: false }, + }, +}); +const service = new PluginService(pino({ level: "silent" }), store, "0.4.0"); +await service.start(); +try { + for (const action of [ + () => service.enablePlugin("responsive-build"), + () => service.reloadPlugin("responsive-build"), + ]) { + // Node 22.19+ exposes per-thread CPU accounting; the repository's older + // Node declarations do not include it yet. .tool-versions pins Node 22.20. + const threadCpuUsage: (previous?: NodeJS.CpuUsage) => NodeJS.CpuUsage = Reflect.get( + process, + "threadCpuUsage", + ); + const startedAt = performance.now(); + let previous = startedAt; + let cpu = threadCpuUsage(); + let longestWall = 0; + let longestBlocking = 0; + let longestWallCpu = 0; + let workerStarts = 0; + let portCallbacks = 0; + let longestWallEvents = { workerStarts: 0, portCallbacks: 0 }; + let longestBlockingWindow = { wallMs: 0, cpuMs: 0, resource: "" }; + let probe: ReturnType; + const callbacks = new Map(); + const recordBlocking = ( + blockingMs: number, + window: { wallMs: number; cpuMs: number; resource: string }, + ) => { + if (blockingMs > longestBlocking) { + longestBlocking = blockingMs; + longestBlockingWindow = window; + } + }; + let longestGc = 0; + const observeGc = (entries: readonly import("node:perf_hooks").PerformanceEntry[]) => { + for (const entry of entries) { + if (entry.startTime >= startedAt) longestGc = Math.max(longestGc, entry.duration); + } + }; + const gc = new PerformanceObserver((list) => observeGc(list.getEntries())); + gc.observe({ entryTypes: ["gc"] }); + // Measure work within callbacks, not CPU-counter increments attributed to + // the idle interval between them. Exclude this probe's own timer callback. + const activity = createHook({ + init(_id, type) { + if (type === "WORKER") workerStarts++; + }, + before(id) { + const resource = executionAsyncResource(); + if (resource instanceof MessagePort) portCallbacks++; + if (resource === probe) return; + callbacks.set(id, { + wall: performance.now(), + cpu: threadCpuUsage(), + resource: Object.prototype.toString.call(resource), + }); + }, + after(id) { + const start = callbacks.get(id); + if (!start) return; + callbacks.delete(id); + const wallMs = performance.now() - start.wall; + const { user, system } = threadCpuUsage(start.cpu); + const cpuMs = (user + system) / 1_000; + // A callback prevents the loop from serving IO for its entire span. + // Use the high-resolution clock, not Windows' coarse CPU increments. + recordBlocking(wallMs, { wallMs, cpuMs, resource: start.resource }); + }, + }).enable(); + const sample = () => { + const now = performance.now(); + const currentCpu = threadCpuUsage(); + const wallMs = now - previous; + const cpuMs = (currentCpu.user - cpu.user + currentCpu.system - cpu.system) / 1_000; + // Native worker construction and message deserialization can precede + // async_hooks.before. Include their interval when those events occurred. + if (workerStarts || portCallbacks) { + recordBlocking(Math.min(wallMs, cpuMs), { + wallMs, + cpuMs, + resource: "worker/port interval", + }); + } + if (wallMs > longestWall) { + longestWall = wallMs; + longestWallCpu = cpuMs; + longestWallEvents = { workerStarts, portCallbacks }; + } + workerStarts = 0; + portCallbacks = 0; + previous = now; + cpu = currentCpu; + }; + // Sleep between probes: a continuously runnable setImmediate loop consumes + // a core while the worker compiles. Raw gaps remain diagnostics; the + // assertion measures elapsed daemon-thread callback spans and GC pauses. + probe = setInterval(sample, 1); + try { + // The caller's current callback began before the hook was enabled. + // Include synchronous work before the action's first await explicitly. + const actionStartedAt = performance.now(); + const actionCpu = threadCpuUsage(); + const pending = action(); + const actionWallMs = performance.now() - actionStartedAt; + const actionCost = threadCpuUsage(actionCpu); + recordBlocking(actionWallMs, { + wallMs: actionWallMs, + cpuMs: (actionCost.user + actionCost.system) / 1_000, + resource: "action()", + }); + await pending; + await new Promise((resolve) => setImmediate(resolve)); + sample(); + observeGc(gc.takeRecords()); + } finally { + clearInterval(probe); + activity.disable(); + gc.disconnect(); + } + longestBlocking = Math.max(longestBlocking, longestGc); + process.stderr.write( + `plugin compile main-thread blocking: ${longestBlocking.toFixed(2)} ms ${JSON.stringify({ longestWall, longestWallCpu, ...longestWallEvents, longestBlockingWindow, longestGc })}\n`, + ); + assert.ok(service.catalog()[0]?.clientBundle.includes("compiled fixture")); + results.push({ blockingMs: longestBlocking }); + } + // Evaluate after both probes so parsing the returned code is not charged to compilation. + const factory = globalThis.eval(service.catalog()[0]!.clientBundle) as ( + require: (specifier: string) => never, + ) => { default: () => () => string }; + const result = factory((specifier) => { + throw new Error(`Unexpected fixture dependency: ${specifier}`); + }).default()(); + assert.equal(result, "compiled fixture"); +} finally { + await service.stopAllPlugins(); +} +process.stdout.write(JSON.stringify(results));