feat: integrate mobile work chat and Gitea delivery
This commit is contained in:
sai karthik
@@ -1,3 +1,6 @@
|
||||
import { exec, execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import { api } from "@code/backend/convex/_generated/api";
|
||||
import type { Id } from "@code/backend/convex/_generated/dataModel";
|
||||
import { parseAgentEnv } from "@code/env/agent";
|
||||
@@ -9,6 +12,64 @@ import {
|
||||
createGiteaHttpTransport,
|
||||
runPostRunGiteaLifecycle,
|
||||
} from "../git/gitea";
|
||||
import { AgentOsSandboxApi } from "../sandboxes/agent-os";
|
||||
import {
|
||||
clonePublicRepository,
|
||||
mirrorSandboxToHostCheckout,
|
||||
} from "../sandboxes/host-repository-bridge";
|
||||
|
||||
const execAsync = promisify(exec);
|
||||
const execFileAsync = promisify(execFile);
|
||||
const GIT_OUTPUT_LIMIT_BYTES = 8 * 1024 * 1024;
|
||||
|
||||
const createHostGitRunner = () => ({
|
||||
async run(
|
||||
command: string,
|
||||
options?: { cwd?: string; env?: Record<string, string> }
|
||||
) {
|
||||
try {
|
||||
const result = await execAsync(command, {
|
||||
cwd: options?.cwd,
|
||||
env: options?.env,
|
||||
maxBuffer: GIT_OUTPUT_LIMIT_BYTES,
|
||||
});
|
||||
return {
|
||||
exitCode: 0,
|
||||
stderr: String(result.stderr),
|
||||
stdout: String(result.stdout),
|
||||
};
|
||||
} catch (error) {
|
||||
const failure = error as Error & {
|
||||
readonly code?: number;
|
||||
readonly stderr?: string;
|
||||
readonly stdout?: string;
|
||||
};
|
||||
return {
|
||||
exitCode:
|
||||
typeof failure.code === "number" && failure.code > 0
|
||||
? failure.code
|
||||
: 1,
|
||||
stderr: failure.stderr ?? failure.message,
|
||||
stdout: failure.stdout ?? "",
|
||||
};
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const makeAuthenticatedRepositoryUrl = (
|
||||
repositoryUrl: string,
|
||||
repositoryPath: string,
|
||||
token: string
|
||||
): string => {
|
||||
const [owner] = repositoryPath.split("/");
|
||||
if (!owner) {
|
||||
throw new Error("Gitea repository path has no owner");
|
||||
}
|
||||
const url = new URL(repositoryUrl);
|
||||
url.username = owner;
|
||||
url.password = token;
|
||||
return url.toString();
|
||||
};
|
||||
|
||||
const pullRequestOutput = v.object({
|
||||
baseBranch: v.string(),
|
||||
@@ -49,7 +110,7 @@ export const createFinalizeGiteaLifecycle = (
|
||||
}),
|
||||
name: "finalize_gitea_lifecycle",
|
||||
output,
|
||||
async run({ harness, input, log }) {
|
||||
async run({ input, log }) {
|
||||
const context = await client.query(api.agentWorkspace.get, {
|
||||
issueId,
|
||||
token: env.FLUE_DB_TOKEN,
|
||||
@@ -65,21 +126,49 @@ export const createFinalizeGiteaLifecycle = (
|
||||
"GITEA_TOKEN is required for Gitea pull request creation"
|
||||
);
|
||||
}
|
||||
if (!context.run) {
|
||||
throw new Error("Project work run is not initialized");
|
||||
}
|
||||
|
||||
const runner = {
|
||||
async run(
|
||||
command: string,
|
||||
options?: { cwd?: string; env?: Record<string, string> }
|
||||
) {
|
||||
return await harness.shell(command, options);
|
||||
},
|
||||
};
|
||||
const sandbox = new AgentOsSandboxApi(
|
||||
client,
|
||||
env.DAEMON_ID,
|
||||
context.run.actorKey
|
||||
);
|
||||
const checkout = await clonePublicRepository({
|
||||
branchName:
|
||||
context.run.branchName ?? `work/issue-${context.issue.number}`,
|
||||
repositoryUrl: context.source.url,
|
||||
});
|
||||
const runner = createHostGitRunner();
|
||||
const transport = createGiteaHttpTransport({
|
||||
baseUrl: env.GITEA_URL,
|
||||
token: env.GITEA_TOKEN,
|
||||
});
|
||||
|
||||
try {
|
||||
await mirrorSandboxToHostCheckout({
|
||||
checkoutDirectory: checkout.checkoutDirectory,
|
||||
sandbox,
|
||||
sandboxDirectory: context.run.checkoutPath ?? "/workspace/repository",
|
||||
});
|
||||
await execFileAsync(
|
||||
"git",
|
||||
[
|
||||
"remote",
|
||||
"set-url",
|
||||
"origin",
|
||||
makeAuthenticatedRepositoryUrl(
|
||||
context.source.url,
|
||||
context.source.repositoryPath,
|
||||
env.GITEA_TOKEN
|
||||
),
|
||||
],
|
||||
{
|
||||
cwd: checkout.checkoutDirectory,
|
||||
maxBuffer: GIT_OUTPUT_LIMIT_BYTES,
|
||||
}
|
||||
);
|
||||
const result = await runPostRunGiteaLifecycle({
|
||||
baseBranch: context.source.defaultBranch,
|
||||
body: `Verified changes for project issue #${context.issue.number}. Merge remains a manual review action.`,
|
||||
@@ -91,7 +180,7 @@ export const createFinalizeGiteaLifecycle = (
|
||||
title: `Issue #${context.issue.number}: ${context.issue.title}`,
|
||||
transport,
|
||||
verification: input.verified ? "passed" : "failed",
|
||||
workspace: "/workspace/repository",
|
||||
workspace: checkout.checkoutDirectory,
|
||||
});
|
||||
await client.mutation(api.agentWorkspace.recordGiteaLifecycle, {
|
||||
baseBranch: result.baseBranch,
|
||||
@@ -113,7 +202,7 @@ export const createFinalizeGiteaLifecycle = (
|
||||
return result;
|
||||
} catch (error) {
|
||||
const branchResult = await runner.run("git branch --show-current", {
|
||||
cwd: "/workspace/repository",
|
||||
cwd: checkout.checkoutDirectory,
|
||||
});
|
||||
const branch = branchResult.stdout.trim() || "unknown";
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
@@ -126,6 +215,8 @@ export const createFinalizeGiteaLifecycle = (
|
||||
token: env.FLUE_DB_TOKEN,
|
||||
});
|
||||
throw error;
|
||||
} finally {
|
||||
await checkout.cleanup();
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
@@ -22,7 +22,7 @@ export default defineAgent(({ env, id }) => {
|
||||
|
||||
Start every run by calling lookup_issue_context, reading /workspace/control/issue.md, the canonical context files under /workspace/control/context, and the operational artifacts under /workspace/control/artifacts. Call report_work_status with working before editing.
|
||||
|
||||
Work only on the bound issue. The repository checkout is the current working directory; inspect existing source files before changing them. If a missing product decision makes safe progress impossible, publish the current work.md and steps.md, report needs-input, then ask one focused question. Otherwise implement the complete issue, run the relevant install, edit, test, or scenario command in AgentOS, and preserve command evidence.
|
||||
Work only on the bound issue. The repository checkout is the current working directory; inspect existing source files before changing them. Git metadata stays on the trusted host bridge, so do not run git commands inside AgentOS—the final lifecycle action mirrors verified files into the host checkout before committing and pushing. If a missing product decision makes safe progress impossible, publish the current work.md and steps.md, report needs-input, then ask one focused question. Otherwise implement the complete issue, run the relevant install, edit, test, or scenario command in AgentOS, and preserve command evidence.
|
||||
|
||||
Before finishing, update the operational files under /workspace/control/artifacts and publish each changed canonical artifact with publish_project_artifact. After verification, call finalize_gitea_lifecycle with verified=true; it owns the post-run Git/Gitea lifecycle and never merges. Report completed only after that action succeeds or reports no_changes. On an unrecoverable error, report failed with the exact blocker. Never claim a repository change or command result you did not observe.`,
|
||||
model: `${AGENT_MODEL_PROVIDER}/${AGENT_MODEL_NAME}`,
|
||||
|
||||
@@ -105,7 +105,7 @@ describe("Gitea lifecycle adapter", () => {
|
||||
"git add --all && git commit -m 'feat(issue-5): Publish verified changes'",
|
||||
"git rev-parse HEAD",
|
||||
"git status --porcelain=v1",
|
||||
"git push 'ssh://git@git.openputer.com:2222/puter/zopu-code.git' HEAD:'work/issue-5'",
|
||||
"git push origin HEAD:'work/issue-5'",
|
||||
]);
|
||||
expect(pullRequests).toHaveLength(1);
|
||||
});
|
||||
|
||||
@@ -248,7 +248,7 @@ export const runPostRunGiteaLifecycle = async (
|
||||
};
|
||||
}
|
||||
|
||||
const repository = await input.transport.getRepository(input.repositoryPath);
|
||||
await input.transport.getRepository(input.repositoryPath);
|
||||
|
||||
await runChecked(
|
||||
input.runner,
|
||||
@@ -276,7 +276,7 @@ export const runPostRunGiteaLifecycle = async (
|
||||
|
||||
await runChecked(
|
||||
input.runner,
|
||||
`git push ${shellQuote(repository.sshUrl)} HEAD:${shellQuote(inspection.branch)}`,
|
||||
`git push origin HEAD:${shellQuote(inspection.branch)}`,
|
||||
{ cwd: input.workspace }
|
||||
);
|
||||
|
||||
|
||||
@@ -11,6 +11,11 @@ import { ConvexHttpClient } from "convex/browser";
|
||||
import { Effect } from "effect";
|
||||
import * as v from "valibot";
|
||||
|
||||
import {
|
||||
clonePublicRepository,
|
||||
mirrorHostCheckoutToSandbox,
|
||||
} from "./host-repository-bridge";
|
||||
|
||||
const execResultSchema = v.object({
|
||||
exitCode: v.number(),
|
||||
stderr: v.string(),
|
||||
@@ -29,7 +34,7 @@ interface ExecuteOptions {
|
||||
readonly timeoutMs?: number;
|
||||
}
|
||||
|
||||
class AgentOsSandboxApi implements SandboxApi {
|
||||
export class AgentOsSandboxApi implements SandboxApi {
|
||||
readonly #actorKey: string[];
|
||||
readonly #client: ConvexHttpClient;
|
||||
readonly #daemonId: string;
|
||||
@@ -258,13 +263,27 @@ export const agentOs = (
|
||||
if (!checkoutOperation || checkoutOperation._tag !== "Exec") {
|
||||
throw new Error("Issue workspace plan must include a checkout command");
|
||||
}
|
||||
const checkoutCommandResult = await sandbox.exec(
|
||||
checkoutOperation.command,
|
||||
{
|
||||
cwd: checkoutOperation.cwd,
|
||||
timeoutMs: checkoutOperation.timeoutMs,
|
||||
const hostCheckout = await clonePublicRepository({
|
||||
branchName: plan.branchName,
|
||||
repositoryUrl: plan.sourceUrl,
|
||||
timeoutMs: checkoutOperation.timeoutMs,
|
||||
});
|
||||
try {
|
||||
if (!(await sandbox.exists(plan.checkoutPath))) {
|
||||
await mirrorHostCheckoutToSandbox({
|
||||
checkoutDirectory: hostCheckout.checkoutDirectory,
|
||||
sandbox,
|
||||
sandboxDirectory: plan.checkoutPath,
|
||||
});
|
||||
}
|
||||
);
|
||||
} finally {
|
||||
await hostCheckout.cleanup();
|
||||
}
|
||||
const checkoutCommandResult = {
|
||||
exitCode: 0,
|
||||
stderr: "",
|
||||
stdout: `${plan.branchName} ${hostCheckout.headSha}\n`,
|
||||
};
|
||||
|
||||
await Promise.all(
|
||||
stagingOperations
|
||||
|
||||
297
packages/agents/src/sandboxes/host-repository-bridge.test.ts
Normal file
297
packages/agents/src/sandboxes/host-repository-bridge.test.ts
Normal file
@@ -0,0 +1,297 @@
|
||||
import {
|
||||
lstat,
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
rm,
|
||||
stat,
|
||||
symlink,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import nodePath from "node:path";
|
||||
|
||||
import type { FileStat, SandboxApi } from "@flue/runtime";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
mirrorHostCheckoutToSandbox,
|
||||
mirrorSandboxToHostCheckout,
|
||||
parseGitCommitSha,
|
||||
parsePublicGitUrl,
|
||||
} from "./host-repository-bridge";
|
||||
|
||||
class FakeSandbox implements SandboxApi {
|
||||
readonly files = new Map<string, Uint8Array>();
|
||||
readonly directories = new Set<string>(["/"]);
|
||||
|
||||
async readFile(path: string): Promise<string> {
|
||||
return new TextDecoder().decode(await this.readFileBuffer(path));
|
||||
}
|
||||
|
||||
readFileBuffer(path: string): Promise<Uint8Array> {
|
||||
const content = this.files.get(path);
|
||||
if (!content) {
|
||||
throw new Error(`Missing fake sandbox file: ${path}`);
|
||||
}
|
||||
return Promise.resolve(Uint8Array.from(content));
|
||||
}
|
||||
|
||||
writeFile(path: string, content: string | Uint8Array): Promise<void> {
|
||||
this.files.set(
|
||||
path,
|
||||
typeof content === "string"
|
||||
? new TextEncoder().encode(content)
|
||||
: Uint8Array.from(content)
|
||||
);
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
stat(path: string): Promise<FileStat> {
|
||||
const content = this.files.get(path);
|
||||
if (content) {
|
||||
return Promise.resolve({
|
||||
isDirectory: false,
|
||||
isFile: true,
|
||||
size: content.byteLength,
|
||||
});
|
||||
}
|
||||
if (this.directories.has(path)) {
|
||||
return Promise.resolve({ isDirectory: true, isFile: false });
|
||||
}
|
||||
throw new Error(`Missing fake sandbox path: ${path}`);
|
||||
}
|
||||
|
||||
readdir(path: string): Promise<string[]> {
|
||||
const prefix = path === "/" ? "/" : `${path}/`;
|
||||
const entries = new Set<string>();
|
||||
for (const directory of this.directories) {
|
||||
if (directory.startsWith(prefix)) {
|
||||
const [entry] = directory.slice(prefix.length).split("/");
|
||||
if (entry) {
|
||||
entries.add(entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const file of this.files.keys()) {
|
||||
if (file.startsWith(prefix)) {
|
||||
const [entry] = file.slice(prefix.length).split("/");
|
||||
if (entry) {
|
||||
entries.add(entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
return Promise.resolve([...entries]);
|
||||
}
|
||||
|
||||
exists(path: string): Promise<boolean> {
|
||||
return Promise.resolve(this.files.has(path) || this.directories.has(path));
|
||||
}
|
||||
|
||||
mkdir(
|
||||
path: string,
|
||||
options?: { readonly recursive?: boolean }
|
||||
): Promise<void> {
|
||||
if (!options?.recursive) {
|
||||
this.directories.add(path);
|
||||
return Promise.resolve();
|
||||
}
|
||||
const parts = path.split("/").filter(Boolean);
|
||||
let current = "";
|
||||
for (const part of parts) {
|
||||
current += `/${part}`;
|
||||
this.directories.add(current);
|
||||
}
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
rm(
|
||||
path: string,
|
||||
options?: { readonly force?: boolean; readonly recursive?: boolean }
|
||||
): Promise<void> {
|
||||
const prefix = `${path}/`;
|
||||
if (options?.recursive) {
|
||||
for (const file of this.files.keys()) {
|
||||
if (file === path || file.startsWith(prefix)) {
|
||||
this.files.delete(file);
|
||||
}
|
||||
}
|
||||
for (const directory of this.directories) {
|
||||
if (directory === path || directory.startsWith(prefix)) {
|
||||
this.directories.delete(directory);
|
||||
}
|
||||
}
|
||||
return Promise.resolve();
|
||||
}
|
||||
this.files.delete(path);
|
||||
this.directories.delete(path);
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
exec(): Promise<{
|
||||
exitCode: number;
|
||||
stderr: string;
|
||||
stdout: string;
|
||||
}> {
|
||||
void this.directories;
|
||||
return Promise.resolve({ exitCode: 0, stderr: "", stdout: "" });
|
||||
}
|
||||
}
|
||||
|
||||
const temporaryDirectories: string[] = [];
|
||||
|
||||
const makeCheckout = async (): Promise<string> => {
|
||||
const directory = await mkdtemp(nodePath.join(tmpdir(), "host-bridge-test-"));
|
||||
temporaryDirectories.push(directory);
|
||||
await mkdir(nodePath.join(directory, ".git"), { recursive: true });
|
||||
await writeFile(nodePath.join(directory, ".git", "keep"), "git metadata");
|
||||
return directory;
|
||||
};
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
temporaryDirectories
|
||||
.splice(0)
|
||||
.map((directory) => rm(directory, { force: true, recursive: true }))
|
||||
);
|
||||
});
|
||||
|
||||
describe("parsePublicGitUrl", () => {
|
||||
it("accepts public HTTP(S) repositories", () => {
|
||||
expect(
|
||||
parsePublicGitUrl("https://git.openputer.com/team/repository.git")
|
||||
.hostname
|
||||
).toBe("git.openputer.com");
|
||||
});
|
||||
|
||||
it.each([
|
||||
"file:///tmp/repository",
|
||||
"ssh://git@example.com/team/repository.git",
|
||||
"https://token@example.com/team/repository.git",
|
||||
"http://127.0.0.1/repository.git",
|
||||
"http://192.168.1.10/repository.git",
|
||||
])("rejects non-public repository URL %s", (repositoryUrl) => {
|
||||
expect(() => parsePublicGitUrl(repositoryUrl)).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseGitCommitSha", () => {
|
||||
it("normalizes a full SHA returned by git", () => {
|
||||
const sha = "a".repeat(40);
|
||||
expect(parseGitCommitSha(` ${sha}\n`)).toBe(sha);
|
||||
});
|
||||
|
||||
it.each(["", "abc123", "g".repeat(40), "a".repeat(41)])(
|
||||
"rejects invalid commit SHA %s",
|
||||
(sha) => {
|
||||
expect(() => parseGitCommitSha(sha)).toThrow("invalid HEAD commit SHA");
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
describe("repository mirroring", () => {
|
||||
it("copies host files into a sandbox without copying .git", async () => {
|
||||
const checkoutDirectory = await makeCheckout();
|
||||
await mkdir(nodePath.join(checkoutDirectory, "src"), {
|
||||
recursive: true,
|
||||
});
|
||||
await writeFile(nodePath.join(checkoutDirectory, "README.md"), "hello");
|
||||
await writeFile(
|
||||
nodePath.join(checkoutDirectory, "src", "binary.bin"),
|
||||
Uint8Array.from([0, 255, 1])
|
||||
);
|
||||
const sandbox = new FakeSandbox();
|
||||
|
||||
await mirrorHostCheckoutToSandbox({
|
||||
checkoutDirectory,
|
||||
sandbox,
|
||||
sandboxDirectory: "/workspace/repository",
|
||||
});
|
||||
|
||||
expect(
|
||||
new TextDecoder().decode(
|
||||
sandbox.files.get("/workspace/repository/README.md")
|
||||
)
|
||||
).toBe("hello");
|
||||
expect(sandbox.files.get("/workspace/repository/src/binary.bin")).toEqual(
|
||||
Uint8Array.from([0, 255, 1])
|
||||
);
|
||||
expect(
|
||||
[...sandbox.files.keys()].some((filePath) => filePath.includes("/.git/"))
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("stages sandbox files, removes stale files, and preserves .git", async () => {
|
||||
const checkoutDirectory = await makeCheckout();
|
||||
await writeFile(nodePath.join(checkoutDirectory, "stale.txt"), "delete me");
|
||||
const executablePath = nodePath.join(checkoutDirectory, "run.sh");
|
||||
await writeFile(executablePath, "#!/bin/sh\nexit 0\n", { mode: 0o755 });
|
||||
const sandbox = new FakeSandbox();
|
||||
await sandbox.mkdir("/workspace/repository/src", { recursive: true });
|
||||
await sandbox.writeFile(
|
||||
"/workspace/repository/run.sh",
|
||||
"#!/bin/sh\necho changed\n"
|
||||
);
|
||||
await sandbox.writeFile(
|
||||
"/workspace/repository/src/index.ts",
|
||||
"export const value = 1;\n"
|
||||
);
|
||||
|
||||
await mirrorSandboxToHostCheckout({
|
||||
checkoutDirectory,
|
||||
sandbox,
|
||||
sandboxDirectory: "/workspace/repository",
|
||||
});
|
||||
|
||||
await expect(
|
||||
readFile(nodePath.join(checkoutDirectory, "stale.txt"))
|
||||
).rejects.toThrow();
|
||||
await expect(
|
||||
readFile(nodePath.join(checkoutDirectory, ".git", "keep"), "utf-8")
|
||||
).resolves.toBe("git metadata");
|
||||
await expect(
|
||||
readFile(nodePath.join(checkoutDirectory, "src", "index.ts"), "utf-8")
|
||||
).resolves.toBe("export const value = 1;\n");
|
||||
const executableMetadata = await stat(executablePath);
|
||||
expect(executableMetadata.mode % 0o1000).toBe(0o755);
|
||||
});
|
||||
|
||||
it("enforces transfer limits before modifying the sandbox", async () => {
|
||||
const checkoutDirectory = await makeCheckout();
|
||||
await writeFile(nodePath.join(checkoutDirectory, "one.txt"), "1");
|
||||
await writeFile(nodePath.join(checkoutDirectory, "two.txt"), "2");
|
||||
const sandbox = new FakeSandbox();
|
||||
|
||||
await expect(
|
||||
mirrorHostCheckoutToSandbox({
|
||||
checkoutDirectory,
|
||||
limits: { maxFiles: 1 },
|
||||
sandbox,
|
||||
sandboxDirectory: "/workspace/repository",
|
||||
})
|
||||
).rejects.toThrow("limit is 1");
|
||||
expect(sandbox.files.size).toBe(0);
|
||||
});
|
||||
|
||||
it("rejects host symbolic links", async () => {
|
||||
const checkoutDirectory = await makeCheckout();
|
||||
await writeFile(nodePath.join(checkoutDirectory, "outside.txt"), "outside");
|
||||
await symlink(
|
||||
"outside.txt",
|
||||
nodePath.join(checkoutDirectory, "linked.txt")
|
||||
);
|
||||
const sandbox = new FakeSandbox();
|
||||
|
||||
await expect(
|
||||
mirrorHostCheckoutToSandbox({
|
||||
checkoutDirectory,
|
||||
sandbox,
|
||||
sandboxDirectory: "/workspace/repository",
|
||||
})
|
||||
).rejects.toThrow("Symbolic links are not supported");
|
||||
const linkedMetadata = await lstat(
|
||||
nodePath.join(checkoutDirectory, "linked.txt")
|
||||
);
|
||||
expect(linkedMetadata.isSymbolicLink()).toBe(true);
|
||||
});
|
||||
});
|
||||
506
packages/agents/src/sandboxes/host-repository-bridge.ts
Normal file
506
packages/agents/src/sandboxes/host-repository-bridge.ts
Normal file
@@ -0,0 +1,506 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import {
|
||||
chmod,
|
||||
lstat,
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readdir,
|
||||
readFile,
|
||||
rename,
|
||||
rm,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { isIP } from "node:net";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import type { SandboxApi } from "@flue/runtime";
|
||||
|
||||
const DEFAULT_GIT_TIMEOUT_MS = 60_000;
|
||||
const GIT_OUTPUT_LIMIT_BYTES = 1_048_576;
|
||||
|
||||
export interface RepositoryMirrorLimits {
|
||||
readonly maxDepth: number;
|
||||
readonly maxFileBytes: number;
|
||||
readonly maxFiles: number;
|
||||
readonly maxTotalBytes: number;
|
||||
}
|
||||
|
||||
export const DEFAULT_REPOSITORY_MIRROR_LIMITS: RepositoryMirrorLimits = {
|
||||
maxDepth: 64,
|
||||
maxFileBytes: 10 * 1024 * 1024,
|
||||
maxFiles: 10_000,
|
||||
maxTotalBytes: 100 * 1024 * 1024,
|
||||
};
|
||||
|
||||
export interface HostRepositoryCheckout {
|
||||
readonly branchName: string;
|
||||
readonly checkoutDirectory: string;
|
||||
readonly headSha: string;
|
||||
readonly temporaryDirectory: string;
|
||||
readonly cleanup: () => Promise<void>;
|
||||
}
|
||||
|
||||
interface MirrorFile {
|
||||
readonly mode?: number;
|
||||
readonly path: string;
|
||||
readonly size: number;
|
||||
}
|
||||
|
||||
interface GitResult {
|
||||
readonly stderr: string;
|
||||
readonly stdout: string;
|
||||
}
|
||||
|
||||
const gitError = (
|
||||
args: readonly string[],
|
||||
error: Error & { readonly stderr?: string }
|
||||
): Error => {
|
||||
const detail = error.stderr?.trim() || error.message;
|
||||
return new Error(`git ${args.join(" ")} failed: ${detail}`, {
|
||||
cause: error,
|
||||
});
|
||||
};
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
const runGit = async (
|
||||
args: readonly string[],
|
||||
options: { readonly cwd?: string; readonly timeoutMs: number }
|
||||
): Promise<GitResult> => {
|
||||
try {
|
||||
const result = await execFileAsync("git", [...args], {
|
||||
cwd: options.cwd,
|
||||
maxBuffer: GIT_OUTPUT_LIMIT_BYTES,
|
||||
timeout: options.timeoutMs,
|
||||
});
|
||||
return {
|
||||
stderr: String(result.stderr),
|
||||
stdout: String(result.stdout),
|
||||
};
|
||||
} catch (error) {
|
||||
if (!(error instanceof Error)) {
|
||||
throw error;
|
||||
}
|
||||
throw gitError(
|
||||
args,
|
||||
error as Error & {
|
||||
readonly stderr?: string;
|
||||
}
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const isPrivateIpv4 = (hostname: string): boolean => {
|
||||
const octets = hostname.split(".").map(Number);
|
||||
const [first = -1, second = -1] = octets;
|
||||
return (
|
||||
first === 0 ||
|
||||
first === 10 ||
|
||||
first === 127 ||
|
||||
(first === 169 && second === 254) ||
|
||||
(first === 172 && second >= 16 && second <= 31) ||
|
||||
(first === 192 && second === 168)
|
||||
);
|
||||
};
|
||||
|
||||
const isPrivateIpv6 = (hostname: string): boolean => {
|
||||
const normalized = hostname
|
||||
.toLowerCase()
|
||||
.replaceAll("[", "")
|
||||
.replaceAll("]", "");
|
||||
return (
|
||||
normalized === "::" ||
|
||||
normalized === "::1" ||
|
||||
normalized.startsWith("fc") ||
|
||||
normalized.startsWith("fd") ||
|
||||
normalized.startsWith("fe8") ||
|
||||
normalized.startsWith("fe9") ||
|
||||
normalized.startsWith("fea") ||
|
||||
normalized.startsWith("feb")
|
||||
);
|
||||
};
|
||||
|
||||
export const parsePublicGitUrl = (repositoryUrl: string): URL => {
|
||||
const url = new URL(repositoryUrl);
|
||||
if (url.protocol !== "https:" && url.protocol !== "http:") {
|
||||
throw new Error("Public Git repositories must use an HTTP(S) URL");
|
||||
}
|
||||
if (url.username || url.password) {
|
||||
throw new Error("Public Git repository URLs must not contain credentials");
|
||||
}
|
||||
|
||||
const hostname = url.hostname.toLowerCase();
|
||||
const addressKind = isIP(hostname);
|
||||
const isPrivateAddress =
|
||||
(addressKind === 4 && isPrivateIpv4(hostname)) ||
|
||||
(addressKind === 6 && isPrivateIpv6(hostname));
|
||||
if (
|
||||
hostname === "localhost" ||
|
||||
hostname.endsWith(".localhost") ||
|
||||
hostname.endsWith(".local") ||
|
||||
isPrivateAddress
|
||||
) {
|
||||
throw new Error("Public Git repository URLs must use a public host");
|
||||
}
|
||||
return url;
|
||||
};
|
||||
|
||||
export const parseGitCommitSha = (value: string): string => {
|
||||
const sha = value.trim();
|
||||
if (!/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/u.test(sha)) {
|
||||
throw new Error("Git returned an invalid HEAD commit SHA");
|
||||
}
|
||||
return sha;
|
||||
};
|
||||
|
||||
export const clonePublicRepository = async (options: {
|
||||
readonly branchName: string;
|
||||
readonly repositoryUrl: string;
|
||||
readonly timeoutMs?: number;
|
||||
}): Promise<HostRepositoryCheckout> => {
|
||||
const timeoutMs = options.timeoutMs ?? DEFAULT_GIT_TIMEOUT_MS;
|
||||
if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) {
|
||||
throw new Error("Git timeout must be a positive integer");
|
||||
}
|
||||
|
||||
const repositoryUrl = parsePublicGitUrl(options.repositoryUrl).toString();
|
||||
await runGit(["check-ref-format", "--branch", options.branchName], {
|
||||
timeoutMs,
|
||||
});
|
||||
|
||||
const temporaryDirectory = await mkdtemp(
|
||||
path.join(tmpdir(), "zopu-host-repository-")
|
||||
);
|
||||
const checkoutDirectory = path.join(temporaryDirectory, "repository");
|
||||
let headSha = "";
|
||||
|
||||
try {
|
||||
await runGit(
|
||||
[
|
||||
"clone",
|
||||
"--depth",
|
||||
"1",
|
||||
"--single-branch",
|
||||
"--no-tags",
|
||||
"--",
|
||||
repositoryUrl,
|
||||
checkoutDirectory,
|
||||
],
|
||||
{ timeoutMs }
|
||||
);
|
||||
await runGit(["switch", "-c", options.branchName], {
|
||||
cwd: checkoutDirectory,
|
||||
timeoutMs,
|
||||
});
|
||||
const headResult = await runGit(["rev-parse", "HEAD"], {
|
||||
cwd: checkoutDirectory,
|
||||
timeoutMs,
|
||||
});
|
||||
headSha = parseGitCommitSha(headResult.stdout);
|
||||
} catch (error) {
|
||||
await rm(temporaryDirectory, { force: true, recursive: true });
|
||||
throw error;
|
||||
}
|
||||
|
||||
return {
|
||||
branchName: options.branchName,
|
||||
checkoutDirectory,
|
||||
cleanup: () =>
|
||||
rm(temporaryDirectory, {
|
||||
force: true,
|
||||
recursive: true,
|
||||
}),
|
||||
headSha,
|
||||
temporaryDirectory,
|
||||
};
|
||||
};
|
||||
|
||||
const resolveLimits = (
|
||||
limits: Partial<RepositoryMirrorLimits> | undefined
|
||||
): RepositoryMirrorLimits => {
|
||||
const resolvedLimits = {
|
||||
...DEFAULT_REPOSITORY_MIRROR_LIMITS,
|
||||
...limits,
|
||||
};
|
||||
for (const [name, value] of Object.entries(resolvedLimits)) {
|
||||
if (!Number.isSafeInteger(value) || value <= 0) {
|
||||
throw new Error(`${name} must be a positive integer`);
|
||||
}
|
||||
}
|
||||
return resolvedLimits;
|
||||
};
|
||||
|
||||
const assertSafeEntryName = (name: string): void => {
|
||||
if (
|
||||
!name ||
|
||||
name === "." ||
|
||||
name === ".." ||
|
||||
name.includes("/") ||
|
||||
name.includes("\\")
|
||||
) {
|
||||
throw new Error(`Sandbox returned an unsafe directory entry: ${name}`);
|
||||
}
|
||||
};
|
||||
|
||||
const assertWithinCheckout = (checkoutDirectory: string): string => {
|
||||
const absoluteCheckout = path.resolve(checkoutDirectory);
|
||||
if (absoluteCheckout === path.resolve(path.sep)) {
|
||||
throw new Error("Repository checkout cannot be the filesystem root");
|
||||
}
|
||||
return absoluteCheckout;
|
||||
};
|
||||
|
||||
const assertAbsoluteSandboxDirectory = (sandboxDirectory: string): string => {
|
||||
if (!path.posix.isAbsolute(sandboxDirectory)) {
|
||||
throw new Error("Sandbox directory must be absolute");
|
||||
}
|
||||
return path.posix.normalize(sandboxDirectory);
|
||||
};
|
||||
|
||||
const enforceFileBounds = (
|
||||
files: readonly MirrorFile[],
|
||||
limits: RepositoryMirrorLimits
|
||||
): void => {
|
||||
if (files.length > limits.maxFiles) {
|
||||
throw new Error(
|
||||
`Repository contains ${files.length} files; limit is ${limits.maxFiles}`
|
||||
);
|
||||
}
|
||||
let totalBytes = 0;
|
||||
for (const file of files) {
|
||||
if (file.size > limits.maxFileBytes) {
|
||||
throw new Error(
|
||||
`${file.path} is ${file.size} bytes; per-file limit is ${limits.maxFileBytes}`
|
||||
);
|
||||
}
|
||||
totalBytes += file.size;
|
||||
if (totalBytes > limits.maxTotalBytes) {
|
||||
throw new Error(
|
||||
`Repository files exceed the ${limits.maxTotalBytes}-byte total limit`
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const listHostFiles = async (
|
||||
checkoutDirectory: string,
|
||||
limits: RepositoryMirrorLimits
|
||||
): Promise<MirrorFile[]> => {
|
||||
const files: MirrorFile[] = [];
|
||||
|
||||
const visit = async (
|
||||
directory: string,
|
||||
relativeDirectory: string
|
||||
): Promise<void> => {
|
||||
const depth = relativeDirectory
|
||||
? relativeDirectory.split(path.sep).length
|
||||
: 0;
|
||||
if (depth > limits.maxDepth) {
|
||||
throw new Error(`Repository directory depth exceeds ${limits.maxDepth}`);
|
||||
}
|
||||
|
||||
const entries = await readdir(directory, { withFileTypes: true });
|
||||
entries.sort((left, right) => left.name.localeCompare(right.name));
|
||||
for (const entry of entries) {
|
||||
if (!relativeDirectory && entry.name === ".git") {
|
||||
continue;
|
||||
}
|
||||
const relativePath = path.join(relativeDirectory, entry.name);
|
||||
const absolutePath = path.join(directory, entry.name);
|
||||
if (entry.isSymbolicLink()) {
|
||||
throw new Error(`Symbolic links are not supported: ${relativePath}`);
|
||||
}
|
||||
if (entry.isDirectory()) {
|
||||
// oxlint-disable-next-line no-await-in-loop -- deterministic bounded traversal.
|
||||
await visit(absolutePath, relativePath);
|
||||
continue;
|
||||
}
|
||||
if (!entry.isFile()) {
|
||||
throw new Error(`Unsupported repository entry: ${relativePath}`);
|
||||
}
|
||||
// oxlint-disable-next-line no-await-in-loop -- metadata is bounded before file transfer.
|
||||
const metadata = await lstat(absolutePath);
|
||||
files.push({
|
||||
mode: metadata.mode % 0o1000,
|
||||
path: relativePath.split(path.sep).join(path.posix.sep),
|
||||
size: metadata.size,
|
||||
});
|
||||
enforceFileBounds(files, limits);
|
||||
}
|
||||
};
|
||||
|
||||
await visit(checkoutDirectory, "");
|
||||
return files;
|
||||
};
|
||||
|
||||
const listSandboxFiles = async (
|
||||
sandbox: SandboxApi,
|
||||
sandboxDirectory: string,
|
||||
limits: RepositoryMirrorLimits
|
||||
): Promise<MirrorFile[]> => {
|
||||
const files: MirrorFile[] = [];
|
||||
|
||||
const visit = async (
|
||||
directory: string,
|
||||
relativeDirectory: string
|
||||
): Promise<void> => {
|
||||
const depth = relativeDirectory
|
||||
? relativeDirectory.split(path.posix.sep).length
|
||||
: 0;
|
||||
if (depth > limits.maxDepth) {
|
||||
throw new Error(`Sandbox directory depth exceeds ${limits.maxDepth}`);
|
||||
}
|
||||
|
||||
const entries = [...(await sandbox.readdir(directory))].toSorted(
|
||||
(left, right) => left.localeCompare(right)
|
||||
);
|
||||
for (const entry of entries) {
|
||||
assertSafeEntryName(entry);
|
||||
if (!relativeDirectory && entry === ".git") {
|
||||
continue;
|
||||
}
|
||||
const relativePath = path.posix.join(relativeDirectory, entry);
|
||||
const absolutePath = path.posix.join(directory, entry);
|
||||
// oxlint-disable-next-line no-await-in-loop -- sandbox metadata is read sequentially and bounded.
|
||||
const metadata = await sandbox.stat(absolutePath);
|
||||
if (metadata.isSymbolicLink) {
|
||||
throw new Error(`Symbolic links are not supported: ${relativePath}`);
|
||||
}
|
||||
if (metadata.isDirectory) {
|
||||
// oxlint-disable-next-line no-await-in-loop -- deterministic bounded traversal.
|
||||
await visit(absolutePath, relativePath);
|
||||
continue;
|
||||
}
|
||||
if (!metadata.isFile) {
|
||||
throw new Error(`Unsupported sandbox entry: ${relativePath}`);
|
||||
}
|
||||
if (metadata.size === undefined) {
|
||||
throw new Error(`Sandbox did not report a size for ${relativePath}`);
|
||||
}
|
||||
files.push({ path: relativePath, size: metadata.size });
|
||||
enforceFileBounds(files, limits);
|
||||
}
|
||||
};
|
||||
|
||||
await visit(sandboxDirectory, "");
|
||||
return files;
|
||||
};
|
||||
|
||||
export const mirrorHostCheckoutToSandbox = async (options: {
|
||||
readonly checkoutDirectory: string;
|
||||
readonly limits?: Partial<RepositoryMirrorLimits>;
|
||||
readonly sandbox: SandboxApi;
|
||||
readonly sandboxDirectory: string;
|
||||
}): Promise<void> => {
|
||||
const checkoutDirectory = assertWithinCheckout(options.checkoutDirectory);
|
||||
const sandboxDirectory = assertAbsoluteSandboxDirectory(
|
||||
options.sandboxDirectory
|
||||
);
|
||||
const limits = resolveLimits(options.limits);
|
||||
const files = await listHostFiles(checkoutDirectory, limits);
|
||||
|
||||
await options.sandbox.mkdir(sandboxDirectory, { recursive: true });
|
||||
for (const file of files) {
|
||||
const hostPath = path.join(
|
||||
checkoutDirectory,
|
||||
...file.path.split(path.posix.sep)
|
||||
);
|
||||
const sandboxPath = path.posix.join(sandboxDirectory, file.path);
|
||||
// oxlint-disable-next-line no-await-in-loop -- bounded sequential writes avoid overwhelming remote sandboxes.
|
||||
await options.sandbox.mkdir(path.posix.dirname(sandboxPath), {
|
||||
recursive: true,
|
||||
});
|
||||
// oxlint-disable-next-line no-await-in-loop -- bounded sequential transfer limits peak memory.
|
||||
const content = await readFile(hostPath);
|
||||
if (content.byteLength !== file.size) {
|
||||
throw new Error(`${file.path} changed while it was being mirrored`);
|
||||
}
|
||||
// oxlint-disable-next-line no-await-in-loop -- bounded sequential writes avoid overwhelming remote sandboxes.
|
||||
await options.sandbox.writeFile(sandboxPath, content);
|
||||
}
|
||||
};
|
||||
|
||||
const clearCheckoutFiles = async (checkoutDirectory: string): Promise<void> => {
|
||||
const entries = await readdir(checkoutDirectory);
|
||||
for (const entry of entries) {
|
||||
if (entry === ".git") {
|
||||
continue;
|
||||
}
|
||||
// oxlint-disable-next-line no-await-in-loop -- deletion is deliberately scoped to checkout children.
|
||||
await rm(path.join(checkoutDirectory, entry), {
|
||||
force: true,
|
||||
recursive: true,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
export const mirrorSandboxToHostCheckout = async (options: {
|
||||
readonly checkoutDirectory: string;
|
||||
readonly limits?: Partial<RepositoryMirrorLimits>;
|
||||
readonly sandbox: SandboxApi;
|
||||
readonly sandboxDirectory: string;
|
||||
}): Promise<void> => {
|
||||
const checkoutDirectory = assertWithinCheckout(options.checkoutDirectory);
|
||||
const sandboxDirectory = assertAbsoluteSandboxDirectory(
|
||||
options.sandboxDirectory
|
||||
);
|
||||
const limits = resolveLimits(options.limits);
|
||||
const files = await listSandboxFiles(
|
||||
options.sandbox,
|
||||
sandboxDirectory,
|
||||
limits
|
||||
);
|
||||
const originalFiles = await listHostFiles(checkoutDirectory, limits);
|
||||
const originalModes = new Map(
|
||||
originalFiles.map((file) => [file.path, file.mode])
|
||||
);
|
||||
const stagingDirectory = await mkdtemp(
|
||||
path.join(path.dirname(checkoutDirectory), ".zopu-sandbox-mirror-")
|
||||
);
|
||||
|
||||
try {
|
||||
let transferredBytes = 0;
|
||||
for (const file of files) {
|
||||
const sandboxPath = path.posix.join(sandboxDirectory, file.path);
|
||||
// oxlint-disable-next-line no-await-in-loop -- bounded sequential reads limit peak memory.
|
||||
const content = await options.sandbox.readFileBuffer(sandboxPath);
|
||||
if (content.byteLength !== file.size) {
|
||||
throw new Error(`${file.path} changed while it was being mirrored`);
|
||||
}
|
||||
transferredBytes += content.byteLength;
|
||||
if (transferredBytes > limits.maxTotalBytes) {
|
||||
throw new Error(
|
||||
`Sandbox files exceed the ${limits.maxTotalBytes}-byte total limit`
|
||||
);
|
||||
}
|
||||
|
||||
const stagedPath = path.join(
|
||||
stagingDirectory,
|
||||
...file.path.split(path.posix.sep)
|
||||
);
|
||||
// oxlint-disable-next-line no-await-in-loop -- staged writes keep the checkout intact until transfer succeeds.
|
||||
await mkdir(path.dirname(stagedPath), { recursive: true });
|
||||
// oxlint-disable-next-line no-await-in-loop -- bounded sequential writes limit peak memory.
|
||||
await writeFile(stagedPath, content);
|
||||
const originalMode = originalModes.get(file.path);
|
||||
if (originalMode !== undefined) {
|
||||
// oxlint-disable-next-line no-await-in-loop -- preserve executable bits for existing files.
|
||||
await chmod(stagedPath, originalMode);
|
||||
}
|
||||
}
|
||||
|
||||
await clearCheckoutFiles(checkoutDirectory);
|
||||
const stagedEntries = await readdir(stagingDirectory);
|
||||
for (const entry of stagedEntries) {
|
||||
// oxlint-disable-next-line no-await-in-loop -- each top-level entry is moved exactly once.
|
||||
await rename(
|
||||
path.join(stagingDirectory, entry),
|
||||
path.join(checkoutDirectory, entry)
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
await rm(stagingDirectory, { force: true, recursive: true });
|
||||
}
|
||||
};
|
||||
Reference in New Issue
Block a user