From a6bf07829bf997109dc45743293ef53939c75468 Mon Sep 17 00:00:00 2001 From: sai karthik Date: Fri, 24 Jul 2026 02:57:07 +0530 Subject: [PATCH] feat(backend): import public GitHub repositories --- packages/backend/convex/_generated/api.d.ts | 2 + packages/backend/convex/projects.ts | 34 ++++-- packages/backend/convex/publicGit.ts | 125 ++++++++++++++++++++ 3 files changed, 152 insertions(+), 9 deletions(-) create mode 100644 packages/backend/convex/publicGit.ts diff --git a/packages/backend/convex/_generated/api.d.ts b/packages/backend/convex/_generated/api.d.ts index 6e039f5..7386dfe 100644 --- a/packages/backend/convex/_generated/api.d.ts +++ b/packages/backend/convex/_generated/api.d.ts @@ -25,6 +25,7 @@ import type * as projectArtifacts from "../projectArtifacts.js"; import type * as projectIssues from "../projectIssues.js"; import type * as projectStore from "../projectStore.js"; import type * as projects from "../projects.js"; +import type * as publicGit from "../publicGit.js"; import type * as signals from "../signals.js"; import type * as todos from "../todos.js"; @@ -52,6 +53,7 @@ declare const fullApi: ApiFromModules<{ projectIssues: typeof projectIssues; projectStore: typeof projectStore; projects: typeof projects; + publicGit: typeof publicGit; signals: typeof signals; todos: typeof todos; }>; diff --git a/packages/backend/convex/projects.ts b/packages/backend/convex/projects.ts index b3881a5..c26627c 100644 --- a/packages/backend/convex/projects.ts +++ b/packages/backend/convex/projects.ts @@ -1,4 +1,5 @@ import { + decodePublicGitImportResult, preparePublicGitSource, type ProjectImportOutcome, type ProjectView, @@ -8,15 +9,17 @@ import { import { v } from "convex/values"; import { Effect } from "effect"; +import { internal } from "./_generated/api"; import type { Id } from "./_generated/dataModel"; import { action, internalMutation, query } from "./_generated/server"; import { createInitialArtifacts } from "./artifactModel"; -import { requireCurrentOrganization } from "./authz"; +import { requireAuthUserId, requireCurrentOrganization } from "./authz"; import { makeProjectMutationStore, makeProjectQueryStore, persistPublicGitImportTransaction, } from "./projectStore"; +import { inspectPublicGit } from "./publicGit"; // --------------------------------------------------------------------------- // Internal: persist a public Git import in one transaction @@ -182,18 +185,31 @@ export const get = query({ }); // --------------------------------------------------------------------------- -// Public action: import a public Git repository -// (daemon wiring added in the Daemon phase) +// Public action: import a public GitHub repository // --------------------------------------------------------------------------- export const importPublicGit = action({ args: { repositoryUrl: v.string() }, - handler: async (_ctx, args): Promise => { - // Normalize through the domain layer to validate the URL before the - // daemon adapter is wired. - await Effect.runPromise(preparePublicGitSource(args.repositoryUrl)); - // The daemon PublicGit adapter is wired in the Daemon phase. - throw new Error("PublicGit import is not yet wired to the daemon adapter"); + handler: async (ctx, args): Promise => { + const userId = await requireAuthUserId(ctx); + const source = await Effect.runPromise( + preparePublicGitSource(args.repositoryUrl) + ); + const remote = await inspectPublicGit(source); + const validatedRemote = await Effect.runPromise( + decodePublicGitImportResult(remote) + ); + return await ctx.runMutation(internal.projects.persistPublicGitImport, { + remote: { + defaultBranch: validatedRemote.defaultBranch, + documents: validatedRemote.documents.map((document) => ({ + ...document, + })), + warnings: validatedRemote.warnings.map((warning) => ({ ...warning })), + }, + source, + userId, + }); }, }); diff --git a/packages/backend/convex/publicGit.ts b/packages/backend/convex/publicGit.ts new file mode 100644 index 0000000..eaebe66 --- /dev/null +++ b/packages/backend/convex/publicGit.ts @@ -0,0 +1,125 @@ +import { + CONTEXT_KINDS, + contextPathForKind, + MAX_CONTEXT_DOCUMENT_CHARACTERS, + type PreparedPublicGitSource, + type PublicGitImportResult, + type RepositoryContextDocument, +} from "@code/primitives/project"; + +const GITHUB_HOST = "github.com"; +const REQUEST_HEADERS = { + Accept: "application/vnd.github+json", + "User-Agent": "zopu-project-importer", +} as const; + +const candidatePaths = { + agents: ["AGENTS.md"], + business: ["business.md", "docs/business.md"], + design: ["design.md", "docs/design.md"], + product: ["product.md", "docs/product.md"], + readme: ["README.md", "README", "readme.md"], + tech: ["tech.md", "docs/tech.md"], +} as const; + +const encodePath = (path: string) => + path + .split("/") + .map((segment) => encodeURIComponent(segment)) + .join("/"); + +const fetchTextCandidate = async ( + repositoryPath: string, + branch: string, + path: string +): Promise => { + const url = `https://raw.githubusercontent.com/${encodePath(repositoryPath)}/${encodePath(branch)}/${encodePath(path)}`; + const response = await fetch(url, { headers: REQUEST_HEADERS }); + if (response.status === 404) { + return null; + } + if (!response.ok) { + throw new Error(`GitHub returned ${response.status} for ${path}`); + } + const content = await response.text(); + if (content.trim().length === 0) { + return null; + } + if (content.length > MAX_CONTEXT_DOCUMENT_CHARACTERS) { + throw new Error(`${path} exceeds the 200000 character import limit`); + } + return content; +}; + +const fetchContextDocument = async ( + source: PreparedPublicGitSource, + branch: string, + kind: (typeof CONTEXT_KINDS)[number] +): Promise => { + for (const candidate of candidatePaths[kind]) { + const content = await fetchTextCandidate( + source.repositoryPath, + branch, + candidate + ); + if (content) { + return { + content, + kind, + path: contextPathForKind(kind), + }; + } + } + return null; +}; + +const readDefaultBranch = async ( + source: PreparedPublicGitSource +): Promise => { + if (source.host !== GITHUB_HOST) { + throw new Error("Repository import currently supports public GitHub URLs"); + } + const response = await fetch( + `https://api.github.com/repos/${encodePath(source.repositoryPath)}`, + { headers: REQUEST_HEADERS } + ); + if (!response.ok) { + throw new Error( + response.status === 404 + ? "Public GitHub repository not found" + : `GitHub returned ${response.status} while reading the repository` + ); + } + const metadata = (await response.json()) as { default_branch?: unknown }; + if ( + typeof metadata.default_branch !== "string" || + metadata.default_branch.length === 0 + ) { + throw new Error("GitHub did not return a default branch"); + } + return metadata.default_branch; +}; + +export const inspectPublicGit = async ( + source: PreparedPublicGitSource +): Promise => { + const defaultBranch = await readDefaultBranch(source); + const candidates = await Promise.all( + CONTEXT_KINDS.map((kind) => + fetchContextDocument(source, defaultBranch, kind) + ) + ); + const documents = candidates.filter( + (document): document is RepositoryContextDocument => document !== null + ); + return { + defaultBranch, + documents, + warnings: CONTEXT_KINDS.filter( + (kind) => !documents.some((document) => document.kind === kind) + ).map((kind) => ({ + message: "Canonical context file was not found in the repository", + path: contextPathForKind(kind), + })), + }; +};