zopu-code/packages/backend/convex/publicGit.ts

import { CONTEXT_KINDS, contextPathForKind, MAX_CONTEXT_DOCUMENT_CHARACTERS } from '@code/primitives/project';
import type { PreparedPublicGitSource, PublicGitImportResult, RepositoryContextDocument } from '@code/primitives/project';

const GITHUB_HOST = "github.com";
const GITEA_HOST = "git.openputer.com";
const GITHUB_REQUEST_HEADERS = {
  Accept: "application/vnd.github+json",
  "User-Agent": "zopu-project-importer",
} as const;
const GITEA_REQUEST_HEADERS = {
  Accept: "application/json",
  "User-Agent": "zopu-project-importer",
} as const;

type Fetch = (
  input: string | URL | Request,
  init?: RequestInit
) => Promise<Response>;

interface PublicGitProvider {
  readonly headers: Readonly<Record<string, string>>;
  readonly metadataUrl: (source: PreparedPublicGitSource) => string;
  readonly rawFileUrl: (
    source: PreparedPublicGitSource,
    branch: string,
    path: string
  ) => string;
  readonly repositoryNotFoundMessage: string;
}

const candidatePaths = {
  agents: ["AGENTS.md"],
  business: ["business.md", "docs/business.md"],
  design: ["design.md", "docs/design.md"],
  product: ["product.md", "docs/product.md"],
  readme: ["README.md", "README", "readme.md"],
  tech: ["tech.md", "docs/tech.md"],
} as const;

const encodePath = (path: string) =>
  path
    .split("/")
    .map((segment) => encodeURIComponent(segment))
    .join("/");

const providerForSource = (
  source: PreparedPublicGitSource
): PublicGitProvider => {
  if (source.host === GITHUB_HOST) {
    return {
      headers: GITHUB_REQUEST_HEADERS,
      metadataUrl: (candidate) =>
        `https://api.github.com/repos/${encodePath(candidate.repositoryPath)}`,
      rawFileUrl: (candidate, branch, path) =>
        `https://raw.githubusercontent.com/${encodePath(candidate.repositoryPath)}/${encodePath(branch)}/${encodePath(path)}`,
      repositoryNotFoundMessage: "Public GitHub repository not found",
    };
  }
  if (source.host === GITEA_HOST) {
    return {
      headers: GITEA_REQUEST_HEADERS,
      metadataUrl: (candidate) =>
        `https://${GITEA_HOST}/api/v1/repos/${encodePath(candidate.repositoryPath)}`,
      rawFileUrl: (candidate, branch, path) =>
        `https://${GITEA_HOST}/api/v1/repos/${encodePath(candidate.repositoryPath)}/raw/${encodePath(path)}?ref=${encodeURIComponent(branch)}`,
      repositoryNotFoundMessage: "Public Gitea repository not found",
    };
  }
  throw new Error(
    "Repository import supports public GitHub and git.openputer.com URLs"
  );
};

const fetchTextCandidate = async (
  fetcher: Fetch,
  provider: PublicGitProvider,
  source: PreparedPublicGitSource,
  branch: string,
  path: string
): Promise<string | null> => {
  const response = await fetcher(provider.rawFileUrl(source, branch, path), {
    headers: provider.headers,
  });
  if (response.status === 404) {
    return null;
  }
  if (!response.ok) {
    throw new Error(
      `${source.host} returned ${response.status} while reading ${path}`
    );
  }
  const content = await response.text();
  if (content.trim().length === 0) {
    return null;
  }
  if (content.length > MAX_CONTEXT_DOCUMENT_CHARACTERS) {
    throw new Error(`${path} exceeds the 200000 character import limit`);
  }
  return content;
};

const fetchContextDocument = async (
  fetcher: Fetch,
  provider: PublicGitProvider,
  source: PreparedPublicGitSource,
  branch: string,
  kind: (typeof CONTEXT_KINDS)[number]
): Promise<RepositoryContextDocument | null> => {
  for (const candidate of candidatePaths[kind]) {
    const content = await fetchTextCandidate(
      fetcher,
      provider,
      source,
      branch,
      candidate
    );
    if (content) {
      return {
        content,
        kind,
        path: contextPathForKind(kind),
      };
    }
  }
  return null;
};

const readDefaultBranch = async (
  fetcher: Fetch,
  provider: PublicGitProvider,
  source: PreparedPublicGitSource
): Promise<string> => {
  const response = await fetcher(provider.metadataUrl(source), {
    headers: provider.headers,
  });
  if (!response.ok) {
    throw new Error(
      response.status === 404
        ? provider.repositoryNotFoundMessage
        : `${source.host} returned ${response.status} while reading the repository`
    );
  }
  const metadata = (await response.json()) as { default_branch?: unknown };
  if (
    typeof metadata.default_branch !== "string" ||
    metadata.default_branch.length === 0
  ) {
    throw new Error(`${source.host} did not return a default branch`);
  }
  return metadata.default_branch;
};

export const inspectPublicGit = async (
  source: PreparedPublicGitSource,
  options: { readonly fetch?: Fetch } = {}
): Promise<PublicGitImportResult> => {
  const fetcher = options.fetch ?? globalThis.fetch;
  const provider = providerForSource(source);
  const defaultBranch = await readDefaultBranch(fetcher, provider, source);
  const candidates = await Promise.all(
    CONTEXT_KINDS.map((kind) =>
      fetchContextDocument(fetcher, provider, source, defaultBranch, kind)
    )
  );
  const documents = candidates.filter(
    (document): document is RepositoryContextDocument => document !== null
  );
  return {
    defaultBranch,
    documents,
    warnings: CONTEXT_KINDS.filter(
      (kind) => !documents.some((document) => document.kind === kind)
    ).map((kind) => ({
      message: "Canonical context file was not found in the repository",
      path: contextPathForKind(kind),
    })),
  };
};