growqr-backend/scripts/service-registry-content-quality.mjs

#!/usr/bin/env node

const args = new Map();
for (let i = 2; i < process.argv.length; i += 1) {
  const key = process.argv[i];
  if (!key.startsWith("--")) continue;
  const next = process.argv[i + 1];
  args.set(key.slice(2), next && !next.startsWith("--") ? next : "true");
  if (next && !next.startsWith("--")) i += 1;
}

const baseUrl = (args.get("base-url") || process.env.BACKEND_BASE_URL || "http://127.0.0.1:4000").replace(/\/$/, "");
const userId = args.get("user-id") || process.env.SMOKE_USER_ID || "registry-content-quality";
const iterations = Number(args.get("iterations") || process.env.SMOKE_ITERATIONS || 1);
const previewTimeoutMs = Number(args.get("preview-timeout-ms") || process.env.SMOKE_PREVIEW_TIMEOUT_MS || 180000);
const serviceToken = process.env.SERVICE_TOKEN;

if (!serviceToken) {
  throw new Error("SERVICE_TOKEN is required for authenticated content-quality probes.");
}

const badMarkers = [/placeholder/i, /dummy/i, /not implemented/i, /fallback/i, /lorem/i, /todo/i, /undefined/i];

function assert(condition, message, detail) {
  if (condition) return;
  const suffix = detail === undefined ? "" : `\n${JSON.stringify(detail, null, 2).slice(0, 3000)}`;
  throw new Error(`${message}${suffix}`);
}

function outlineOf(json) {
  return Array.isArray(json?.question_outline) ? json.question_outline : json?.prompt_outline;
}

function walk(value, path = "$", strings = [], nulls = []) {
  if (value === null) nulls.push(path);
  else if (typeof value === "string") strings.push(value);
  else if (Array.isArray(value)) value.forEach((item, index) => walk(item, `${path}[${index}]`, strings, nulls));
  else if (value && typeof value === "object") Object.entries(value).forEach(([key, item]) => walk(item, `${path}.${key}`, strings, nulls));
  return { strings, nulls };
}

async function post(name, path, payload) {
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), previewTimeoutMs);
  const started = Date.now();
  try {
    const response = await fetch(`${baseUrl}${path}`, {
      method: "POST",
      signal: controller.signal,
      headers: {
        authorization: `Bearer ${serviceToken}`,
        "x-growqr-user": userId,
        "content-type": "application/json",
      },
      body: JSON.stringify(payload),
    });
    const text = await response.text();
    const durationMs = Date.now() - started;
    assert(response.ok, `${name} returned HTTP ${response.status}`, { text, durationMs });
    return { json: JSON.parse(text), durationMs };
  } catch (error) {
    if (error?.name === "AbortError") {
      throw new Error(`${name} timed out after ${Date.now() - started}ms`, { cause: error });
    }
    throw error;
  } finally {
    clearTimeout(timer);
  }
}

function validatePreview(name, json) {
  const outline = outlineOf(json);
  assert(Array.isArray(outline) && outline.length >= 3, `${name} needs at least 3 outline items`, outline);

  const { strings, nulls } = walk(json);
  assert(nulls.length === 0, `${name} contains null fields`, nulls.slice(0, 30));

  const cleanStrings = strings.map((item) => item.trim()).filter(Boolean);
  for (const marker of badMarkers) {
    assert(!cleanStrings.some((item) => marker.test(item)), `${name} contains marker ${marker}`, cleanStrings.filter((item) => marker.test(item)).slice(0, 10));
  }

  const prompts = outline
    .map((item) => String(item.question || item.prompt || item.text || "").replace(/\s+/g, " ").trim())
    .filter(Boolean);
  assert(prompts.length >= 3, `${name} outline prompts are missing text`, outline);
  assert(prompts.every((prompt) => prompt.length >= 35), `${name} outline prompts are too shallow`, prompts);
  assert(new Set(prompts.map((prompt) => prompt.toLowerCase())).size === prompts.length, `${name} outline prompts duplicate`, prompts);
  assert(String(json.opening_prompt || "").trim().length >= 35, `${name} opening prompt too short`, json.opening_prompt);

  const briefText = walk(json.candidate_brief).strings.join(" ").replace(/\s+/g, " ").trim();
  assert(briefText.length >= 300, `${name} candidate brief too thin`, briefText);
}

async function runIteration(iteration) {
  const user = `${userId}-${iteration}`;
  const interview = await post(`[content ${iteration}] interview preview`, "/services/interview/preview", {
    user_id: user,
    org_id: "growqr",
    persona_id: "emma",
    interview_type: "behavioral",
    duration_minutes: 5,
    context: {
      target_role: "Product Manager",
      company_name: "GrowQR Quality",
      difficulty: "medium",
      source: "registry-content-quality",
      personalize: false,
    },
  });
  validatePreview(`[content ${iteration}] interview preview`, interview.json);

  const roleplay = await post(`[content ${iteration}] roleplay preview`, "/services/roleplay/preview", {
    user_id: user,
    org_id: "growqr",
    persona_id: "emma",
    duration_minutes: 5,
    roleplay_type: "custom",
    brief: "Practice a concise salary negotiation opening for a product manager offer.",
    metadata: {
      target_role: "Product Manager",
      candidate_role: "Product Manager",
      difficulty: "medium",
      source: "registry-content-quality",
      personalize: false,
    },
  });
  validatePreview(`[content ${iteration}] roleplay preview`, roleplay.json);
  assert(roleplay.json.scenario?.candidate_role === "Product Manager", `[content ${iteration}] roleplay did not expose explicit candidate_role`, roleplay.json.scenario);
  assert(typeof roleplay.json.scenario?.persona_role === "string" && roleplay.json.scenario.persona_role.length > 0, `[content ${iteration}] roleplay did not expose persona_role`, roleplay.json.scenario);

  return {
    iteration,
    interviewSession: interview.json.session_id,
    interviewPreviewMs: interview.durationMs,
    roleplaySession: roleplay.json.session_id,
    roleplayPreviewMs: roleplay.durationMs,
  };
}

const results = [];
for (let i = 1; i <= iterations; i += 1) {
  const result = await runIteration(i);
  results.push(result);
  console.log(JSON.stringify(result));
}

console.log(JSON.stringify({ ok: true, iterations, results }));