04e6fb64bb
Phase 0 — measurement gate: - tests/test_contracts.py: BOARDS↔NORMALIZERS alignment, funnel-shape guards, overlap() None contract, and the frontend↔backend EXACT-STRING vocab check (scout.ts options must resolve in coerce.py, else silent keyword-fold). - tests/run_regression.py + fixtures/regression_set.json (+ _seed_regression.py): a frozen 120-pair set (tech + non-tech) Opus-judged for recruiter-fit targets. Two metrics: score MAE and SIFT TOP-K MEMBERSHIP RECALL (do the best jobs reach Opus?). Baseline: MAE 24.7, recall 0.68 — i.e. ~32% of the genuinely-best jobs are cut before the curator ever sees them (worse for non-tech). Phase 1 — coverage (India non-tech): - Enable Indeed (misceres) + enrich indeed_to_scoutjob with details.description + location_mode (jobType is employment type, not skills → required_skills now []). - Add WorkIndia (shahidirfan) — India blue/grey-collar non-tech: build_workindia_input + workindia_to_scoutjob (real skills + description; per-job apply URL from job_id since source_url is generic and would dedup-collapse the deck). - Drop Wellfound from the stack (US-startup-heavy + 400s); kept registered-but-off. - BOARDS_ENABLED = naukri,foundit,linkedin,indeed,workindia. Pool ~90 → ~140 jobs/search. Regression scores unchanged (engine scoring untouched). 31 tests pass.
80 lines
4.4 KiB
Python
80 lines
4.4 KiB
Python
"""One-off seeder for the regression fixture. Runs live sweeps (tech + non-tech), freezes ~12 candidate
|
|
jobs per case, and Opus-judges each (profile, job) for a recruiter-fit target. Writes
|
|
tests/fixtures/regression_set.json. Re-run only to refresh the baseline set (costs Apify + Opus)."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from app.engine import search
|
|
from app.engine.llm import gateway_client
|
|
from app.config import get_settings
|
|
|
|
OUT = Path(__file__).parent / "fixtures" / "regression_set.json"
|
|
N_PER_CASE = 40 # MUST exceed SIFT_TOP_K so the sift actually cuts → membership-recall is meaningful
|
|
|
|
CASES = [
|
|
{"label": "tech: backend engineer · bangalore",
|
|
"prefs": {"title": "Backend Engineer", "role": ["Engineering"], "location": ["Bangalore · India"]},
|
|
"user_context": {"id": "reg-tech", "skills": ["Python", "Django", "PostgreSQL", "AWS", "Docker", "REST APIs"],
|
|
"current_role": "Software Engineer"}},
|
|
{"label": "non-tech: sales manager · mumbai",
|
|
"prefs": {"title": "Sales Manager", "role": ["Sales"], "location": ["Mumbai · India"]},
|
|
"user_context": {"id": "reg-sales", "skills": ["B2B Sales", "Account Management", "Negotiation", "CRM", "Lead Generation"],
|
|
"current_role": "Business Development Manager"}},
|
|
{"label": "non-tech sparse: operations manager · pune",
|
|
"prefs": {"title": "Operations Manager", "role": ["Ops"], "location": ["Pune · India"]},
|
|
"user_context": {"id": "reg-ops", "skills": [], "current_role": "Operations Lead"}},
|
|
]
|
|
|
|
_JUDGE = """You are a veteran Indian recruiter. Given a CANDIDATE profile and a list of JOBS, score each job
|
|
0-100 for how good a match it is for THIS candidate (role fit + seniority + location + skills/responsibilities).
|
|
Be realistic and calibrated: a genuinely strong fit is 80-92, a solid fit 70-80, a stretch 55-70, off-target
|
|
<50. Respond ONLY with JSON: {"scores":{"<job_id>": <int>, ...}} using each job's exact id."""
|
|
|
|
|
|
async def judge(client, prefs, ctx, jobs):
|
|
brief = {"candidate": {"target": prefs.get("title"), "role": prefs.get("role"),
|
|
"location": prefs.get("location"), "skills": ctx.get("skills"),
|
|
"current_role": ctx.get("current_role")},
|
|
"jobs": [{"id": j["id"], "title": j.get("title"), "company": j.get("organization"),
|
|
"location": f"{j.get('location_city') or ''} {j.get('location_country') or ''}".strip(),
|
|
"skills": (j.get("details") or {}).get("skills") or j.get("required_skills"),
|
|
"snippet": ((j.get("details") or {}).get("description") or "")[:400]} for j in jobs]}
|
|
resp = client.chat.completions.create(
|
|
model=get_settings().CURATE_MODEL, max_tokens=2000,
|
|
messages=[{"role": "system", "content": _JUDGE}, {"role": "user", "content": json.dumps(brief, ensure_ascii=False)}])
|
|
txt = resp.choices[0].message.content
|
|
i, j = txt.find("{"), txt.rfind("}")
|
|
return {str(k): int(v) for k, v in json.loads(txt[i:j + 1])["scores"].items()}
|
|
|
|
|
|
async def main():
|
|
client = gateway_client()
|
|
assert client, "no gateway client (DSPY_API_BASE/KEY) — cannot judge"
|
|
out_cases = []
|
|
for c in CASES:
|
|
prefs = {**c["prefs"], "_fresh": True}
|
|
sweep = await search.run_sweep(prefs, fresh=True)
|
|
import random
|
|
pool = list(sweep["opportunities"])
|
|
random.Random(42).shuffle(pool) # board-diverse sample, deterministic
|
|
jobs = pool[:N_PER_CASE]
|
|
if not jobs:
|
|
print(f" ⚠️ {c['label']}: 0 jobs swept — skipping"); continue
|
|
targets = await judge(client, c["prefs"], c["user_context"], jobs)
|
|
# keep only jobs that got a target
|
|
jobs = [j for j in jobs if j["id"] in targets]
|
|
out_cases.append({"label": c["label"], "prefs": c["prefs"], "user_context": c["user_context"],
|
|
"jobs": jobs, "targets": {j["id"]: targets[j["id"]] for j in jobs}})
|
|
print(f" ✅ {c['label']}: {len(jobs)} jobs judged "
|
|
f"(sources={sweep['sources']}, target range {min(targets.values())}-{max(targets.values())})")
|
|
OUT.parent.mkdir(parents=True, exist_ok=True)
|
|
OUT.write_text(json.dumps({"version": "phase0-baseline", "cases": out_cases}, indent=2, ensure_ascii=False))
|
|
print(f" → wrote {len(out_cases)} cases, {sum(len(c['jobs']) for c in out_cases)} pairs to {OUT}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|