Engine: regression harness (Phase 0) + India non-tech actors (Phase 1)
Phase 0 — measurement gate: - tests/test_contracts.py: BOARDS↔NORMALIZERS alignment, funnel-shape guards, overlap() None contract, and the frontend↔backend EXACT-STRING vocab check (scout.ts options must resolve in coerce.py, else silent keyword-fold). - tests/run_regression.py + fixtures/regression_set.json (+ _seed_regression.py): a frozen 120-pair set (tech + non-tech) Opus-judged for recruiter-fit targets. Two metrics: score MAE and SIFT TOP-K MEMBERSHIP RECALL (do the best jobs reach Opus?). Baseline: MAE 24.7, recall 0.68 — i.e. ~32% of the genuinely-best jobs are cut before the curator ever sees them (worse for non-tech). Phase 1 — coverage (India non-tech): - Enable Indeed (misceres) + enrich indeed_to_scoutjob with details.description + location_mode (jobType is employment type, not skills → required_skills now []). - Add WorkIndia (shahidirfan) — India blue/grey-collar non-tech: build_workindia_input + workindia_to_scoutjob (real skills + description; per-job apply URL from job_id since source_url is generic and would dedup-collapse the deck). - Drop Wellfound from the stack (US-startup-heavy + 400s); kept registered-but-off. - BOARDS_ENABLED = naukri,foundit,linkedin,indeed,workindia. Pool ~90 → ~140 jobs/search. Regression scores unchanged (engine scoring untouched). 31 tests pass.
This commit is contained in:
raulgupta
@@ -0,0 +1,79 @@
|
||||
"""One-off seeder for the regression fixture. Runs live sweeps (tech + non-tech), freezes ~12 candidate
|
||||
jobs per case, and Opus-judges each (profile, job) for a recruiter-fit target. Writes
|
||||
tests/fixtures/regression_set.json. Re-run only to refresh the baseline set (costs Apify + Opus)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from app.engine import search
|
||||
from app.engine.llm import gateway_client
|
||||
from app.config import get_settings
|
||||
|
||||
OUT = Path(__file__).parent / "fixtures" / "regression_set.json"
|
||||
N_PER_CASE = 40 # MUST exceed SIFT_TOP_K so the sift actually cuts → membership-recall is meaningful
|
||||
|
||||
CASES = [
|
||||
{"label": "tech: backend engineer · bangalore",
|
||||
"prefs": {"title": "Backend Engineer", "role": ["Engineering"], "location": ["Bangalore · India"]},
|
||||
"user_context": {"id": "reg-tech", "skills": ["Python", "Django", "PostgreSQL", "AWS", "Docker", "REST APIs"],
|
||||
"current_role": "Software Engineer"}},
|
||||
{"label": "non-tech: sales manager · mumbai",
|
||||
"prefs": {"title": "Sales Manager", "role": ["Sales"], "location": ["Mumbai · India"]},
|
||||
"user_context": {"id": "reg-sales", "skills": ["B2B Sales", "Account Management", "Negotiation", "CRM", "Lead Generation"],
|
||||
"current_role": "Business Development Manager"}},
|
||||
{"label": "non-tech sparse: operations manager · pune",
|
||||
"prefs": {"title": "Operations Manager", "role": ["Ops"], "location": ["Pune · India"]},
|
||||
"user_context": {"id": "reg-ops", "skills": [], "current_role": "Operations Lead"}},
|
||||
]
|
||||
|
||||
_JUDGE = """You are a veteran Indian recruiter. Given a CANDIDATE profile and a list of JOBS, score each job
|
||||
0-100 for how good a match it is for THIS candidate (role fit + seniority + location + skills/responsibilities).
|
||||
Be realistic and calibrated: a genuinely strong fit is 80-92, a solid fit 70-80, a stretch 55-70, off-target
|
||||
<50. Respond ONLY with JSON: {"scores":{"<job_id>": <int>, ...}} using each job's exact id."""
|
||||
|
||||
|
||||
async def judge(client, prefs, ctx, jobs):
|
||||
brief = {"candidate": {"target": prefs.get("title"), "role": prefs.get("role"),
|
||||
"location": prefs.get("location"), "skills": ctx.get("skills"),
|
||||
"current_role": ctx.get("current_role")},
|
||||
"jobs": [{"id": j["id"], "title": j.get("title"), "company": j.get("organization"),
|
||||
"location": f"{j.get('location_city') or ''} {j.get('location_country') or ''}".strip(),
|
||||
"skills": (j.get("details") or {}).get("skills") or j.get("required_skills"),
|
||||
"snippet": ((j.get("details") or {}).get("description") or "")[:400]} for j in jobs]}
|
||||
resp = client.chat.completions.create(
|
||||
model=get_settings().CURATE_MODEL, max_tokens=2000,
|
||||
messages=[{"role": "system", "content": _JUDGE}, {"role": "user", "content": json.dumps(brief, ensure_ascii=False)}])
|
||||
txt = resp.choices[0].message.content
|
||||
i, j = txt.find("{"), txt.rfind("}")
|
||||
return {str(k): int(v) for k, v in json.loads(txt[i:j + 1])["scores"].items()}
|
||||
|
||||
|
||||
async def main():
|
||||
client = gateway_client()
|
||||
assert client, "no gateway client (DSPY_API_BASE/KEY) — cannot judge"
|
||||
out_cases = []
|
||||
for c in CASES:
|
||||
prefs = {**c["prefs"], "_fresh": True}
|
||||
sweep = await search.run_sweep(prefs, fresh=True)
|
||||
import random
|
||||
pool = list(sweep["opportunities"])
|
||||
random.Random(42).shuffle(pool) # board-diverse sample, deterministic
|
||||
jobs = pool[:N_PER_CASE]
|
||||
if not jobs:
|
||||
print(f" ⚠️ {c['label']}: 0 jobs swept — skipping"); continue
|
||||
targets = await judge(client, c["prefs"], c["user_context"], jobs)
|
||||
# keep only jobs that got a target
|
||||
jobs = [j for j in jobs if j["id"] in targets]
|
||||
out_cases.append({"label": c["label"], "prefs": c["prefs"], "user_context": c["user_context"],
|
||||
"jobs": jobs, "targets": {j["id"]: targets[j["id"]] for j in jobs}})
|
||||
print(f" ✅ {c['label']}: {len(jobs)} jobs judged "
|
||||
f"(sources={sweep['sources']}, target range {min(targets.values())}-{max(targets.values())})")
|
||||
OUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
OUT.write_text(json.dumps({"version": "phase0-baseline", "cases": out_cases}, indent=2, ensure_ascii=False))
|
||||
print(f" → wrote {len(out_cases)} cases, {sum(len(c['jobs']) for c in out_cases)} pairs to {OUT}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Vendored
+5787
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,84 @@
|
||||
"""Regression runner — the measurement gate for every engine-improvement phase.
|
||||
|
||||
The fixture `tests/fixtures/regression_set.json` freezes a set of CASES, each = {prefs, user_context,
|
||||
jobs[], targets{job_id: recruiter_score}}. The jobs are captured from a real sweep so the run is
|
||||
reproducible WITHOUT re-hitting Apify (deterministic, cheap, comparable across phases).
|
||||
|
||||
For each case this runs the REAL engine (rank → sift) over the frozen jobs and reports two metrics:
|
||||
• SCORE fidelity — MAE / correlation of the engine's matchScore vs the human/judge target.
|
||||
• SIFT TOP-K MEMBERSHIP RECALL — of the genuinely-best jobs (target ≥ 75), how many reach the top-K
|
||||
pool Opus actually sees. This is the "is the funnel feeding Opus the best jobs?" metric.
|
||||
|
||||
Usage: python tests/run_regression.py # prints the table for the current engine
|
||||
python tests/run_regression.py --json # machine-readable (to diff phases)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from app.engine import rank as _rank
|
||||
from app.engine import sift as _sift
|
||||
from app.config import get_settings
|
||||
|
||||
FIXTURE = Path(__file__).parent / "fixtures" / "regression_set.json"
|
||||
GREAT = 75 # target ≥ this = a "genuinely good" job that SHOULD reach Opus
|
||||
|
||||
|
||||
def _mae(pairs):
|
||||
return round(sum(abs(a - b) for a, b in pairs) / len(pairs), 1) if pairs else None
|
||||
|
||||
|
||||
def run(emit_json: bool = False):
|
||||
data = json.loads(FIXTURE.read_text())
|
||||
cases = data["cases"]
|
||||
k = get_settings().SIFT_TOP_K
|
||||
rows, all_score_pairs, recalls = [], [], []
|
||||
|
||||
for c in cases:
|
||||
prefs, ctx, jobs = c["prefs"], c.get("user_context"), c["jobs"]
|
||||
targets = {str(j): int(s) for j, s in c["targets"].items()}
|
||||
# engine: white-box scores + the sift top-K the curator would see
|
||||
ranked = _rank.rank(prefs, ctx, [dict(j) for j in jobs])["opportunities"]
|
||||
eng = {j["id"]: j.get("matchScore", 0) for j in ranked}
|
||||
top, _ = _sift.sift(prefs, ctx, [dict(j) for j in jobs], k=k)
|
||||
top_ids = {j["id"] for j in top}
|
||||
|
||||
score_pairs = [(eng[jid], targets[jid]) for jid in eng if jid in targets]
|
||||
all_score_pairs += score_pairs
|
||||
# membership recall: of target-great jobs, how many made the sift top-K
|
||||
great = [jid for jid, t in targets.items() if t >= GREAT and jid in eng]
|
||||
in_top = [jid for jid in great if jid in top_ids]
|
||||
recall = round(len(in_top) / len(great), 2) if great else None
|
||||
if recall is not None:
|
||||
recalls.append(recall)
|
||||
rows.append({
|
||||
"case": c["label"], "n_jobs": len(jobs), "n_great": len(great),
|
||||
"great_in_topK": len(in_top), "membership_recall": recall,
|
||||
"score_mae": _mae(score_pairs),
|
||||
})
|
||||
|
||||
agg = {
|
||||
"sift_top_k": k,
|
||||
"overall_score_mae": _mae(all_score_pairs),
|
||||
"overall_membership_recall": round(sum(recalls) / len(recalls), 2) if recalls else None,
|
||||
"n_cases": len(cases), "n_pairs": len(all_score_pairs),
|
||||
}
|
||||
if emit_json:
|
||||
print(json.dumps({"cases": rows, "aggregate": agg}, indent=2))
|
||||
return
|
||||
print(f"\n Regression — engine scoring vs target (SIFT_TOP_K={k})")
|
||||
print(" " + "-" * 78)
|
||||
print(f" {'case':<40}{'jobs':>5}{'great':>7}{'in-topK':>9}{'recall':>8}{'MAE':>6}")
|
||||
for r in rows:
|
||||
print(f" {r['case'][:40]:<40}{r['n_jobs']:>5}{r['n_great']:>7}{r['great_in_topK']:>9}"
|
||||
f"{str(r['membership_recall']):>8}{str(r['score_mae']):>6}")
|
||||
print(" " + "-" * 78)
|
||||
print(f" AGGREGATE → score MAE {agg['overall_score_mae']} · "
|
||||
f"sift membership recall {agg['overall_membership_recall']} "
|
||||
f"({agg['n_pairs']} pairs, {agg['n_cases']} cases)\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(emit_json="--json" in sys.argv)
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Phase-0 contract checks — the guardrails that catch silent failures BEFORE they ship.
|
||||
|
||||
These assert the load-bearing contracts the engine-improvement plan depends on:
|
||||
1. BOARDS ↔ NORMALIZERS alignment (an enabled board with no normalizer = silent drop).
|
||||
2. The funnel shapes (sift fusion weights, embed text fns, SIFT_TOP_K) are where the plan expects.
|
||||
3. overlap() returns None when a side has no skills (the floor-free contract).
|
||||
4. The frontend↔backend EXACT-STRING vocab contract: every ScoutPrefs ROLE/INDUSTRY option the UI
|
||||
offers must resolve to a coerce.py board code, else it silently degrades to a keyword fold.
|
||||
(Cross-repo read of growqr-demo's scout.ts; skipped if the sibling repo isn't present.)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.engine import search, normalize, sift, embed
|
||||
from app.engine.board_adapters import coerce as C
|
||||
from app.engine import skills as S
|
||||
from app.config import get_settings
|
||||
|
||||
|
||||
# ── 1. BOARDS ↔ NORMALIZERS alignment ────────────────────────────────────────
|
||||
def test_boards_have_normalizers():
|
||||
for board in search.BOARDS:
|
||||
assert board in normalize.NORMALIZERS, f"BOARDS has '{board}' with no NORMALIZERS entry"
|
||||
|
||||
|
||||
def test_enabled_boards_are_registered():
|
||||
enabled = [b.strip() for b in get_settings().BOARDS_ENABLED.split(",") if b.strip()]
|
||||
for b in enabled:
|
||||
assert b in search.BOARDS, f"BOARDS_ENABLED lists '{b}' which is not a registered board"
|
||||
assert b in normalize.NORMALIZERS, f"enabled board '{b}' has no normalizer"
|
||||
|
||||
|
||||
# ── 2. Funnel shapes (guards the Phase-2 edits) ──────────────────────────────
|
||||
def test_sift_fusion_weights_present_and_normalized():
|
||||
assert hasattr(sift, "W_WHITEBOX") and hasattr(sift, "W_VIBE")
|
||||
assert abs((sift.W_WHITEBOX + sift.W_VIBE) - 1.0) < 1e-9, "sift fusion weights must sum to 1"
|
||||
|
||||
|
||||
def test_embed_text_builders_exist():
|
||||
# the Phase-2 richer-embedding work edits these; assert they exist + return strings
|
||||
assert callable(embed._job_text) and callable(embed._profile_text)
|
||||
assert isinstance(embed._job_text({"title": "X", "organization": "Y"}), str)
|
||||
assert isinstance(embed._profile_text({"title": "X"}, None), str)
|
||||
|
||||
|
||||
def test_sift_top_k_configured():
|
||||
assert isinstance(get_settings().SIFT_TOP_K, int) and get_settings().SIFT_TOP_K > 0
|
||||
|
||||
|
||||
# ── 3. Floor-free skill contract ─────────────────────────────────────────────
|
||||
def test_overlap_none_when_a_side_is_empty():
|
||||
assert S.overlap([], ["python", "sql"]) is None, "empty user skills must yield None (not a floor)"
|
||||
assert S.overlap(["python"], []) is None, "empty job skills must yield None (not a floor)"
|
||||
# both present → a real dict with a score
|
||||
o = S.overlap(["python", "sql"], ["python", "django"])
|
||||
assert isinstance(o, dict) and "score" in o
|
||||
|
||||
|
||||
# ── 4. Frontend↔backend EXACT-STRING vocab contract ──────────────────────────
|
||||
# Values the UI may send that are intentionally NOT board codes (keyword-only / "open").
|
||||
_ROLE_KEYWORD_ONLY = {"Open to any"}
|
||||
_INDUSTRY_KEYWORD_ONLY = {"Any", "Open to any", ""}
|
||||
|
||||
_SCOUT_TS = Path(__file__).resolve().parents[2] / "growqr-demo/frontend/src/scout/data/scout.ts"
|
||||
|
||||
|
||||
def _extract_array(name: str, text: str) -> list[str]:
|
||||
m = re.search(rf"{name}\s*[:=]\s*\[(.*?)\]", text, re.S)
|
||||
if not m:
|
||||
return []
|
||||
return re.findall(r'"([^"]+)"', m.group(1))
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _SCOUT_TS.exists(), reason="growqr-demo frontend not present (cross-repo)")
|
||||
def test_frontend_roles_resolve_in_backend():
|
||||
text = _SCOUT_TS.read_text()
|
||||
roles = _extract_array("ROLE_OPTIONS", text)
|
||||
assert roles, "could not extract ROLE_OPTIONS from scout.ts — regex/contract drift"
|
||||
unmapped = [r for r in roles if r not in C.ROLE_NAUKRI_DEPT and r not in _ROLE_KEYWORD_ONLY]
|
||||
assert not unmapped, f"frontend ROLE options with no coerce.py code (silent keyword-fold): {unmapped}"
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _SCOUT_TS.exists(), reason="growqr-demo frontend not present (cross-repo)")
|
||||
def test_frontend_industries_resolve_in_backend():
|
||||
text = _SCOUT_TS.read_text()
|
||||
inds = _extract_array("INDUSTRY_OPTIONS", text)
|
||||
assert inds, "could not extract INDUSTRY_OPTIONS from scout.ts — regex/contract drift"
|
||||
unmapped = [i for i in inds if i not in C.INDUSTRY_NAUKRI and i not in _INDUSTRY_KEYWORD_ONLY]
|
||||
assert not unmapped, f"frontend INDUSTRY options with no coerce.py code (silent keyword-fold): {unmapped}"
|
||||
Reference in New Issue
Block a user