Engine: regression harness (Phase 0) + India non-tech actors (Phase 1)
Phase 0 — measurement gate: - tests/test_contracts.py: BOARDS↔NORMALIZERS alignment, funnel-shape guards, overlap() None contract, and the frontend↔backend EXACT-STRING vocab check (scout.ts options must resolve in coerce.py, else silent keyword-fold). - tests/run_regression.py + fixtures/regression_set.json (+ _seed_regression.py): a frozen 120-pair set (tech + non-tech) Opus-judged for recruiter-fit targets. Two metrics: score MAE and SIFT TOP-K MEMBERSHIP RECALL (do the best jobs reach Opus?). Baseline: MAE 24.7, recall 0.68 — i.e. ~32% of the genuinely-best jobs are cut before the curator ever sees them (worse for non-tech). Phase 1 — coverage (India non-tech): - Enable Indeed (misceres) + enrich indeed_to_scoutjob with details.description + location_mode (jobType is employment type, not skills → required_skills now []). - Add WorkIndia (shahidirfan) — India blue/grey-collar non-tech: build_workindia_input + workindia_to_scoutjob (real skills + description; per-job apply URL from job_id since source_url is generic and would dedup-collapse the deck). - Drop Wellfound from the stack (US-startup-heavy + 400s); kept registered-but-off. - BOARDS_ENABLED = naukri,foundit,linkedin,indeed,workindia. Pool ~90 → ~140 jobs/search. Regression scores unchanged (engine scoring untouched). 31 tests pass.
This commit is contained in:
@@ -52,7 +52,7 @@ class Settings(BaseSettings):
|
||||
# Active boards — "Balanced" India stack: all three filter to the exact city at the board
|
||||
# (hyper-relevant) and surface offsite apply links. Naukri=blackfalcondata feed,
|
||||
# Foundit=Monster India, LinkedIn=geoId/city. Others (ats/indeed/naukri_v1) stay registered, off.
|
||||
BOARDS_ENABLED: str = "naukri,foundit,linkedin"
|
||||
BOARDS_ENABLED: str = "naukri,foundit,linkedin,indeed,workindia"
|
||||
|
||||
# Per-board fetch budget (the #1 Apify cost lever). Demo-sized to conserve credits.
|
||||
NAUKRI_MAX_JOBS: int = 15 # blackfalcondata maxResults (no floor); fetchDetails=True for offsite apply links
|
||||
@@ -62,6 +62,8 @@ class Settings(BaseSettings):
|
||||
ATS_MAX_PER_COMPANY: int = 4
|
||||
ATS_COMPANIES: str = "stripe,databricks,gitlab,figma,ramp,notion,razorpay,zerodha"
|
||||
INDEED_MAX_JOBS: int = 30
|
||||
WELLFOUND_MAX_JOBS: int = 15 # blackfalcondata/wellfound-scraper — registered, OFF (US-heavy)
|
||||
WORKINDIA_MAX_JOBS: int = 20 # shahidirfan/workindia-jobs-scraper — India blue/grey-collar non-tech
|
||||
|
||||
# QScore (competence proxy) — consumed, not computed. Dashboard Momentum/QX reads it via REST.
|
||||
QSCORE_BASE_URL: str = "http://localhost:8004"
|
||||
|
||||
@@ -208,6 +208,38 @@ def build_ats_input(prefs: Prefs, *, companies: list[dict], recent_days: int = 3
|
||||
return out
|
||||
|
||||
|
||||
# ── WorkIndia (India blue/grey-collar — the non-tech workhorse: sales, BPO, retail, field, ops) ──
|
||||
def build_workindia_input(prefs: Prefs, *, results_wanted: int = 20) -> dict:
|
||||
# keyword + city; includeDetails=True pulls the real skills + full description (verified live Phase 0).
|
||||
# No page param → per-user seen-net dedups. City is a lowercased name slug ("mumbai", "pune").
|
||||
city = next((lp["city"] for lp in C.parsed_locations(prefs) if lp["city"]), "")
|
||||
out: dict[str, Any] = {
|
||||
"keyword": C.title_of(prefs) or C.build_keyword(prefs),
|
||||
"results_wanted": results_wanted,
|
||||
"includeDetails": True,
|
||||
}
|
||||
if city:
|
||||
out["city"] = city.lower()
|
||||
return out
|
||||
|
||||
|
||||
# ── Wellfound / AngelList (startup roles — equity + skills; remote-friendly for India) ──
|
||||
def build_wellfound_input(prefs: Prefs, *, max_results: int = 15) -> dict:
|
||||
# query=title; city → location slug (lowercase-hyphen); remote honored. enrichDetail=True is
|
||||
# REQUIRED — it pulls skills[] + the full description the curator scores on (the core fields
|
||||
# alone are thin). No page param → dedup via the per-user seen-net (like Naukri/Foundit).
|
||||
city = next((lp["city"] for lp in C.parsed_locations(prefs) if lp["city"]), "")
|
||||
out: dict[str, Any] = {
|
||||
"query": C.title_of(prefs) or C.build_keyword(prefs),
|
||||
"maxResults": max_results,
|
||||
"enrichDetail": True, # → skills + description (the curator's signal)
|
||||
"remote": C.is_remote(prefs),
|
||||
}
|
||||
if city:
|
||||
out["location"] = city.lower().replace(" ", "-") # "New Delhi" → "new-delhi"
|
||||
return out
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import json
|
||||
demo: Prefs = {
|
||||
|
||||
@@ -200,6 +200,13 @@ def indeed_to_scoutjob(it: dict, board: str = "Indeed") -> dict | None:
|
||||
ext = it.get("externalApplyLink")
|
||||
offsite = bool(ext and ext != "None") # externalApplyLink = real offsite employer link
|
||||
apply_url = ext if offsite else it.get("url") # offsite first, Indeed listing fallback
|
||||
mode = "remote" if any("remote" in t.lower() for t in types) else None
|
||||
# jobType is employment classification ("Full-time", "Remote") — NOT skills. The real signal for
|
||||
# embedding + the curator is the description, which the actor DOES return (verified Phase 0).
|
||||
details = _compact({
|
||||
"description": _plain(it.get("description") or it.get("descriptionHTML")),
|
||||
"employment_type": ", ".join(types) or None,
|
||||
})
|
||||
return {
|
||||
"id": f"indeed-{it.get('id')}",
|
||||
"title": title,
|
||||
@@ -208,15 +215,17 @@ def indeed_to_scoutjob(it: dict, board: str = "Indeed") -> dict | None:
|
||||
"posted_date": _posted_date(it),
|
||||
"location_city": city,
|
||||
"location_country": "India",
|
||||
"location_mode": mode,
|
||||
"matchScore": 0,
|
||||
"salary_lpa": lpa,
|
||||
"payLabel": pay,
|
||||
"required_skills": types,
|
||||
"required_skills": [], # Indeed exposes no skill list; description carries the signal
|
||||
"tags": types[:3],
|
||||
"applicants": None,
|
||||
"note": f"Live from {board}",
|
||||
"apply_url": apply_url,
|
||||
"offsite_apply": offsite,
|
||||
"details": details,
|
||||
"qx": 10,
|
||||
}
|
||||
|
||||
@@ -415,6 +424,109 @@ def foundit_to_scoutjob(it: dict, board: str = "Foundit") -> dict | None:
|
||||
}
|
||||
|
||||
|
||||
def workindia_to_scoutjob(it: dict, board: str = "WorkIndia") -> dict | None:
|
||||
"""shahidirfan/workindia-jobs-scraper (includeDetails=True) — India blue/grey-collar non-tech.
|
||||
Verified live (Phase 0): returns real `skills` + `profile_job_description`. Apply is on-platform."""
|
||||
title = (it.get("profile_job_title") or "").strip()
|
||||
org = (it.get("branch_company_name") or "").strip()
|
||||
if not title or not org:
|
||||
return None
|
||||
city = (str(it.get("branch_location_city_name") or "").strip() or None)
|
||||
skills = it.get("skills") or []
|
||||
if isinstance(skills, str):
|
||||
skills = [s.strip() for s in skills.split(",") if s.strip()]
|
||||
skills = [str(s) for s in skills if s]
|
||||
seniority = None
|
||||
m = re.search(r"\d+", str(it.get("job_experience") or it.get("experience") or ""))
|
||||
if m:
|
||||
seniority = _seniority_from_years(int(m.group()))
|
||||
# Blue/grey-collar pay is usually monthly ₹ and noisy — show the structure text, don't risk the ₹L band.
|
||||
details = _compact({
|
||||
"description": _plain(it.get("profile_job_description") or it.get("profile_html_description")
|
||||
or it.get("profile_short_description")),
|
||||
"experience": it.get("job_experience") or it.get("experience"),
|
||||
"employment_type": it.get("employment_type"),
|
||||
"education": it.get("degree") or it.get("profile_qualification_required"),
|
||||
"industry": it.get("profile_industry_display_name"),
|
||||
"skills": skills,
|
||||
})
|
||||
return {
|
||||
"id": f"workindia-{it.get('job_id') or it.get('id')}",
|
||||
"title": title.title() if title.isupper() else title,
|
||||
"organization": org,
|
||||
"logo": _logo(org, _logo_url(it)),
|
||||
"posted_date": it.get("job_published_on_platform") or it.get("created_at"),
|
||||
"location_city": city,
|
||||
"location_country": "India",
|
||||
"matchScore": 0,
|
||||
"salary_lpa": None,
|
||||
"payLabel": it.get("profile_salary_structure") or "Not disclosed",
|
||||
"seniority_level": seniority,
|
||||
"required_skills": skills[:6],
|
||||
"tags": skills[:3],
|
||||
"applicants": None,
|
||||
"note": f"Live from {board}",
|
||||
# source_url is a GENERIC listing page (same for every job) → would dedup-collapse the deck.
|
||||
# Build a per-job URL from job_id so each is unique AND clickable.
|
||||
"apply_url": (f"https://www.workindia.in/jobs/{it.get('job_id')}/"
|
||||
if it.get("job_id") else it.get("source_url")),
|
||||
"offsite_apply": False, # WorkIndia is a platform — apply on WorkIndia
|
||||
"details": details,
|
||||
"qx": 10,
|
||||
}
|
||||
|
||||
|
||||
def wellfound_to_scoutjob(it: dict, board: str = "Wellfound") -> dict | None:
|
||||
"""blackfalcondata/wellfound-scraper (enrichDetail=True) — startup roles with skills, equity,
|
||||
funding stage. Apply is on-platform (Wellfound), so apply_url = the listing (offsite_apply=False)."""
|
||||
title = (it.get("title") or "").strip()
|
||||
org = (it.get("companyName") or "").strip()
|
||||
if not title or not org:
|
||||
return None
|
||||
locs = it.get("locationNames") or []
|
||||
city = (str(locs[0]).split(",")[0].strip() or None) if isinstance(locs, list) and locs else None
|
||||
skills = [str(s) for s in (it.get("skills") or []) if s]
|
||||
# Salary → LPA only when the listing is in INR (don't pollute the India band with USD); else the
|
||||
# raw compensation string is the honest label.
|
||||
lpa, pay = None, "Not disclosed"
|
||||
cur = (it.get("salaryCurrency") or "").upper()
|
||||
if it.get("salaryMax") and cur == "INR":
|
||||
lpa, pay = _salary_from_amount(int(it["salaryMax"]), per_year=True)
|
||||
elif it.get("compensation"):
|
||||
pay = str(it["compensation"])
|
||||
elif it.get("salaryMin") or it.get("salaryMax"):
|
||||
pay = f"{cur or '$'} {it.get('salaryMin') or ''}–{it.get('salaryMax') or ''}".strip()
|
||||
details = _compact({
|
||||
"description": _plain(it.get("description")),
|
||||
"employment_type": (it.get("jobType") or "").replace("_", " ").title() or None,
|
||||
"skills": skills,
|
||||
"funding_stage": it.get("companyFundingStage"),
|
||||
"equity": it.get("equityMin") or it.get("equity"),
|
||||
"visa": it.get("visaSponsorship"),
|
||||
})
|
||||
return {
|
||||
"id": f"wellfound-{it.get('id')}",
|
||||
"title": title,
|
||||
"organization": org,
|
||||
"logo": _logo(org, _logo_url(it)),
|
||||
"posted_date": _posted_date(it),
|
||||
"location_city": city,
|
||||
"location_country": None,
|
||||
"location_mode": "remote" if it.get("remote") else None,
|
||||
"matchScore": 0,
|
||||
"salary_lpa": lpa,
|
||||
"payLabel": pay,
|
||||
"required_skills": skills[:6],
|
||||
"tags": skills[:3],
|
||||
"applicants": None,
|
||||
"note": f"Live from {board}",
|
||||
"apply_url": it.get("portalUrl"),
|
||||
"offsite_apply": False, # apply happens on Wellfound (on-platform), not an offsite employer redirect
|
||||
"details": details,
|
||||
"qx": 10,
|
||||
}
|
||||
|
||||
|
||||
# board key → normalizer
|
||||
NORMALIZERS = {
|
||||
"naukri": naukri_feed_to_scoutjob, # blackfalcondata (primary)
|
||||
@@ -423,4 +535,6 @@ NORMALIZERS = {
|
||||
"naukri_v1": naukri_to_scoutjob, # muhammetakkurtt (fallback)
|
||||
"indeed": indeed_to_scoutjob,
|
||||
"ats": ats_to_scoutjob,
|
||||
"wellfound": wellfound_to_scoutjob, # blackfalcondata (startup roles)
|
||||
"workindia": workindia_to_scoutjob, # shahidirfan (India blue/grey-collar non-tech)
|
||||
}
|
||||
|
||||
@@ -52,6 +52,13 @@ BOARDS = {
|
||||
"indeed": ("misceres~indeed-scraper",
|
||||
lambda p: A.build_indeed_input(p, max_items=get_settings().INDEED_MAX_JOBS),
|
||||
N.indeed_to_scoutjob),
|
||||
"wellfound": ("blackfalcondata~wellfound-scraper", # registered, OFF — US-startup-heavy, not the India fit
|
||||
lambda p: A.build_wellfound_input(p, max_results=get_settings().WELLFOUND_MAX_JOBS),
|
||||
N.wellfound_to_scoutjob),
|
||||
"workindia": ("shahidirfan~workindia-jobs-scraper",
|
||||
# India blue/grey-collar non-tech; no page param → per-user seen-net dedups.
|
||||
lambda p: A.build_workindia_input(p, results_wanted=get_settings().WORKINDIA_MAX_JOBS),
|
||||
N.workindia_to_scoutjob),
|
||||
}
|
||||
|
||||
|
||||
|
||||
79
tests/_seed_regression.py
Normal file
79
tests/_seed_regression.py
Normal file
@@ -0,0 +1,79 @@
|
||||
"""One-off seeder for the regression fixture. Runs live sweeps (tech + non-tech), freezes ~12 candidate
|
||||
jobs per case, and Opus-judges each (profile, job) for a recruiter-fit target. Writes
|
||||
tests/fixtures/regression_set.json. Re-run only to refresh the baseline set (costs Apify + Opus)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from app.engine import search
|
||||
from app.engine.llm import gateway_client
|
||||
from app.config import get_settings
|
||||
|
||||
OUT = Path(__file__).parent / "fixtures" / "regression_set.json"
|
||||
N_PER_CASE = 40 # MUST exceed SIFT_TOP_K so the sift actually cuts → membership-recall is meaningful
|
||||
|
||||
CASES = [
|
||||
{"label": "tech: backend engineer · bangalore",
|
||||
"prefs": {"title": "Backend Engineer", "role": ["Engineering"], "location": ["Bangalore · India"]},
|
||||
"user_context": {"id": "reg-tech", "skills": ["Python", "Django", "PostgreSQL", "AWS", "Docker", "REST APIs"],
|
||||
"current_role": "Software Engineer"}},
|
||||
{"label": "non-tech: sales manager · mumbai",
|
||||
"prefs": {"title": "Sales Manager", "role": ["Sales"], "location": ["Mumbai · India"]},
|
||||
"user_context": {"id": "reg-sales", "skills": ["B2B Sales", "Account Management", "Negotiation", "CRM", "Lead Generation"],
|
||||
"current_role": "Business Development Manager"}},
|
||||
{"label": "non-tech sparse: operations manager · pune",
|
||||
"prefs": {"title": "Operations Manager", "role": ["Ops"], "location": ["Pune · India"]},
|
||||
"user_context": {"id": "reg-ops", "skills": [], "current_role": "Operations Lead"}},
|
||||
]
|
||||
|
||||
_JUDGE = """You are a veteran Indian recruiter. Given a CANDIDATE profile and a list of JOBS, score each job
|
||||
0-100 for how good a match it is for THIS candidate (role fit + seniority + location + skills/responsibilities).
|
||||
Be realistic and calibrated: a genuinely strong fit is 80-92, a solid fit 70-80, a stretch 55-70, off-target
|
||||
<50. Respond ONLY with JSON: {"scores":{"<job_id>": <int>, ...}} using each job's exact id."""
|
||||
|
||||
|
||||
async def judge(client, prefs, ctx, jobs):
|
||||
brief = {"candidate": {"target": prefs.get("title"), "role": prefs.get("role"),
|
||||
"location": prefs.get("location"), "skills": ctx.get("skills"),
|
||||
"current_role": ctx.get("current_role")},
|
||||
"jobs": [{"id": j["id"], "title": j.get("title"), "company": j.get("organization"),
|
||||
"location": f"{j.get('location_city') or ''} {j.get('location_country') or ''}".strip(),
|
||||
"skills": (j.get("details") or {}).get("skills") or j.get("required_skills"),
|
||||
"snippet": ((j.get("details") or {}).get("description") or "")[:400]} for j in jobs]}
|
||||
resp = client.chat.completions.create(
|
||||
model=get_settings().CURATE_MODEL, max_tokens=2000,
|
||||
messages=[{"role": "system", "content": _JUDGE}, {"role": "user", "content": json.dumps(brief, ensure_ascii=False)}])
|
||||
txt = resp.choices[0].message.content
|
||||
i, j = txt.find("{"), txt.rfind("}")
|
||||
return {str(k): int(v) for k, v in json.loads(txt[i:j + 1])["scores"].items()}
|
||||
|
||||
|
||||
async def main():
|
||||
client = gateway_client()
|
||||
assert client, "no gateway client (DSPY_API_BASE/KEY) — cannot judge"
|
||||
out_cases = []
|
||||
for c in CASES:
|
||||
prefs = {**c["prefs"], "_fresh": True}
|
||||
sweep = await search.run_sweep(prefs, fresh=True)
|
||||
import random
|
||||
pool = list(sweep["opportunities"])
|
||||
random.Random(42).shuffle(pool) # board-diverse sample, deterministic
|
||||
jobs = pool[:N_PER_CASE]
|
||||
if not jobs:
|
||||
print(f" ⚠️ {c['label']}: 0 jobs swept — skipping"); continue
|
||||
targets = await judge(client, c["prefs"], c["user_context"], jobs)
|
||||
# keep only jobs that got a target
|
||||
jobs = [j for j in jobs if j["id"] in targets]
|
||||
out_cases.append({"label": c["label"], "prefs": c["prefs"], "user_context": c["user_context"],
|
||||
"jobs": jobs, "targets": {j["id"]: targets[j["id"]] for j in jobs}})
|
||||
print(f" ✅ {c['label']}: {len(jobs)} jobs judged "
|
||||
f"(sources={sweep['sources']}, target range {min(targets.values())}-{max(targets.values())})")
|
||||
OUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
OUT.write_text(json.dumps({"version": "phase0-baseline", "cases": out_cases}, indent=2, ensure_ascii=False))
|
||||
print(f" → wrote {len(out_cases)} cases, {sum(len(c['jobs']) for c in out_cases)} pairs to {OUT}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
5787
tests/fixtures/regression_set.json
vendored
Normal file
5787
tests/fixtures/regression_set.json
vendored
Normal file
File diff suppressed because it is too large
Load Diff
84
tests/run_regression.py
Normal file
84
tests/run_regression.py
Normal file
@@ -0,0 +1,84 @@
|
||||
"""Regression runner — the measurement gate for every engine-improvement phase.
|
||||
|
||||
The fixture `tests/fixtures/regression_set.json` freezes a set of CASES, each = {prefs, user_context,
|
||||
jobs[], targets{job_id: recruiter_score}}. The jobs are captured from a real sweep so the run is
|
||||
reproducible WITHOUT re-hitting Apify (deterministic, cheap, comparable across phases).
|
||||
|
||||
For each case this runs the REAL engine (rank → sift) over the frozen jobs and reports two metrics:
|
||||
• SCORE fidelity — MAE / correlation of the engine's matchScore vs the human/judge target.
|
||||
• SIFT TOP-K MEMBERSHIP RECALL — of the genuinely-best jobs (target ≥ 75), how many reach the top-K
|
||||
pool Opus actually sees. This is the "is the funnel feeding Opus the best jobs?" metric.
|
||||
|
||||
Usage: python tests/run_regression.py # prints the table for the current engine
|
||||
python tests/run_regression.py --json # machine-readable (to diff phases)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from app.engine import rank as _rank
|
||||
from app.engine import sift as _sift
|
||||
from app.config import get_settings
|
||||
|
||||
FIXTURE = Path(__file__).parent / "fixtures" / "regression_set.json"
|
||||
GREAT = 75 # target ≥ this = a "genuinely good" job that SHOULD reach Opus
|
||||
|
||||
|
||||
def _mae(pairs):
|
||||
return round(sum(abs(a - b) for a, b in pairs) / len(pairs), 1) if pairs else None
|
||||
|
||||
|
||||
def run(emit_json: bool = False):
|
||||
data = json.loads(FIXTURE.read_text())
|
||||
cases = data["cases"]
|
||||
k = get_settings().SIFT_TOP_K
|
||||
rows, all_score_pairs, recalls = [], [], []
|
||||
|
||||
for c in cases:
|
||||
prefs, ctx, jobs = c["prefs"], c.get("user_context"), c["jobs"]
|
||||
targets = {str(j): int(s) for j, s in c["targets"].items()}
|
||||
# engine: white-box scores + the sift top-K the curator would see
|
||||
ranked = _rank.rank(prefs, ctx, [dict(j) for j in jobs])["opportunities"]
|
||||
eng = {j["id"]: j.get("matchScore", 0) for j in ranked}
|
||||
top, _ = _sift.sift(prefs, ctx, [dict(j) for j in jobs], k=k)
|
||||
top_ids = {j["id"] for j in top}
|
||||
|
||||
score_pairs = [(eng[jid], targets[jid]) for jid in eng if jid in targets]
|
||||
all_score_pairs += score_pairs
|
||||
# membership recall: of target-great jobs, how many made the sift top-K
|
||||
great = [jid for jid, t in targets.items() if t >= GREAT and jid in eng]
|
||||
in_top = [jid for jid in great if jid in top_ids]
|
||||
recall = round(len(in_top) / len(great), 2) if great else None
|
||||
if recall is not None:
|
||||
recalls.append(recall)
|
||||
rows.append({
|
||||
"case": c["label"], "n_jobs": len(jobs), "n_great": len(great),
|
||||
"great_in_topK": len(in_top), "membership_recall": recall,
|
||||
"score_mae": _mae(score_pairs),
|
||||
})
|
||||
|
||||
agg = {
|
||||
"sift_top_k": k,
|
||||
"overall_score_mae": _mae(all_score_pairs),
|
||||
"overall_membership_recall": round(sum(recalls) / len(recalls), 2) if recalls else None,
|
||||
"n_cases": len(cases), "n_pairs": len(all_score_pairs),
|
||||
}
|
||||
if emit_json:
|
||||
print(json.dumps({"cases": rows, "aggregate": agg}, indent=2))
|
||||
return
|
||||
print(f"\n Regression — engine scoring vs target (SIFT_TOP_K={k})")
|
||||
print(" " + "-" * 78)
|
||||
print(f" {'case':<40}{'jobs':>5}{'great':>7}{'in-topK':>9}{'recall':>8}{'MAE':>6}")
|
||||
for r in rows:
|
||||
print(f" {r['case'][:40]:<40}{r['n_jobs']:>5}{r['n_great']:>7}{r['great_in_topK']:>9}"
|
||||
f"{str(r['membership_recall']):>8}{str(r['score_mae']):>6}")
|
||||
print(" " + "-" * 78)
|
||||
print(f" AGGREGATE → score MAE {agg['overall_score_mae']} · "
|
||||
f"sift membership recall {agg['overall_membership_recall']} "
|
||||
f"({agg['n_pairs']} pairs, {agg['n_cases']} cases)\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(emit_json="--json" in sys.argv)
|
||||
93
tests/test_contracts.py
Normal file
93
tests/test_contracts.py
Normal file
@@ -0,0 +1,93 @@
|
||||
"""Phase-0 contract checks — the guardrails that catch silent failures BEFORE they ship.
|
||||
|
||||
These assert the load-bearing contracts the engine-improvement plan depends on:
|
||||
1. BOARDS ↔ NORMALIZERS alignment (an enabled board with no normalizer = silent drop).
|
||||
2. The funnel shapes (sift fusion weights, embed text fns, SIFT_TOP_K) are where the plan expects.
|
||||
3. overlap() returns None when a side has no skills (the floor-free contract).
|
||||
4. The frontend↔backend EXACT-STRING vocab contract: every ScoutPrefs ROLE/INDUSTRY option the UI
|
||||
offers must resolve to a coerce.py board code, else it silently degrades to a keyword fold.
|
||||
(Cross-repo read of growqr-demo's scout.ts; skipped if the sibling repo isn't present.)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.engine import search, normalize, sift, embed
|
||||
from app.engine.board_adapters import coerce as C
|
||||
from app.engine import skills as S
|
||||
from app.config import get_settings
|
||||
|
||||
|
||||
# ── 1. BOARDS ↔ NORMALIZERS alignment ────────────────────────────────────────
|
||||
def test_boards_have_normalizers():
|
||||
for board in search.BOARDS:
|
||||
assert board in normalize.NORMALIZERS, f"BOARDS has '{board}' with no NORMALIZERS entry"
|
||||
|
||||
|
||||
def test_enabled_boards_are_registered():
|
||||
enabled = [b.strip() for b in get_settings().BOARDS_ENABLED.split(",") if b.strip()]
|
||||
for b in enabled:
|
||||
assert b in search.BOARDS, f"BOARDS_ENABLED lists '{b}' which is not a registered board"
|
||||
assert b in normalize.NORMALIZERS, f"enabled board '{b}' has no normalizer"
|
||||
|
||||
|
||||
# ── 2. Funnel shapes (guards the Phase-2 edits) ──────────────────────────────
|
||||
def test_sift_fusion_weights_present_and_normalized():
|
||||
assert hasattr(sift, "W_WHITEBOX") and hasattr(sift, "W_VIBE")
|
||||
assert abs((sift.W_WHITEBOX + sift.W_VIBE) - 1.0) < 1e-9, "sift fusion weights must sum to 1"
|
||||
|
||||
|
||||
def test_embed_text_builders_exist():
|
||||
# the Phase-2 richer-embedding work edits these; assert they exist + return strings
|
||||
assert callable(embed._job_text) and callable(embed._profile_text)
|
||||
assert isinstance(embed._job_text({"title": "X", "organization": "Y"}), str)
|
||||
assert isinstance(embed._profile_text({"title": "X"}, None), str)
|
||||
|
||||
|
||||
def test_sift_top_k_configured():
|
||||
assert isinstance(get_settings().SIFT_TOP_K, int) and get_settings().SIFT_TOP_K > 0
|
||||
|
||||
|
||||
# ── 3. Floor-free skill contract ─────────────────────────────────────────────
|
||||
def test_overlap_none_when_a_side_is_empty():
|
||||
assert S.overlap([], ["python", "sql"]) is None, "empty user skills must yield None (not a floor)"
|
||||
assert S.overlap(["python"], []) is None, "empty job skills must yield None (not a floor)"
|
||||
# both present → a real dict with a score
|
||||
o = S.overlap(["python", "sql"], ["python", "django"])
|
||||
assert isinstance(o, dict) and "score" in o
|
||||
|
||||
|
||||
# ── 4. Frontend↔backend EXACT-STRING vocab contract ──────────────────────────
|
||||
# Values the UI may send that are intentionally NOT board codes (keyword-only / "open").
|
||||
_ROLE_KEYWORD_ONLY = {"Open to any"}
|
||||
_INDUSTRY_KEYWORD_ONLY = {"Any", "Open to any", ""}
|
||||
|
||||
_SCOUT_TS = Path(__file__).resolve().parents[2] / "growqr-demo/frontend/src/scout/data/scout.ts"
|
||||
|
||||
|
||||
def _extract_array(name: str, text: str) -> list[str]:
|
||||
m = re.search(rf"{name}\s*[:=]\s*\[(.*?)\]", text, re.S)
|
||||
if not m:
|
||||
return []
|
||||
return re.findall(r'"([^"]+)"', m.group(1))
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _SCOUT_TS.exists(), reason="growqr-demo frontend not present (cross-repo)")
|
||||
def test_frontend_roles_resolve_in_backend():
|
||||
text = _SCOUT_TS.read_text()
|
||||
roles = _extract_array("ROLE_OPTIONS", text)
|
||||
assert roles, "could not extract ROLE_OPTIONS from scout.ts — regex/contract drift"
|
||||
unmapped = [r for r in roles if r not in C.ROLE_NAUKRI_DEPT and r not in _ROLE_KEYWORD_ONLY]
|
||||
assert not unmapped, f"frontend ROLE options with no coerce.py code (silent keyword-fold): {unmapped}"
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _SCOUT_TS.exists(), reason="growqr-demo frontend not present (cross-repo)")
|
||||
def test_frontend_industries_resolve_in_backend():
|
||||
text = _SCOUT_TS.read_text()
|
||||
inds = _extract_array("INDUSTRY_OPTIONS", text)
|
||||
assert inds, "could not extract INDUSTRY_OPTIONS from scout.ts — regex/contract drift"
|
||||
unmapped = [i for i in inds if i not in C.INDUSTRY_NAUKRI and i not in _INDUSTRY_KEYWORD_ONLY]
|
||||
assert not unmapped, f"frontend INDUSTRY options with no coerce.py code (silent keyword-fold): {unmapped}"
|
||||
Reference in New Issue
Block a user