Engine v2: embeddings sift + Opus curator + Postgres feed persistence

- A1: fire the hard filter (Apify is precise); embedding vibe-ranker (embed.py)
  blends with the floor-free white-box (rank.py) to sift ~90 -> top 18
- A2: Opus curator (curate.py) reads the 18 -> honest <=count + Gemini-voiced
  report cards; graceful fallback to the white-box + templated cards
- LLM via opencode.ai/zen gateway (llm.py): Opus chat + direct-OpenAI embeddings
- Run LLM work off the event loop (asyncio.to_thread) so the Redis response publishes
- C1: dedicated Postgres (app/db/) persists the per-user feed; get_scout_feed
  replays it so matches survive navigation/refresh
- match contract + report-card fields (schema.py); skills.py; tests/

tests/test_contract.py

@@ -0,0 +1,50 @@
+"""Phase 0 — the `match` contract test.
+
+The engine's `match` block must (a) validate against the Pydantic model and (b) serialize
+to EXACTLY the keys the frontend `scoutJobToMatchRole` reads (scout/lib/jobMatching.ts →
+`JobMatch`). If either drifts, backend ⇄ frontend have silently diverged — this test fails.
+"""
+import pytest
+from pydantic import ValidationError
+
+from app.engine.schema import Growth, MatchDim, MatchResult
+
+# The exact field sets the frontend reads. Keep in lockstep with `JobMatch` / `MatchDim`.
+FRONTEND_MATCH_KEYS = {"score", "fit", "archetype", "one_line", "reason", "breakdown", "growth",
+                       "coach_note", "proofReady", "factors"}
+FRONTEND_DIM_KEYS = {"name", "score", "level", "note"}
+FRONTEND_GROWTH_KEYS = {"text", "from", "to"}
+
+
+def _golden() -> MatchResult:
+    return MatchResult(
+        score=88, fit="stretch", reason="Matches 5/7 skills · Senior fit",
+        breakdown=[MatchDim(name="Product Management", score=92, level="Strong"),
+                   MatchDim(name="SQL", score=48, level="Light")],
+        growth=Growth(text="Add an SQL proof point", **{"from": 88, "to": 95}),
+        proofReady=True, factors={"skill": 0.71, "experience": 0.9, "salary": None},
+    )
+
+
+def test_match_block_keys_align_with_frontend():
+    d = _golden().as_dict()
+    assert set(d) == FRONTEND_MATCH_KEYS, f"match keys drifted: {set(d) ^ FRONTEND_MATCH_KEYS}"
+    assert set(d["breakdown"][0]) == FRONTEND_DIM_KEYS
+    assert set(d["growth"]) == FRONTEND_GROWTH_KEYS
+    # the `from` keyword must serialize under its alias, not `from_`
+    assert d["growth"]["from"] == 88 and d["growth"]["to"] == 95
+
+
+def test_constrained_enums_reject_bad_values():
+    with pytest.raises(ValidationError):
+        MatchResult(score=1, fit="maybe", reason="x")        # fit ∉ {fit, stretch}
+    with pytest.raises(ValidationError):
+        MatchDim(name="x", score=1, level="Great")           # level ∉ {Strong, Solid, Light}
+
+
+def test_minimal_match_is_valid_and_floor_free_shaped():
+    d = MatchResult(score=50, fit="fit", reason="ok").as_dict()
+    assert d["growth"] is None and d["breakdown"] == [] and d["factors"] == {}
+    # factors may carry None (an *absent* factor) — that's the coverage-aware contract, not a floor
+    d2 = MatchResult(score=50, fit="fit", reason="ok", factors={"salary": None}).as_dict()
+    assert d2["factors"]["salary"] is None

tests/test_coverage.py

@@ -0,0 +1,49 @@
+"""Phase 1 — the floor-free + coverage-aware invariants (the v1-collapse guardrails).
+
+This is the test that would have caught the old engine: every factor is float-or-`None`
+(no constant floors), and a sparse profile must NOT collapse all jobs to one neutral score.
+"""
+from app.engine import rank as R
+
+_JOBS = [
+    {"title": "Product Manager", "seniority_level": "senior", "location_city": "New Delhi",
+     "location_country": "India", "salary_lpa": 30,
+     "details": {"skills": ["Product Management", "SQL"], "industry": "Fintech",
+                 "description": "product manager roadmap stakeholders"}},
+    {"title": "Sales Lead", "seniority_level": "lead", "location_city": "Mumbai",
+     "location_country": "India", "salary_lpa": None,
+     "details": {"skills": ["Sales", "CRM"], "industry": "SaaS", "description": "sales quota targets"}},
+    {"title": "Product Manager", "seniority_level": "mid", "location_mode": "remote",
+     "location_city": "Remote", "details": {"skills": ["Product Management"], "description": "product manager"}},
+    {"title": "Designer", "seniority_level": None, "location_city": None, "salary_lpa": None, "details": {}},
+]
+
+
+def test_every_factor_returns_float_or_none():
+    p_full = R.profile_from({"title": "Product Manager", "location": ["New Delhi · India"],
+                             "industry": ["Fintech"], "experience": ["Senior"], "years": 9,
+                             "role": ["Product"], "targetComp": "₹25–30L"})
+    p_sparse = R.profile_from({"title": "Product Manager"})
+    for p in (p_full, p_sparse):
+        for job in _JOBS:
+            for name, fn in R.FACTORS.items():
+                v = fn(p, job)
+                assert v is None or (isinstance(v, float) and 0.0 <= v <= 1.0), f"{name} → {v!r}"
+
+
+def test_sparse_profile_does_not_collapse():
+    # The v1 bug: sparse profile → everyone ~60. Assert real score variance across varied jobs.
+    p = R.profile_from({"title": "Product Manager", "location": ["New Delhi · India"]})
+    us = [u for u in (R.utility(p, j)[0] for j in _JOBS) if u is not None]
+    assert len(us) >= 2 and (max(us) - min(us)) > 0.1, f"collapsed to a flat band: {us}"
+
+
+def test_absent_factor_redistributes_weight_not_floors():
+    # A job missing salary/industry/skills must still score on what IS present (no 60 injected).
+    p = R.profile_from({"title": "Product Manager", "location": ["New Delhi · India"]})
+    bare = {"title": "Product Manager", "location_city": "New Delhi", "location_country": "India",
+            "details": {"description": "product manager"}}
+    U, factors = R.utility(p, bare)
+    assert U is not None
+    assert factors["salary"] is None and factors["skill"] is None     # absent
+    assert factors["location"] == 1.0                                 # present, real

tests/test_curate.py

@@ -0,0 +1,25 @@
+"""A2 — curator: tolerant JSON parse, contract mapping, and the disabled→None safety net.
+(Offline — the live Opus call is exercised in the cached e2e verification.)"""
+from app.engine import curate as C
+
+
+def test_parse_json_strips_fences_and_prose():
+    assert C._parse_json('```json\n{"kept":[]}\n```') == {"kept": []}
+    assert C._parse_json('here: {"kept":[{"id":"1"}]} thanks')["kept"][0]["id"] == "1"
+
+
+def test_to_match_builds_full_contract():
+    m = C._to_match({
+        "id": "1", "score": 78, "fit": "fit", "archetype": "The Fit", "one_line": "strong but a gap",
+        "breakdown": [{"name": "Skill match", "score": 80, "level": "Strong", "note": "real SQL"}],
+        "coach_note": "lead with X", "growth": {"text": "close Y", "from": 78, "to": 85},
+    })
+    d = m.as_dict()
+    assert d["score"] == 78 and d["archetype"] == "The Fit" and d["fit"] == "fit"
+    assert d["breakdown"][0]["note"] == "real SQL" and d["coach_note"] == "lead with X"
+    assert d["growth"]["from"] == 78 and d["growth"]["to"] == 85 and d["proofReady"] is True
+
+
+def test_curate_disabled_returns_none(monkeypatch):
+    monkeypatch.setattr(C, "curate_enabled", lambda: False)
+    assert C.curate({"title": "PM"}, None, [{"id": "1", "title": "PM"}]) is None

tests/test_rank.py

@@ -0,0 +1,75 @@
+"""Phase 1 — factor primitives + end-to-end shortlist on CACHED data ($0)."""
+import glob
+import json
+
+import pytest
+
+from app.engine import normalize as N
+from app.engine import rank as R
+
+_NORM = {"blackfalcondata": N.naukri_feed_to_scoutjob, "shahidirfan": N.foundit_to_scoutjob,
+         "harvestapi": N.linkedin_to_scoutjob}
+
+
+def _cached_jobs() -> list[dict]:
+    jobs = []
+    for f in glob.glob(".apify_cache/*.json"):
+        fn = _NORM.get(f.split("/")[-1].split("_")[0])
+        if not fn:
+            continue
+        for it in json.load(open(f)):
+            sj = fn(it)
+            if sj:
+                jobs.append(sj)
+    return jobs
+
+
+# ── factor primitives ──
+def test_location_gate_and_tiers():
+    p = R.profile_from({"title": "PM", "location": ["New Delhi · India"]})
+    assert R.f_location(p, {"location_city": "New Delhi", "location_country": "India"}) == 1.0
+    assert R.f_location(p, {"location_city": "Bengaluru", "location_country": "India"}) == 0.6
+    assert R.f_location(p, {"location_city": "London", "location_country": "UK"}) == 0.0
+    assert R.f_location(p, {"location_mode": "remote"}) == 1.0
+    p_open = R.profile_from({"title": "PM"})
+    assert R.f_location(p_open, {"location_city": "X", "location_country": "Y"}) is None  # Anywhere → absent
+
+
+def test_experience_band():
+    p = R.profile_from({"title": "PM", "years": 5})
+    assert R.f_experience(p, {"seniority_level": "mid"}) == 1.0          # 5 in [3,8]
+    assert R.f_experience(p, {"seniority_level": "lead"}) < 1.0          # under-qualified
+    assert R.f_experience(p, {"seniority_level": None}) is None         # absent
+
+
+def test_e2e_ranks_all_gate_passing_no_early_cut():
+    jobs = _cached_jobs()
+    if len(jobs) < 30:
+        pytest.skip("no cached sweep data — run a search first")
+    prefs = {"title": "Product Manager", "role": ["Product"], "location": ["New Delhi · India"],
+             "experience": ["Mid-level"], "stretch": "balanced"}
+    res = R.rank(prefs, None, jobs)
+    opp = res["opportunities"]
+    # the cheap white-box does NOT cut to 25 — it ranks ALL gate-passing jobs (cut is Phase 3)
+    assert len(opp) >= 1
+    assert res["ranked"] == len(opp) and res["scanned"] == len(jobs)
+    scores = [j["matchScore"] for j in opp]
+    assert scores == sorted(scores, reverse=True)                        # ranked best-first
+    for j in opp:
+        m = j["match"]
+        assert m["fit"] in ("fit", "stretch") and isinstance(m["breakdown"], list)
+        assert m["score"] == j["matchScore"]
+    # determinism
+    res2 = R.rank(prefs, None, _cached_jobs())
+    assert [j["id"] for j in res2["opportunities"]] == [j["id"] for j in opp]
+
+
+def test_select_is_separate_and_capped():
+    # the final cut (Phase 3) — caps at ~25, floor so never empty, by fused matchScore
+    ranked = [{"id": str(i), "matchScore": 100 - i} for i in range(80)]
+    sel = R.select(ranked, "balanced")
+    assert 1 <= len(sel) <= R.CAP
+    assert [j["matchScore"] for j in sel] == sorted((j["matchScore"] for j in sel), reverse=True)
+    # floor: even if nothing clears the bar, return at least `floor`
+    low = [{"id": str(i), "matchScore": 10} for i in range(40)]
+    assert len(R.select(low, "safe")) == R.FLOOR

tests/test_sift.py

@@ -0,0 +1,25 @@
+"""A1 — sift: rank-fusion math + graceful degradation when the vibe-engineer is unavailable.
+(Offline — no network; the embedding path is exercised in the cached e2e verification.)"""
+from app.engine import sift as S
+
+
+def test_ranks_normalization():
+    r = S._ranks({"a": 0.9, "b": 0.5, "c": 0.1})
+    assert r["a"] == 1.0 and r["c"] == 0.0 and 0.0 < r["b"] < 1.0
+    assert S._ranks({"x": 0.5}) == {"x": 1.0}                 # single item → top
+
+
+def test_sift_degrades_to_whitebox_when_no_vibe(monkeypatch):
+    monkeypatch.setattr(S._embed, "vibe_scores", lambda *a, **k: None)   # embeddings unavailable
+    jobs = [{"id": str(i), "title": "Product Manager", "organization": "X",
+             "location_city": "New Delhi", "location_country": "India",
+             "details": {"description": "product manager roadmap"}} for i in range(30)]
+    prefs = {"title": "Product Manager", "location": ["New Delhi · India"]}
+    top, dbg = S.sift(prefs, None, jobs, k=18)
+    assert dbg["mode"] == "whitebox-only" and len(top) == 18
+    assert all("match" in j for j in top)                    # white-box match block rides along
+
+
+def test_sift_empty_input():
+    top, dbg = S.sift({"title": "PM"}, None, [], k=18)
+    assert top == [] and dbg["mode"] == "empty"

tests/test_skills.py

@@ -0,0 +1,24 @@
+"""Phase 1 — skill normalization + coverage-aware (floor-free) overlap."""
+from app.engine import skills as S
+
+
+def test_synonym_and_fold():
+    assert S.norm("React.js") == "react"
+    assert S.norm("K8s") == "kubernetes"
+    assert S.norm("  ML ") == "machine learning"
+    c = S.canon(["ML", "Python", "react.js"])
+    assert {"machine learning", "python", "react"} <= c
+
+
+def test_overlap_is_floor_free():
+    assert S.overlap([], ["python"]) is None          # empty user → absent (NOT 0 or 60)
+    assert S.overlap(["python"], []) is None           # empty job → absent
+    o = S.overlap(["python", "sql", "django"], ["Python", "Django", "AWS"])
+    assert o is not None and 0.0 < o["score"] <= 1.0
+    assert "python" in o["matched"] and "amazon web services" in o["missing"]
+
+
+def test_dims_levels():
+    rows = {r["name"]: r["level"] for r in S.dims(["product management"], ["Product Management", "SQL"], k=2)}
+    assert rows["Product Management"] == "Strong"
+    assert rows["Sql"] == "Light"