Engine Phase 4: curate robustness + calibration + seniority inference

The measurement showed the USER-FACING Opus score is already well-calibrated — the real bug was curate
FAILING and dropping to the bad white-box fallback:
- curate.py: _salvage_kept() brace-scans + parses each kept object independently, so one bad char or a
  max-tokens truncation no longer drops the whole shortlist. max_tokens 4000 → 8000 (28 sifted jobs ×
  full report cards overflowed). This fixed the sales case (was failing → fallback).
- curate.py: calibration nudge in the prompt — a genuinely strong current-state match is a real low-80s,
  partial/stretch 60s-70s; don't under-sell strong matches into the 60s (without inflating weak ones).
- normalize.py + search.py: _seniority_from_title() infers seniority from the title (conservative) when a
  board omits it, applied centrally in the sweep → the experience factor + Opus brief aren't blind.

Verified (live Opus on the regression cases): tech MAE 9.8→4.5, sales FAILED→MAE 5.3 (kept 13),
ops 4.2→3.3 — user-facing scores now MAE 3-5 with zero curate failures. 31 tests pass.
This commit is contained in:
raulgupta
2026-06-25 16:30:06 +05:30
parent 7c5a88dd3f
commit 2b2d67b4c8
3 changed files with 69 additions and 7 deletions

View File

@@ -33,8 +33,10 @@ in THIS job's actual skills/title.
- growth (optional): {"text","from","to"} — the gap to close and the score lift closing it buys.
- score: honest 0-100 overall fit. fit: "fit" (a current-state match) or "stretch" (a genuine reach).
Honesty rules: scores reflect reality — a partial match is in the 60s-70s, not the 90s. Ground every note \
in the job's real content; invent nothing. Frame as helpful guidance, not a hiring verdict.
Honesty rules: scores reflect reality, on a calibrated scale. A genuinely strong CURRENT-STATE match \
(role + seniority + location + most requirements align) is a real low-to-mid 80s; partial/stretch matches \
sit in the 60s-70s; reserve high-80s/90s for a near-perfect fit. Don't inflate weak matches — but don't \
under-sell strong ones into the 60s either. Ground every note in the job's real content; invent nothing.
NON-TECH roles (sales, marketing, finance, HR, operations, support, supply-chain, legal, admin): the SPINE \
of the match is role/function fit + responsibility overlap + industry + seniority — NOT a skill-tag checklist. \
@@ -92,6 +94,30 @@ def _parse_json(text: str) -> dict:
return json.loads(t)
def _salvage_kept(text: str) -> list[dict]:
"""Recover valid kept-job objects from a malformed/truncated Opus JSON: brace-scan each top-level
{...} after "kept" and json.loads it on its own, skipping the one broken/cut-off object. Stops a
single bad character (or a max-tokens truncation) from dropping the WHOLE shortlist to the fallback."""
i = text.find('"kept"')
s = text[i:] if i != -1 else text
out: list[dict] = []
depth, start = 0, None
for idx, ch in enumerate(s):
if ch == "{":
if depth == 0:
start = idx
depth += 1
elif ch == "}" and depth:
depth -= 1
if depth == 0 and start is not None:
try:
out.append(json.loads(s[start : idx + 1]))
except Exception: # noqa: BLE001 — skip the broken object, keep the rest
pass
start = None
return out
def _to_match(k: dict) -> MatchResult:
dims = [
MatchDim(name=str(d["name"]), score=int(d["score"]), level=d["level"], note=d.get("note"))
@@ -128,12 +154,22 @@ def curate(prefs: dict, ctx: dict | None, sifted_jobs: list[dict]) -> list[dict]
{"role": "system", "content": _SYSTEM},
{"role": "user", "content": json.dumps(payload, ensure_ascii=False)},
],
max_tokens=4000,
max_tokens=8000, # was 4000 — 28 sifted jobs × full report cards overflowed → truncation
)
data = _parse_json(resp.choices[0].message.content)
except Exception as e: # noqa: BLE001 — any failure → fall back to the sift
logger.warning("curate (Opus) failed, falling back to sift: %s", e)
content = resp.choices[0].message.content
except Exception as e: # noqa: BLE001 — API failure → fall back to the sift
logger.warning("curate (Opus) call failed, falling back to sift: %s", e)
return None
# Robust parse: a single bad char / truncation must NOT drop the whole shortlist to the fallback.
try:
data = {"kept": _parse_json(content).get("kept") or []}
except Exception: # noqa: BLE001
salvaged = _salvage_kept(content)
if not salvaged:
logger.warning("curate JSON unparseable + nothing salvageable — falling back to sift")
return None
logger.warning("curate JSON malformed; salvaged %d/%d kept items", len(salvaged), len(sifted_jobs))
data = {"kept": salvaged}
by_id = {j["id"]: j for j in sifted_jobs}
out: list[dict] = []

View File

@@ -106,6 +106,25 @@ def _salary_from_amount(amount: int | None, per_year: bool) -> tuple[float | Non
return lpa, f"{lpa:g}L/yr"
_TITLE_SENIORITY = [
(r"\b(intern|trainee|fresher|graduate|entry[- ]?level|jr|junior)\b", "junior"),
(r"\b(associate)\b", "mid"),
(r"\b(senior|sr|staff|manager|specialist)\b", "senior"),
(r"\b(lead|principal|head|director|vp|chief|cxo|founding)\b", "lead"),
]
def _seniority_from_title(title: str | None):
"""Infer seniority from a job title when the board omits it — so the experience factor + Opus brief
aren't blind. Conservative: only clear signal words; ambiguous titles stay None."""
t = (title or "").lower()
hit = None
for pat, sen in _TITLE_SENIORITY: # last match wins → "Lead"/"Director" beats "Senior"
if re.search(pat, t):
hit = sen
return hit
def _seniority_from_years(min_yrs: int | None):
if min_yrs is None:
return None

View File

@@ -65,7 +65,14 @@ BOARDS = {
async def _fetch_board(key: str, prefs: dict, cache_mode: str | None = None):
actor, build, norm = BOARDS[key]
items = await run_actor(actor, build(prefs), cache_mode=cache_mode)
jobs = [sj for it in items if (sj := norm(it))]
jobs = []
for it in items:
sj = norm(it)
if not sj:
continue
if not sj.get("seniority_level"): # board omitted it → infer from the title
sj["seniority_level"] = N._seniority_from_title(sj.get("title"))
jobs.append(sj)
return key, jobs