Engine Phase 4: curate robustness + calibration + seniority inference
The measurement showed the USER-FACING Opus score is already well-calibrated — the real bug was curate FAILING and dropping to the bad white-box fallback: - curate.py: _salvage_kept() brace-scans + parses each kept object independently, so one bad char or a max-tokens truncation no longer drops the whole shortlist. max_tokens 4000 → 8000 (28 sifted jobs × full report cards overflowed). This fixed the sales case (was failing → fallback). - curate.py: calibration nudge in the prompt — a genuinely strong current-state match is a real low-80s, partial/stretch 60s-70s; don't under-sell strong matches into the 60s (without inflating weak ones). - normalize.py + search.py: _seniority_from_title() infers seniority from the title (conservative) when a board omits it, applied centrally in the sweep → the experience factor + Opus brief aren't blind. Verified (live Opus on the regression cases): tech MAE 9.8→4.5, sales FAILED→MAE 5.3 (kept 13), ops 4.2→3.3 — user-facing scores now MAE 3-5 with zero curate failures. 31 tests pass.
This commit is contained in:
@@ -33,8 +33,10 @@ in THIS job's actual skills/title.
|
||||
- growth (optional): {"text","from","to"} — the gap to close and the score lift closing it buys.
|
||||
- score: honest 0-100 overall fit. fit: "fit" (a current-state match) or "stretch" (a genuine reach).
|
||||
|
||||
Honesty rules: scores reflect reality — a partial match is in the 60s-70s, not the 90s. Ground every note \
|
||||
in the job's real content; invent nothing. Frame as helpful guidance, not a hiring verdict.
|
||||
Honesty rules: scores reflect reality, on a calibrated scale. A genuinely strong CURRENT-STATE match \
|
||||
(role + seniority + location + most requirements align) is a real low-to-mid 80s; partial/stretch matches \
|
||||
sit in the 60s-70s; reserve high-80s/90s for a near-perfect fit. Don't inflate weak matches — but don't \
|
||||
under-sell strong ones into the 60s either. Ground every note in the job's real content; invent nothing.
|
||||
|
||||
NON-TECH roles (sales, marketing, finance, HR, operations, support, supply-chain, legal, admin): the SPINE \
|
||||
of the match is role/function fit + responsibility overlap + industry + seniority — NOT a skill-tag checklist. \
|
||||
@@ -92,6 +94,30 @@ def _parse_json(text: str) -> dict:
|
||||
return json.loads(t)
|
||||
|
||||
|
||||
def _salvage_kept(text: str) -> list[dict]:
|
||||
"""Recover valid kept-job objects from a malformed/truncated Opus JSON: brace-scan each top-level
|
||||
{...} after "kept" and json.loads it on its own, skipping the one broken/cut-off object. Stops a
|
||||
single bad character (or a max-tokens truncation) from dropping the WHOLE shortlist to the fallback."""
|
||||
i = text.find('"kept"')
|
||||
s = text[i:] if i != -1 else text
|
||||
out: list[dict] = []
|
||||
depth, start = 0, None
|
||||
for idx, ch in enumerate(s):
|
||||
if ch == "{":
|
||||
if depth == 0:
|
||||
start = idx
|
||||
depth += 1
|
||||
elif ch == "}" and depth:
|
||||
depth -= 1
|
||||
if depth == 0 and start is not None:
|
||||
try:
|
||||
out.append(json.loads(s[start : idx + 1]))
|
||||
except Exception: # noqa: BLE001 — skip the broken object, keep the rest
|
||||
pass
|
||||
start = None
|
||||
return out
|
||||
|
||||
|
||||
def _to_match(k: dict) -> MatchResult:
|
||||
dims = [
|
||||
MatchDim(name=str(d["name"]), score=int(d["score"]), level=d["level"], note=d.get("note"))
|
||||
@@ -128,12 +154,22 @@ def curate(prefs: dict, ctx: dict | None, sifted_jobs: list[dict]) -> list[dict]
|
||||
{"role": "system", "content": _SYSTEM},
|
||||
{"role": "user", "content": json.dumps(payload, ensure_ascii=False)},
|
||||
],
|
||||
max_tokens=4000,
|
||||
max_tokens=8000, # was 4000 — 28 sifted jobs × full report cards overflowed → truncation
|
||||
)
|
||||
data = _parse_json(resp.choices[0].message.content)
|
||||
except Exception as e: # noqa: BLE001 — any failure → fall back to the sift
|
||||
logger.warning("curate (Opus) failed, falling back to sift: %s", e)
|
||||
content = resp.choices[0].message.content
|
||||
except Exception as e: # noqa: BLE001 — API failure → fall back to the sift
|
||||
logger.warning("curate (Opus) call failed, falling back to sift: %s", e)
|
||||
return None
|
||||
# Robust parse: a single bad char / truncation must NOT drop the whole shortlist to the fallback.
|
||||
try:
|
||||
data = {"kept": _parse_json(content).get("kept") or []}
|
||||
except Exception: # noqa: BLE001
|
||||
salvaged = _salvage_kept(content)
|
||||
if not salvaged:
|
||||
logger.warning("curate JSON unparseable + nothing salvageable — falling back to sift")
|
||||
return None
|
||||
logger.warning("curate JSON malformed; salvaged %d/%d kept items", len(salvaged), len(sifted_jobs))
|
||||
data = {"kept": salvaged}
|
||||
|
||||
by_id = {j["id"]: j for j in sifted_jobs}
|
||||
out: list[dict] = []
|
||||
|
||||
@@ -106,6 +106,25 @@ def _salary_from_amount(amount: int | None, per_year: bool) -> tuple[float | Non
|
||||
return lpa, f"₹{lpa:g}L/yr"
|
||||
|
||||
|
||||
_TITLE_SENIORITY = [
|
||||
(r"\b(intern|trainee|fresher|graduate|entry[- ]?level|jr|junior)\b", "junior"),
|
||||
(r"\b(associate)\b", "mid"),
|
||||
(r"\b(senior|sr|staff|manager|specialist)\b", "senior"),
|
||||
(r"\b(lead|principal|head|director|vp|chief|cxo|founding)\b", "lead"),
|
||||
]
|
||||
|
||||
|
||||
def _seniority_from_title(title: str | None):
|
||||
"""Infer seniority from a job title when the board omits it — so the experience factor + Opus brief
|
||||
aren't blind. Conservative: only clear signal words; ambiguous titles stay None."""
|
||||
t = (title or "").lower()
|
||||
hit = None
|
||||
for pat, sen in _TITLE_SENIORITY: # last match wins → "Lead"/"Director" beats "Senior"
|
||||
if re.search(pat, t):
|
||||
hit = sen
|
||||
return hit
|
||||
|
||||
|
||||
def _seniority_from_years(min_yrs: int | None):
|
||||
if min_yrs is None:
|
||||
return None
|
||||
|
||||
@@ -65,7 +65,14 @@ BOARDS = {
|
||||
async def _fetch_board(key: str, prefs: dict, cache_mode: str | None = None):
|
||||
actor, build, norm = BOARDS[key]
|
||||
items = await run_actor(actor, build(prefs), cache_mode=cache_mode)
|
||||
jobs = [sj for it in items if (sj := norm(it))]
|
||||
jobs = []
|
||||
for it in items:
|
||||
sj = norm(it)
|
||||
if not sj:
|
||||
continue
|
||||
if not sj.get("seniority_level"): # board omitted it → infer from the title
|
||||
sj["seniority_level"] = N._seniority_from_title(sj.get("title"))
|
||||
jobs.append(sj)
|
||||
return key, jobs
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user