From 2b2d67b4c8520ed2ebc2aea9637be3ed37bdc6c1 Mon Sep 17 00:00:00 2001 From: raulgupta Date: Thu, 25 Jun 2026 16:30:06 +0530 Subject: [PATCH] Engine Phase 4: curate robustness + calibration + seniority inference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The measurement showed the USER-FACING Opus score is already well-calibrated — the real bug was curate FAILING and dropping to the bad white-box fallback: - curate.py: _salvage_kept() brace-scans + parses each kept object independently, so one bad char or a max-tokens truncation no longer drops the whole shortlist. max_tokens 4000 → 8000 (28 sifted jobs × full report cards overflowed). This fixed the sales case (was failing → fallback). - curate.py: calibration nudge in the prompt — a genuinely strong current-state match is a real low-80s, partial/stretch 60s-70s; don't under-sell strong matches into the 60s (without inflating weak ones). - normalize.py + search.py: _seniority_from_title() infers seniority from the title (conservative) when a board omits it, applied centrally in the sweep → the experience factor + Opus brief aren't blind. Verified (live Opus on the regression cases): tech MAE 9.8→4.5, sales FAILED→MAE 5.3 (kept 13), ops 4.2→3.3 — user-facing scores now MAE 3-5 with zero curate failures. 31 tests pass. --- app/engine/curate.py | 48 +++++++++++++++++++++++++++++++++++------ app/engine/normalize.py | 19 ++++++++++++++++ app/engine/search.py | 9 +++++++- 3 files changed, 69 insertions(+), 7 deletions(-) diff --git a/app/engine/curate.py b/app/engine/curate.py index 272cf6a..cab4ee1 100644 --- a/app/engine/curate.py +++ b/app/engine/curate.py @@ -33,8 +33,10 @@ in THIS job's actual skills/title. - growth (optional): {"text","from","to"} — the gap to close and the score lift closing it buys. - score: honest 0-100 overall fit. fit: "fit" (a current-state match) or "stretch" (a genuine reach). -Honesty rules: scores reflect reality — a partial match is in the 60s-70s, not the 90s. Ground every note \ -in the job's real content; invent nothing. Frame as helpful guidance, not a hiring verdict. +Honesty rules: scores reflect reality, on a calibrated scale. A genuinely strong CURRENT-STATE match \ +(role + seniority + location + most requirements align) is a real low-to-mid 80s; partial/stretch matches \ +sit in the 60s-70s; reserve high-80s/90s for a near-perfect fit. Don't inflate weak matches — but don't \ +under-sell strong ones into the 60s either. Ground every note in the job's real content; invent nothing. NON-TECH roles (sales, marketing, finance, HR, operations, support, supply-chain, legal, admin): the SPINE \ of the match is role/function fit + responsibility overlap + industry + seniority — NOT a skill-tag checklist. \ @@ -92,6 +94,30 @@ def _parse_json(text: str) -> dict: return json.loads(t) +def _salvage_kept(text: str) -> list[dict]: + """Recover valid kept-job objects from a malformed/truncated Opus JSON: brace-scan each top-level + {...} after "kept" and json.loads it on its own, skipping the one broken/cut-off object. Stops a + single bad character (or a max-tokens truncation) from dropping the WHOLE shortlist to the fallback.""" + i = text.find('"kept"') + s = text[i:] if i != -1 else text + out: list[dict] = [] + depth, start = 0, None + for idx, ch in enumerate(s): + if ch == "{": + if depth == 0: + start = idx + depth += 1 + elif ch == "}" and depth: + depth -= 1 + if depth == 0 and start is not None: + try: + out.append(json.loads(s[start : idx + 1])) + except Exception: # noqa: BLE001 — skip the broken object, keep the rest + pass + start = None + return out + + def _to_match(k: dict) -> MatchResult: dims = [ MatchDim(name=str(d["name"]), score=int(d["score"]), level=d["level"], note=d.get("note")) @@ -128,12 +154,22 @@ def curate(prefs: dict, ctx: dict | None, sifted_jobs: list[dict]) -> list[dict] {"role": "system", "content": _SYSTEM}, {"role": "user", "content": json.dumps(payload, ensure_ascii=False)}, ], - max_tokens=4000, + max_tokens=8000, # was 4000 — 28 sifted jobs × full report cards overflowed → truncation ) - data = _parse_json(resp.choices[0].message.content) - except Exception as e: # noqa: BLE001 — any failure → fall back to the sift - logger.warning("curate (Opus) failed, falling back to sift: %s", e) + content = resp.choices[0].message.content + except Exception as e: # noqa: BLE001 — API failure → fall back to the sift + logger.warning("curate (Opus) call failed, falling back to sift: %s", e) return None + # Robust parse: a single bad char / truncation must NOT drop the whole shortlist to the fallback. + try: + data = {"kept": _parse_json(content).get("kept") or []} + except Exception: # noqa: BLE001 + salvaged = _salvage_kept(content) + if not salvaged: + logger.warning("curate JSON unparseable + nothing salvageable — falling back to sift") + return None + logger.warning("curate JSON malformed; salvaged %d/%d kept items", len(salvaged), len(sifted_jobs)) + data = {"kept": salvaged} by_id = {j["id"]: j for j in sifted_jobs} out: list[dict] = [] diff --git a/app/engine/normalize.py b/app/engine/normalize.py index 4233f90..20d09b0 100644 --- a/app/engine/normalize.py +++ b/app/engine/normalize.py @@ -106,6 +106,25 @@ def _salary_from_amount(amount: int | None, per_year: bool) -> tuple[float | Non return lpa, f"₹{lpa:g}L/yr" +_TITLE_SENIORITY = [ + (r"\b(intern|trainee|fresher|graduate|entry[- ]?level|jr|junior)\b", "junior"), + (r"\b(associate)\b", "mid"), + (r"\b(senior|sr|staff|manager|specialist)\b", "senior"), + (r"\b(lead|principal|head|director|vp|chief|cxo|founding)\b", "lead"), +] + + +def _seniority_from_title(title: str | None): + """Infer seniority from a job title when the board omits it — so the experience factor + Opus brief + aren't blind. Conservative: only clear signal words; ambiguous titles stay None.""" + t = (title or "").lower() + hit = None + for pat, sen in _TITLE_SENIORITY: # last match wins → "Lead"/"Director" beats "Senior" + if re.search(pat, t): + hit = sen + return hit + + def _seniority_from_years(min_yrs: int | None): if min_yrs is None: return None diff --git a/app/engine/search.py b/app/engine/search.py index 9ece0b9..f768dff 100644 --- a/app/engine/search.py +++ b/app/engine/search.py @@ -65,7 +65,14 @@ BOARDS = { async def _fetch_board(key: str, prefs: dict, cache_mode: str | None = None): actor, build, norm = BOARDS[key] items = await run_actor(actor, build(prefs), cache_mode=cache_mode) - jobs = [sj for it in items if (sj := norm(it))] + jobs = [] + for it in items: + sj = norm(it) + if not sj: + continue + if not sj.get("seniority_level"): # board omitted it → infer from the title + sj["seniority_level"] = N._seniority_from_title(sj.get("title")) + jobs.append(sj) return key, jobs