28cacfcc7c
The LLM (Opus score stage / Haiku cards stage) can return None or empty content, causing _salvage_objects to crash with 'NoneType' object has no attribute 'find' — which killed the curation pipeline and returned zero matches to the user. - Guard _salvage_objects and _salvage_kept against None/empty input - Add explicit empty-content checks in both stages with logging - Log content[:200] on unparseable score stage for QA diagnostics
302 lines
16 KiB
Python
302 lines
16 KiB
Python
"""The senior recruiter — Opus curates the sifted ~18 into the honest shortlist and writes the report
|
|
cards in the interview-service's Gemini video-analysis voice. **Opus's call is final.**
|
|
|
|
One gateway call: profile + the ~18 sifted jobs → strict JSON of the KEPT jobs (honest count ≤ the
|
|
pool — fewer when warranted, never padded), each carrying the full report-card contract. Returns `None`
|
|
on any failure (no key / bad JSON / API error) so the caller falls back to the sift + templated cards.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
|
|
from app.config import get_settings
|
|
from app.engine import rubric as _rubric
|
|
from app.engine.llm import curate_enabled, gateway_client
|
|
from app.engine.schema import Growth, MatchDim, MatchResult
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Two-stage curation, by capability + cost:
|
|
# STAGE 1 (Opus, _SYSTEM_SCORE): the JUDGMENT — rate the rubric dimensions on evidence. Integrity-
|
|
# critical, so it stays on Opus. Lean output (scores + terse notes, no prose) → tighter calibration
|
|
# AND ~75% less Opus output cost.
|
|
# STAGE 2 (Haiku, _SYSTEM_CARDS): the PROSE — write the report cards FROM Opus's scores+notes. Cheap
|
|
# output ($5/M vs $75/M); Haiku never judges fit or picks a score, only writes.
|
|
# The overall score is COMPUTED by the engine (rubric.aggregate) from Stage-1 ratings — never model-emitted.
|
|
|
|
_SYSTEM_SCORE = (
|
|
"You are Scout's senior recruiter scoring job fit. You receive a candidate PROFILE and a pre-sifted "
|
|
"list of JOBS (already narrowed to the right city + title by cheaper tools). Do two things:\n\n"
|
|
"1. CURATE — keep ONLY jobs genuinely worth this candidate's time. Be honest: if only 8 are real "
|
|
"matches, return 8. NEVER pad, drop weak/duplicate/off-target roles. The OVERALL score is COMPUTED "
|
|
"from your ratings — you do NOT choose it — so a job earns its place on merit, not a number you set.\n\n"
|
|
"2. For each KEPT job, RATE these fixed rubric dimensions 0-100, EACH against its anchor, each with a "
|
|
"SHORT evidence note (a few words) grounded in THIS job's real content. Rate every dimension that "
|
|
'applies (omit "industry" only if the candidate states no industry preference). Never omit a hard '
|
|
"dimension to flatter the score — you rate EVIDENCE, not an outcome.\n"
|
|
+ _rubric.prompt_block() + "\n\n"
|
|
"Rate against the ANCHORS honestly and use the FULL range: a same-function, same-city, same-level job "
|
|
"where the candidate genuinely meets most requirements earns dimension scores in the 90s (so the "
|
|
"computed overall is a confident 90s match — that hyper-relevance is the goal); a partial match earns "
|
|
"less; a weak dimension MUST score low and drag the overall down. Don't inflate, don't undersell.\n\n"
|
|
"NON-TECH roles (sales, finance, HR, ops, support, …): rate \"skills\" as the share of the job's stated "
|
|
"RESPONSIBILITIES the candidate can do — NOT a tag checklist; a missing skill-tag list is not a negative.\n\n"
|
|
'Also emit fit: "fit" (current-state match) or "stretch" (a genuine reach).\n\n'
|
|
"Respond with ONLY a JSON object (no prose, no markdown fences):\n"
|
|
'{"kept":[{"id","fit","dimensions":{"role":{"score","note"},"skills":{"score","note"},'
|
|
'"seniority":{"score","note"},"location":{"score","note"},"industry":{"score","note"},'
|
|
'"experience":{"score","note"}}}]}\n'
|
|
'Use each job\'s exact "id".'
|
|
)
|
|
|
|
_SYSTEM_CARDS = (
|
|
"You are Scout's encouraging career coach. For each scored job below you get its title, company, the "
|
|
"candidate's situation, the overall fit score, and the recruiter's per-dimension EVIDENCE notes. Write "
|
|
"a short REPORT CARD per job, grounded ONLY in those notes — invent nothing, never contradict a score.\n\n"
|
|
"Voice: warm, specific, honest (supportive video-coaching). For each id, write:\n"
|
|
' - archetype: a short characterful label — e.g. "The Stretch Worth Taking", "The Safe Powerhouse".\n'
|
|
" - one_line: ONE balanced sentence — name the real strength AND the honest gap. Never pure hype.\n"
|
|
" - coach_note: one actionable line — what to lead with, or what to shore up before applying.\n"
|
|
' - growth (optional): {"text","from","to"} — text = the gap to close; from/to = NUMBERS, the score '
|
|
"now and the realistic score after closing it (e.g. from 82 to 90).\n\n"
|
|
"Respond with ONLY a JSON object (no prose/fences):\n"
|
|
'{"cards":[{"id","archetype","one_line","coach_note","growth":{"text","from","to"}}]}\n'
|
|
'Use each job\'s exact "id".'
|
|
)
|
|
|
|
|
|
def _compact(d: dict) -> dict:
|
|
"""Drop None / empty values so the LLM payload stays lean (no null noise)."""
|
|
return {k: v for k, v in d.items() if v not in (None, "", [], {})}
|
|
|
|
|
|
def _profile_brief(prefs: dict, ctx: dict | None) -> dict:
|
|
ctx = ctx or {}
|
|
# Richer profile = the rubric can SEE that skills/experience are genuinely met (so those dimensions
|
|
# score what they deserve, not a guess). All fields are free (already in the resume context).
|
|
return _compact({
|
|
"target_title": prefs.get("title"),
|
|
"target_roles": prefs.get("role"),
|
|
"target_location": prefs.get("location"),
|
|
"seniority": prefs.get("experience"),
|
|
"years": prefs.get("years") or ctx.get("years_experience"),
|
|
"target_industry": prefs.get("industry"),
|
|
"skills": (ctx.get("skills") or [])[:20],
|
|
"current_role": ctx.get("current_role"),
|
|
"current_company": ctx.get("current_company"),
|
|
"past_titles": (ctx.get("past_titles") or ctx.get("titles") or [])[:4],
|
|
"experience_summary": str(ctx.get("experience_summary") or ctx.get("summary") or "")[:600] or None,
|
|
"education": ctx.get("education"),
|
|
"industries_worked": ctx.get("industries"),
|
|
"stretch_appetite": prefs.get("stretch", "balanced"),
|
|
})
|
|
|
|
|
|
def _job_brief(j: dict) -> dict:
|
|
d = j.get("details") or {}
|
|
return {
|
|
"id": j["id"],
|
|
"title": j.get("title"),
|
|
"company": j.get("organization"),
|
|
"location": f"{j.get('location_city') or ''} {j.get('location_country') or ''}".strip(),
|
|
"role_category": d.get("role_category"),
|
|
"industry": d.get("industry"),
|
|
"seniority": j.get("seniority_level"),
|
|
"pay": j.get("payLabel"),
|
|
"skills": (d.get("skills") or j.get("required_skills") or [])[:10],
|
|
"responsibilities": (d.get("description") or "")[:600], # enough for Opus to judge role-fit; keeps the payload lean
|
|
"prelim_score": j.get("matchScore"),
|
|
}
|
|
|
|
|
|
def _parse_json(text: str) -> dict:
|
|
"""Tolerant parse — strip markdown fences / surrounding prose if the model added any."""
|
|
t = (text or "").strip()
|
|
if t.startswith("```"):
|
|
t = t.split("```", 2)[1].removeprefix("json").strip() if "```" in t[3:] else t.strip("`")
|
|
i, j = t.find("{"), t.rfind("}")
|
|
if i != -1 and j != -1:
|
|
t = t[i : j + 1]
|
|
return json.loads(t)
|
|
|
|
|
|
def _salvage_objects(text: str | None, *, after: str | None = None) -> list[dict]:
|
|
"""Recover valid objects from malformed/truncated model JSON: brace-scan each top-level {...} and
|
|
json.loads it on its own, skipping the one broken/cut-off object. `after` anchors the scan past a key
|
|
(e.g. '"kept"'); omit it to scan the whole text (the cards stage uses '"cards"', not '"kept"')."""
|
|
if not text:
|
|
return []
|
|
s = text
|
|
if after:
|
|
i = text.find(after)
|
|
s = text[i:] if i != -1 else text
|
|
out: list[dict] = []
|
|
depth, start = 0, None
|
|
for idx, ch in enumerate(s):
|
|
if ch == "{":
|
|
if depth == 0:
|
|
start = idx
|
|
depth += 1
|
|
elif ch == "}" and depth:
|
|
depth -= 1
|
|
if depth == 0 and start is not None:
|
|
try:
|
|
out.append(json.loads(s[start : idx + 1]))
|
|
except Exception: # noqa: BLE001 — skip the broken object, keep the rest
|
|
pass
|
|
start = None
|
|
return out
|
|
def _salvage_kept(text: str | None) -> list[dict]:
|
|
"""Score-stage salvage — recover kept-job objects (anchored after the '"kept"' key)."""
|
|
return _salvage_objects(text, after='"kept"')
|
|
|
|
|
|
def _scored_card(k: dict) -> dict | None:
|
|
"""Stage-1 result → a partial card carrying the COMPUTED overall + the rubric breakdown (no prose yet).
|
|
Returns None for a malformed entry (no dimensions). The engine computes the score — never the model."""
|
|
raw = k.get("dimensions") or {}
|
|
dim_scores = {key: float(raw[key]["score"]) for key in _rubric.KEYS
|
|
if isinstance(raw.get(key), dict) and raw[key].get("score") is not None}
|
|
if not dim_scores:
|
|
return None
|
|
overall = _rubric.aggregate(dim_scores)
|
|
dims = [{"key": key, "name": _rubric.LABELS[key], "score": round(dim_scores[key]),
|
|
"level": _rubric.level_for(dim_scores[key]),
|
|
"note": (raw[key].get("note") if isinstance(raw.get(key), dict) else None)}
|
|
for key in _rubric.KEYS if key in dim_scores]
|
|
return {"id": k.get("id"), "overall": overall,
|
|
"fit": k.get("fit") or ("fit" if overall >= 75 else "stretch"), "dims": dims}
|
|
|
|
|
|
def _to_match(sc: dict, prose: dict | None) -> MatchResult:
|
|
"""Combine a Stage-1 scored card (sc) with Stage-2 prose (or templated fallback) → the MatchResult."""
|
|
prose = prose or {}
|
|
dims = [MatchDim(name=d["name"], score=d["score"], level=d["level"], note=d.get("note")) for d in sc["dims"]]
|
|
growth = None
|
|
g = prose.get("growth")
|
|
if g:
|
|
try:
|
|
growth = Growth(text=g["text"], **{"from": int(g["from"]), "to": int(g["to"])})
|
|
except (KeyError, ValueError, TypeError):
|
|
growth = None
|
|
# Fallback prose (when Haiku gave nothing) is built from Opus's REAL evidence notes — never a generic
|
|
# "Strong on X". A graceful card with real content, even if the writer stage failed.
|
|
strong = [d for d in sc["dims"] if d.get("note") and d["score"] >= 72]
|
|
weak = min((d for d in sc["dims"] if d.get("note")), key=lambda d: d["score"], default=None)
|
|
fb_line = "; ".join(d["note"] for d in strong[:2]) if strong else (
|
|
(max(sc["dims"], key=lambda d: d["score"])["note"] if sc["dims"] and sc["dims"][0].get("note")
|
|
else "A solid match worth a look."))
|
|
fb_coach = (f"Shore up {weak['name'].lower()}: {weak['note']}" if weak and weak["score"] < 72 else
|
|
"Lead with your strongest, most relevant experience.")
|
|
one_line = prose.get("one_line") or fb_line
|
|
return MatchResult(
|
|
score=sc["overall"], fit=sc["fit"],
|
|
archetype=prose.get("archetype") or ("The Strong Match" if sc["overall"] >= 80 else "Worth a Look"),
|
|
one_line=one_line, reason=one_line,
|
|
breakdown=dims, growth=growth, coach_note=prose.get("coach_note") or fb_coach,
|
|
proofReady=any(d.level == "Strong" for d in dims), factors={},
|
|
)
|
|
|
|
|
|
def _score_stage(prefs: dict, ctx: dict | None, sifted_jobs: list[dict], s) -> list[dict] | None:
|
|
"""STAGE 1 (Opus): rate the rubric dimensions → scored cards (computed overall, ≥ floor, best-first).
|
|
None on a hard failure (caller falls back to the sift). Empty list = Opus kept nothing strong."""
|
|
client = gateway_client()
|
|
payload = {"profile": _profile_brief(prefs, ctx), "jobs": [_job_brief(j) for j in sifted_jobs]}
|
|
try:
|
|
resp = client.chat.completions.create(
|
|
model=s.CURATE_MODEL,
|
|
messages=[{"role": "system", "content": _SYSTEM_SCORE},
|
|
{"role": "user", "content": json.dumps(payload, ensure_ascii=False)}],
|
|
max_tokens=4000, # scores + terse notes only (no prose) → far smaller than the old combined call
|
|
timeout=120,
|
|
)
|
|
content = resp.choices[0].message.content
|
|
except Exception as e: # noqa: BLE001
|
|
logger.warning("score stage (Opus) failed, falling back to sift: %s", e)
|
|
return None
|
|
if not content or not content.strip():
|
|
logger.warning("score stage LLM returned empty content — falling back to sift")
|
|
return None
|
|
try:
|
|
kept = _parse_json(content).get("kept") or []
|
|
except Exception: # noqa: BLE001
|
|
kept = _salvage_kept(content)
|
|
if not kept:
|
|
logger.warning("score stage unparseable — falling back to sift (content[:200]=%r)", content[:200])
|
|
return None
|
|
logger.warning("score stage JSON malformed; salvaged %d items", len(kept))
|
|
# Floor check on the RAW computed score (weak never shown), THEN the transparent presentation
|
|
# calibration (monotonic + floor-anchored, so ordering holds and weak never becomes strong).
|
|
cards = [c for c in (_scored_card(k) for k in kept) if c and c["overall"] >= s.MATCH_FLOOR]
|
|
if s.CALIBRATION_ENABLED:
|
|
for c in cards:
|
|
c["overall"] = _rubric.calibrate(c["overall"], floor=s.MATCH_FLOOR, gamma=s.CALIBRATION_GAMMA)
|
|
cards.sort(key=lambda c: -c["overall"]) # best-first by the (calibrated) score
|
|
return cards
|
|
|
|
|
|
def _cards_stage(prefs: dict, ctx: dict | None, scored: list[dict], by_id: dict, s) -> dict:
|
|
"""STAGE 2 (Haiku): write report-card prose FROM Opus's scores+notes. Cheap output, never judges.
|
|
Returns {id: prose}. On any failure → {} so _to_match uses templated prose over the real scores."""
|
|
client = gateway_client()
|
|
if not client or not scored:
|
|
return {}
|
|
briefs = [{"id": c["id"], "title": (by_id.get(c["id"]) or {}).get("title"),
|
|
"company": (by_id.get(c["id"]) or {}).get("organization"),
|
|
"overall": c["overall"], "fit": c["fit"],
|
|
"evidence": {d["key"]: d.get("note") for d in c["dims"]}} for c in scored]
|
|
payload = {"profile": _profile_brief(prefs, ctx), "scored_jobs": briefs}
|
|
try:
|
|
resp = client.chat.completions.create(
|
|
model=s.SUGGEST_MODEL, # Haiku — fast + ~15x cheaper output than Opus
|
|
messages=[{"role": "system", "content": _SYSTEM_CARDS},
|
|
{"role": "user", "content": json.dumps(payload, ensure_ascii=False)}],
|
|
max_tokens=4000, timeout=60,
|
|
)
|
|
content = resp.choices[0].message.content
|
|
except Exception as e: # noqa: BLE001 — prose is non-critical; degrade to templated cards
|
|
logger.warning("cards stage (Haiku) call failed, using evidence-based fallback prose: %s", e)
|
|
return {}
|
|
if not content or not content.strip():
|
|
logger.warning("cards stage LLM returned empty content — using evidence-based fallback prose")
|
|
return {}
|
|
try:
|
|
cards = _parse_json(content).get("cards") or []
|
|
except Exception: # noqa: BLE001
|
|
cards = _salvage_objects(content) # brace-scan any [{…}] (works regardless of the wrapper key)
|
|
prose = {c.get("id"): c for c in cards if c.get("id")}
|
|
# Coverage telemetry — a silent partial/empty cards stage is exactly what shipped generic reads.
|
|
if len(prose) < len(scored):
|
|
logger.warning("cards stage covered %d/%d jobs — %d will use evidence-based fallback prose",
|
|
len(prose), len(scored), len(scored) - len(prose))
|
|
return prose
|
|
|
|
|
|
def curate(prefs: dict, ctx: dict | None, sifted_jobs: list[dict]) -> list[dict] | None:
|
|
"""Two-stage curation: Opus SCORES the rubric (integrity) → Haiku WRITES the cards (cheap prose).
|
|
`None` only on a Stage-1 failure (caller falls back to the sift + templated cards)."""
|
|
s = get_settings()
|
|
if not (curate_enabled() and gateway_client() and sifted_jobs):
|
|
return None
|
|
scored = _score_stage(prefs, ctx, sifted_jobs, s)
|
|
if scored is None: # hard Opus failure → sift fallback
|
|
return None
|
|
by_id = {j["id"]: j for j in sifted_jobs}
|
|
prose = _cards_stage(prefs, ctx, scored, by_id, s) # {} on failure → templated prose, scores intact
|
|
out: list[dict] = []
|
|
for c in scored:
|
|
job = by_id.get(c["id"])
|
|
if not job:
|
|
continue
|
|
m = _to_match(c, prose.get(c["id"]))
|
|
if m.score < s.MATCH_FLOOR: # belt-and-suspenders (Stage 1 already floored)
|
|
continue
|
|
job["match"] = m.as_dict()
|
|
job["matchScore"] = m.score
|
|
out.append(job)
|
|
# Opus succeeded (None only on hard failure) → honor its curation even if empty ("no strong matches"),
|
|
# never fall back to UNFILTERED white-box cards that could surface a sub-floor match.
|
|
return out
|