89ca9ad647
On-demand Scout service (replaces the nightly aggregator): - FastAPI agent-mesh service; card name "matchmaking-service" (HTTP /a2a/tasks + Redis-stream worker matching the sibling-service pattern) - run_search: parallel multi-board sweep (Naukri/blackfalcondata + Foundit + LinkedIn), city-filtered at the board, cheapest-per-result first - per-board adapters + normalizers -> ScoutJob with rich read-only details and offsite apply links; recall mode + MVQ guard - result cache for dev replay ($0); per-board cost knobs; retry-on-5xx - research/: engine design, board cost economics, India-actor shortlist, POC
86 lines
3.6 KiB
Python
86 lines
3.6 KiB
Python
"""On-demand multi-board sweep: ScoutPrefs → all boards (parallel) → merge + dedup → ScoutJob[].
|
|
|
|
This is the retrieve layer (final shape). Slice-1a has NO engine — ranking is a placeholder
|
|
spread; the §3 cascade slots in here later without changing the output contract.
|
|
Degrade-don't-break: a board that errors/needs setup is skipped; the others still return.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
|
|
from app.config import get_settings
|
|
from app.engine.apify_client import run_actor
|
|
from app.engine.board_adapters import adapters as A
|
|
from app.engine.board_adapters import coerce as C
|
|
from app.engine import normalize as N
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
def _ats_companies() -> list[dict]:
|
|
return [{"company": c.strip()} for c in get_settings().ATS_COMPANIES.split(",") if c.strip()]
|
|
|
|
|
|
# board key → (actor_id, build_input, normalizer). "Balanced" India stack = naukri+foundit+linkedin,
|
|
# all city-filtered at the board. The rest stay registered (code ready) but off unless enabled.
|
|
BOARDS = {
|
|
"naukri": ("blackfalcondata~naukri-jobs-feed",
|
|
lambda p: A.build_naukri_feed_input(p, max_jobs=get_settings().NAUKRI_MAX_JOBS),
|
|
N.naukri_feed_to_scoutjob),
|
|
"foundit": ("shahidirfan~Foundit-Jobs-Scraper",
|
|
lambda p: A.build_foundit_input(p, results_wanted=get_settings().FOUNDIT_MAX_JOBS),
|
|
N.foundit_to_scoutjob),
|
|
"linkedin": ("harvestapi~linkedin-job-search",
|
|
lambda p: A.build_linkedin_input(p, recall=True, max_items=get_settings().LINKEDIN_MAX_JOBS),
|
|
N.linkedin_to_scoutjob),
|
|
# ── registered fallbacks / other lanes (off unless added to BOARDS_ENABLED) ──
|
|
"ats": ("bovi~greenhouse-lever-ashby-job-scraper",
|
|
lambda p: A.build_ats_input(p, recall=True, companies=_ats_companies(),
|
|
max_per_company=get_settings().ATS_MAX_PER_COMPANY),
|
|
N.ats_to_scoutjob),
|
|
"naukri_v1": ("muhammetakkurtt~naukri-job-scraper",
|
|
lambda p: A.build_naukri_input(p, recall=True, max_jobs=get_settings().NAUKRI_MAX_JOBS),
|
|
N.naukri_to_scoutjob),
|
|
"indeed": ("misceres~indeed-scraper",
|
|
lambda p: A.build_indeed_input(p, max_items=get_settings().INDEED_MAX_JOBS),
|
|
N.indeed_to_scoutjob),
|
|
}
|
|
|
|
|
|
async def _fetch_board(key: str, prefs: dict):
|
|
actor, build, norm = BOARDS[key]
|
|
items = await run_actor(actor, build(prefs))
|
|
jobs = [sj for it in items if (sj := norm(it))]
|
|
return key, jobs
|
|
|
|
|
|
async def run_sweep(prefs: dict) -> dict:
|
|
enabled = [b.strip() for b in get_settings().BOARDS_ENABLED.split(",") if b.strip() in BOARDS]
|
|
results = await asyncio.gather(*(_fetch_board(k, prefs) for k in enabled), return_exceptions=True)
|
|
|
|
merged: list[dict] = []
|
|
seen: set[str] = set()
|
|
sources: dict[str, int] = {}
|
|
for r in results:
|
|
if isinstance(r, Exception):
|
|
logger.warning("board fetch failed: %s", r)
|
|
continue
|
|
key, jobs = r
|
|
sources[key] = len(jobs)
|
|
for j in jobs:
|
|
dk = (j.get("apply_url") or "").strip() or f"{j['organization']}|{j['title']}|{j.get('location_city')}".lower()
|
|
if dk in seen:
|
|
continue
|
|
seen.add(dk)
|
|
merged.append(j)
|
|
|
|
# placeholder ranking (no engine yet): a descending spread so cards aren't all the same %
|
|
for i, j in enumerate(merged):
|
|
j["matchScore"] = max(55, 95 - i)
|
|
|
|
return {"opportunities": merged, "sources": sources}
|
|
|
|
|
|
def has_mvq(prefs: dict) -> bool:
|
|
return C.has_mvq(prefs)
|