13faed0a06
Boards (signal-weighted, ~150/sweep): - Add TimesJobs (shahidirfan) — richest board: skills+salary+experience+offsite. Build/normalize + register; legacy city alias (New Delhi→Delhi, Bengaluru→Bangalore, Gurugram→Gurgaon) — verified live it returns 0 for "New Delhi", 35 for "Delhi". - Swap LinkedIn harvestapi → curious_coder (structured industry/jobFunction/applicants); harvestapi kept registered as linkedin_v1 fallback. - Drop Indeed from enabled (0% skills, redundant). Budgets: Foundit 40, TimesJobs 35, Naukri 30, LinkedIn 30, WorkIndia 15. Pool — cursorless freshness + safety: - REMOVE pagination/cursors: curious_coder only honors start=0 (start>0 → empty), proven; the rest are date feeds. Freshness now = boards' date-sort + the per-user seen-net + pool_save id-dedup. - Exhaustion guard: a refill adding < POOL_MIN_NEW_PER_REFILL new jobs flags the (user,query) exhausted → the gate relaxes the floor instead of block-fetching dupes (makes the 100/90 floor safe on niche queries). - Storage cap: pool_save trims beyond POOL_MAX_PER_QUERY freshest (kills DB swell). 72h TTL verified. - Thresholds 100/90 (deep pool, pay-for-volume). Scoring / cards: - Evidence-based fallback prose: when Haiku's cards stage gives nothing, the card uses Opus's REAL dimension notes (not a generic "Strong on X") + coverage logging + generic salvage parser. - "Skills & requirements" → "Skills fit" (consistent dimension labels). Bug fixes: - mark_seen dedups ids (was CardinalityViolationError on a duplicate id in one batch). Tests: deep stack contracts + committed board-sample fixture (no cache-pollution flakiness) + offline e2e + opt-in live e2e. 64 pass.
706 lines
30 KiB
Python
706 lines
30 KiB
Python
"""Per-board normalizers: raw Apify item → the frontend `ScoutJob` shape.
|
||
|
||
Slice-1a: no engine, so matchScore/note/qx are placeholders (matchScore is assigned
|
||
in search.py after merge). Each board's output differs — one mapper per board.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import html as _html
|
||
import json
|
||
import re
|
||
|
||
_PALETTE = ["#4F46E5", "#EA580C", "#0F6E56", "#DB2777", "#0EA5E9", "#7C3AED", "#D97706", "#059669"]
|
||
|
||
|
||
def _plain(s, limit: int = 1600) -> str:
|
||
"""HTML/Markdown → readable plain text for the detail modal (keeps paragraph breaks).
|
||
|
||
We surface ONLY what the board/employer provided — and strip the scraper's own
|
||
aggregator disclaimer boilerplate (we add nothing of our own).
|
||
"""
|
||
if not s:
|
||
return ""
|
||
# HTML → text
|
||
s = re.sub(r"<\s*(br|/p|/li|/div|/h\d|/tr)\s*[^>]*>", "\n", str(s), flags=re.I) # breaks → newlines
|
||
s = re.sub(r"<li[^>]*>", "• ", s, flags=re.I)
|
||
s = re.sub(r"<[^>]+>", "", s) # drop remaining tags
|
||
s = _html.unescape(s)
|
||
# Markdown leftovers (when the source is markdown, not HTML)
|
||
s = re.sub(r"(?m)^\s{0,3}#{1,6}\s*", "", s) # ATX headers → plain line
|
||
s = re.sub(r"(?m)^[ \t]*[-*+][ \t]+", "• ", s) # list bullets → •
|
||
s = re.sub(r"\*\*([^*]+)\*\*", r"\1", s) # **bold** → text
|
||
s = re.sub(r"`([^`]+)`", r"\1", s) # `code` → text
|
||
# Broken-tag remnants (e.g. a leaked "> from malformed HTML)
|
||
s = re.sub(r'"\s*>', " ", s)
|
||
s = s.lstrip(' "\'>')
|
||
# Drop the scraper's appended disclaimer boilerplate (NOT employer content)
|
||
s = re.split(r"\n?\s*Disclaimer\s*[:\-]", s, maxsplit=1, flags=re.I)[0]
|
||
# whitespace
|
||
s = re.sub(r"[ \t]+", " ", s)
|
||
s = re.sub(r"\n{3,}", "\n\n", s).strip()
|
||
return (s[:limit].rstrip() + "…") if len(s) > limit else s
|
||
|
||
|
||
def _compact(d: dict) -> dict:
|
||
"""Drop empty values so the details payload stays lean."""
|
||
return {k: v for k, v in d.items() if v not in (None, "", [], {})}
|
||
|
||
|
||
def _logo(org: str, url: str | None = None) -> dict:
|
||
org = (org or "").strip()
|
||
label = org[0].upper() if org else "?"
|
||
bg = _PALETTE[sum(map(ord, org)) % len(_PALETTE)] if org else "#9CA3AF"
|
||
d = {"label": label, "bg": bg}
|
||
# Real company logo from the actor (used by the UI; the label/bg stay as the fallback).
|
||
if url and isinstance(url, str) and url.startswith("http"):
|
||
d["url"] = url
|
||
return d
|
||
|
||
|
||
# actor field names for the company logo, across boards + nesting (Naukri=logoPath,
|
||
# LinkedIn=company.logo, Indeed=companyLogo, …). First http(s) value wins; None → letter badge.
|
||
_LOGO_KEYS = ("logoPath", "logoPathV3", "logo", "companyLogo", "company_logo", "logoUrl", "image", "companyImage")
|
||
|
||
|
||
def _logo_url(it: dict) -> str | None:
|
||
scopes = [it, it.get("company") or {}, it.get("jobDetails") or {},
|
||
(it.get("jobDetails") or {}).get("companyDetail") or {}]
|
||
for sc in scopes:
|
||
if not isinstance(sc, dict):
|
||
continue
|
||
for k in _LOGO_KEYS:
|
||
v = sc.get(k)
|
||
if isinstance(v, str) and v.startswith("http"):
|
||
return v
|
||
return None
|
||
|
||
|
||
# posting-date field across boards (Naukri createdDate · LinkedIn postedDate · Foundit date_posted).
|
||
# Feeds the dashboard "Apply window" (freshness urgency) + honest "posted Nd ago" age signals.
|
||
_POSTED_KEYS = ("postedDate", "createdDate", "date_posted", "postedAt", "listedAt", "datePosted", "published")
|
||
|
||
|
||
def _posted_date(it: dict) -> str | None:
|
||
scopes = [it, it.get("jobDetails") or {}]
|
||
for sc in scopes:
|
||
if not isinstance(sc, dict):
|
||
continue
|
||
for k in _POSTED_KEYS:
|
||
v = sc.get(k)
|
||
if isinstance(v, str) and v[:4].isdigit(): # ISO-ish "2026-06-18T…"
|
||
return v
|
||
return None
|
||
|
||
|
||
def _digits(s: str) -> int | None:
|
||
nums = re.sub(r"[^\d]", "", s or "")
|
||
return int(nums) if nums else None
|
||
|
||
|
||
def _salary_from_amount(amount: int | None, per_year: bool) -> tuple[float | None, str]:
|
||
"""amount in ₹ → (lpa, payLabel)."""
|
||
if not amount:
|
||
return None, "Not disclosed"
|
||
annual = amount if per_year else amount * 12
|
||
lpa = round(annual / 100_000, 1)
|
||
return lpa, f"₹{lpa:g}L/yr"
|
||
|
||
|
||
_TITLE_SENIORITY = [
|
||
(r"\b(intern|trainee|fresher|graduate|entry[- ]?level|jr|junior)\b", "junior"),
|
||
(r"\b(associate)\b", "mid"),
|
||
(r"\b(senior|sr|staff|manager|specialist)\b", "senior"),
|
||
(r"\b(lead|principal|head|director|vp|chief|cxo|founding)\b", "lead"),
|
||
]
|
||
|
||
|
||
def _seniority_from_title(title: str | None):
|
||
"""Infer seniority from a job title when the board omits it — so the experience factor + Opus brief
|
||
aren't blind. Conservative: only clear signal words; ambiguous titles stay None."""
|
||
t = (title or "").lower()
|
||
hit = None
|
||
for pat, sen in _TITLE_SENIORITY: # last match wins → "Lead"/"Director" beats "Senior"
|
||
if re.search(pat, t):
|
||
hit = sen
|
||
return hit
|
||
|
||
|
||
def _seniority_from_years(min_yrs: int | None):
|
||
if min_yrs is None:
|
||
return None
|
||
return ("junior" if min_yrs <= 1 else "junior" if min_yrs <= 4 else
|
||
"mid" if min_yrs <= 8 else "senior" if min_yrs <= 12 else "lead")
|
||
|
||
|
||
# free-text seniority (LinkedIn experienceLevel / ATS seniority) → our Seniority enum
|
||
_SENIORITY_TEXT = {
|
||
"internship": "junior", "entry level": "junior", "entry": "junior", "junior": "junior",
|
||
"associate": "mid", "mid": "mid", "mid-senior level": "senior", "senior": "senior",
|
||
"director": "lead", "executive": "lead", "lead": "lead",
|
||
}
|
||
|
||
|
||
def _seniority_from_text(s: str | None):
|
||
if not s:
|
||
return None
|
||
return _SENIORITY_TEXT.get(s.strip().lower())
|
||
|
||
|
||
_WORKMODE = {"remote": "remote", "hybrid": "hybrid", "on_site": "onsite", "onsite": "onsite"}
|
||
|
||
|
||
def naukri_to_scoutjob(it: dict, board: str = "Naukri") -> dict | None:
|
||
# The naukri-job-scraper (fetchDetails=True) nests the real fields under `jobDetails`.
|
||
jd = it.get("jobDetails") or {}
|
||
title = (jd.get("title") or "").strip()
|
||
org = ((jd.get("companyDetail") or {}).get("name") or "").strip()
|
||
if not title or not org:
|
||
return None
|
||
locs = jd.get("locations") or []
|
||
city = (locs[0].get("label") if locs and isinstance(locs[0], dict) else None) or None
|
||
# keySkills is {other:[{label}], preferred:[{label}], ...} — flatten the labels.
|
||
skills: list[str] = []
|
||
ks = jd.get("keySkills") or {}
|
||
if isinstance(ks, dict):
|
||
for bucket in ks.values():
|
||
for s in bucket or []:
|
||
lbl = (s.get("label") if isinstance(s, dict) else str(s)).strip()
|
||
if lbl and lbl not in skills:
|
||
skills.append(lbl)
|
||
# salaryDetail is already a dict here (not a JSON string).
|
||
lpa, pay = None, "Not disclosed"
|
||
sd = jd.get("salaryDetail") or {}
|
||
if isinstance(sd, dict) and not sd.get("hideSalary") and sd.get("maximumSalary"):
|
||
lpa, pay = _salary_from_amount(int(sd["maximumSalary"]), per_year=True)
|
||
# Offsite employer/ATS apply link first; the Naukri listing (staticUrl) is the fallback.
|
||
redirect = (
|
||
jd.get("applyRedirectUrl") or jd.get("companyApplyUrl")
|
||
or it.get("applyRedirectUrl") or it.get("companyApplyUrl")
|
||
)
|
||
return {
|
||
"id": f"naukri-{jd.get('jobId')}",
|
||
"title": title,
|
||
"organization": org,
|
||
"logo": _logo(org, _logo_url(it)),
|
||
"posted_date": _posted_date(it),
|
||
"location_city": city,
|
||
"location_country": "India",
|
||
"matchScore": 0, # placeholder — set after merge
|
||
"salary_lpa": lpa,
|
||
"payLabel": pay,
|
||
"seniority_level": _seniority_from_years(_digits(str(jd.get("minimumExperience", "")))),
|
||
"required_skills": skills[:6],
|
||
"tags": skills[:3],
|
||
"applicants": jd.get("applyCount"),
|
||
"note": f"Live from {board}",
|
||
"apply_url": redirect or jd.get("staticUrl"), # offsite employer link first, Naukri listing fallback
|
||
"offsite_apply": bool(redirect),
|
||
"qx": 10,
|
||
}
|
||
|
||
|
||
# Indeed (valig actor) leaves location.city empty — only lat/long. Map to the nearest Indian metro so
|
||
# the card + the location factor have a city (the board already filtered to the searched city anyway).
|
||
_INDIA_METROS = {
|
||
"Mumbai": (19.076, 72.877), "Delhi": (28.61, 77.21), "Bengaluru": (12.97, 77.59),
|
||
"Hyderabad": (17.385, 78.486), "Chennai": (13.083, 80.27), "Kolkata": (22.57, 88.36),
|
||
"Pune": (18.52, 73.856), "Ahmedabad": (23.03, 72.58), "Gurugram": (28.46, 77.03),
|
||
"Noida": (28.535, 77.39), "Jaipur": (26.91, 75.79), "Chandigarh": (30.73, 76.78),
|
||
"Kochi": (9.93, 76.27), "Indore": (22.72, 75.86), "Coimbatore": (11.02, 76.96),
|
||
}
|
||
|
||
|
||
def _city_from_latlon(lat, lon) -> str | None:
|
||
try:
|
||
lat, lon = float(lat), float(lon)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
best, bestd = None, 9e9
|
||
for name, (clat, clon) in _INDIA_METROS.items():
|
||
d = (lat - clat) ** 2 + (lon - clon) ** 2
|
||
if d < bestd:
|
||
bestd, best = d, name
|
||
return best if bestd < 1.5 else None # ~130km radius; farther → leave unknown, don't mislabel
|
||
|
||
|
||
def indeed_to_scoutjob(it: dict, board: str = "Indeed") -> dict | None:
|
||
# valig~indeed-jobs-scraper schema (7s vs misceres's 52s): title, employer.name, description.text,
|
||
# location{city/lat/long}, jobUrl (offsite employer site), url (Indeed listing), key, baseSalary.
|
||
if it.get("expired"):
|
||
return None
|
||
title = (it.get("title") or "").strip()
|
||
org = ((it.get("employer") or {}).get("name") or "").strip()
|
||
if not title or not org:
|
||
return None
|
||
loc = it.get("location") or {}
|
||
city = (loc.get("city") or "").strip() or _city_from_latlon(loc.get("latitude"), loc.get("longitude"))
|
||
country = loc.get("countryName") or "India"
|
||
job_url = it.get("jobUrl") or ""
|
||
offsite = bool(job_url and "indeed.com" not in job_url) # jobUrl = real offsite employer link
|
||
apply_url = job_url if offsite else it.get("url") # offsite first, Indeed listing fallback
|
||
bs = it.get("baseSalary") or {}
|
||
lpa, pay = None, "Not disclosed"
|
||
amt = bs.get("max") or bs.get("min")
|
||
if amt and bs.get("currencyCode") in (None, "INR"):
|
||
mult = {"YEAR": 1, "MONTH": 12, "WEEK": 52, "DAY": 260, "HOUR": 2080}.get(bs.get("unitOfWork"), 1)
|
||
lpa = round(amt * mult / 100000, 1)
|
||
pay = f"₹{lpa:g} LPA"
|
||
etypes = [str(v) for v in (it.get("jobTypes") or {}).values()]
|
||
occ = [str(v) for v in (it.get("occupations") or {}).values()]
|
||
mode = "remote" if any("remote" in t.lower() for t in etypes) else None
|
||
details = _compact({
|
||
"description": _plain((it.get("description") or {}).get("text")),
|
||
"employment_type": ", ".join(etypes) or None,
|
||
"role_category": occ[0] if occ else None,
|
||
})
|
||
return {
|
||
"id": f"indeed-{it.get('key')}",
|
||
"title": title,
|
||
"organization": org,
|
||
"logo": _logo(org),
|
||
"posted_date": (it.get("datePublished") or it.get("dateOnIndeed") or None),
|
||
"location_city": city,
|
||
"location_country": country,
|
||
"location_mode": mode,
|
||
"matchScore": 0,
|
||
"salary_lpa": lpa,
|
||
"payLabel": pay,
|
||
"required_skills": [], # Indeed exposes no skill list; description carries the signal
|
||
"tags": etypes[:3],
|
||
"applicants": None,
|
||
"note": f"Live from {board}",
|
||
"apply_url": apply_url,
|
||
"offsite_apply": offsite,
|
||
"details": details,
|
||
"qx": 10,
|
||
}
|
||
|
||
|
||
def linkedin_to_scoutjob(it: dict, board: str = "LinkedIn") -> dict | None:
|
||
title = (it.get("title") or "").strip()
|
||
org = ((it.get("company") or {}).get("name") or "").strip()
|
||
if not title or not org:
|
||
return None
|
||
loc = it.get("location") or {}
|
||
parsed = loc.get("parsed") or {} if isinstance(loc, dict) else {}
|
||
city = parsed.get("city") or None
|
||
country = parsed.get("country") or parsed.get("countryFull") or None
|
||
mode = _WORKMODE.get((it.get("workplaceType") or "").lower())
|
||
industries = [s for s in (it.get("industries") or []) if isinstance(s, str)]
|
||
sal = it.get("salary") or {}
|
||
pay = sal.get("text") if isinstance(sal, dict) and sal.get("text") else "Not disclosed"
|
||
co = it.get("company") or {}
|
||
details = _compact({
|
||
"description": _plain(it.get("descriptionText") or it.get("descriptionHtml")),
|
||
"experience": it.get("experienceLevel"),
|
||
"employment_type": (it.get("employmentType") or "").replace("_", " ").title() or None,
|
||
"industry": ", ".join(industries[:3]) or None,
|
||
"skills": industries,
|
||
"company_about": _plain(co.get("description"), 700) if isinstance(co, dict) else "",
|
||
})
|
||
# Prefer the OFFSITE employer/ATS apply link over the LinkedIn listing — the
|
||
# board URL can go stale, the employer's own apply page is the durable target.
|
||
_apply_method = it.get("applyMethod") or {}
|
||
offsite = (
|
||
it.get("companyApplyUrl")
|
||
or (_apply_method.get("companyApplyUrl") if isinstance(_apply_method, dict) else None)
|
||
or it.get("applyRedirectUrl")
|
||
)
|
||
return {
|
||
"id": f"linkedin-{it.get('id')}",
|
||
"title": title,
|
||
"organization": org,
|
||
"logo": _logo(org, _logo_url(it)),
|
||
"posted_date": _posted_date(it),
|
||
"location_city": city,
|
||
"location_country": country,
|
||
"location_mode": mode,
|
||
"matchScore": 0,
|
||
"salary_lpa": None,
|
||
"payLabel": pay,
|
||
"seniority_level": _seniority_from_text(it.get("experienceLevel")),
|
||
"required_skills": industries[:6],
|
||
"tags": industries[:3],
|
||
"applicants": it.get("applicants"),
|
||
"note": f"Live from {board}",
|
||
"apply_url": offsite or it.get("linkedinUrl"), # offsite employer link first, board link fallback
|
||
"offsite_apply": bool(offsite),
|
||
"details": details,
|
||
"qx": 10,
|
||
}
|
||
|
||
|
||
def ats_to_scoutjob(it: dict, board: str = "Direct") -> dict | None:
|
||
title = (it.get("title") or "").strip()
|
||
org = (it.get("company") or "").strip()
|
||
if not title or not org:
|
||
return None
|
||
org = org.replace("-", " ").replace("_", " ").title() # ats slugs ("stripe") → "Stripe"
|
||
parts = [p.strip() for p in (it.get("location") or "").split(",") if p.strip()]
|
||
city = parts[0] if parts else None
|
||
mode = _WORKMODE.get((it.get("remote_type") or "").lower()) or ("remote" if it.get("remote") else None)
|
||
dept = [d for d in [it.get("department"), it.get("team")] if d]
|
||
return {
|
||
"id": f"ats-{it.get('global_id') or it.get('job_id')}",
|
||
"title": title,
|
||
"organization": org,
|
||
"logo": _logo(org, _logo_url(it)),
|
||
"posted_date": _posted_date(it),
|
||
"location_city": city,
|
||
"location_country": None,
|
||
"location_mode": mode,
|
||
"matchScore": 0,
|
||
"salary_lpa": None,
|
||
"payLabel": "Not disclosed",
|
||
"seniority_level": _seniority_from_text(it.get("seniority")),
|
||
"required_skills": dept[:6],
|
||
"tags": dept[:3],
|
||
"applicants": None,
|
||
"note": f"Live from {board}", # direct employer posting → real offsite apply link
|
||
"apply_url": it.get("apply_url") or it.get("url"),
|
||
"offsite_apply": bool(it.get("apply_url") or it.get("url")), # ATS direct posting = employer's own apply page
|
||
"qx": 10,
|
||
}
|
||
|
||
|
||
def naukri_feed_to_scoutjob(it: dict, board: str = "Naukri") -> dict | None:
|
||
"""blackfalcondata/naukri-jobs-feed (fetchDetails=True) — flat shape with a real
|
||
offsite `applyRedirectUrl` when the employer uses an external ATS."""
|
||
title = (it.get("title") or "").strip()
|
||
org = (it.get("companyName") or it.get("staticCompanyName") or "").strip()
|
||
if not title or not org:
|
||
return None
|
||
loc = it.get("location") or ""
|
||
city = (loc.split("/")[0].split(",")[0].strip() or None) if isinstance(loc, str) else None
|
||
# keySkills = {preferred:[{label}], other:[{label}]} — flatten labels.
|
||
skills: list[str] = []
|
||
ks = it.get("keySkills") or {}
|
||
if isinstance(ks, dict):
|
||
for bucket in ks.values():
|
||
for s in bucket or []:
|
||
lbl = (s.get("label") if isinstance(s, dict) else str(s)).strip()
|
||
if lbl and lbl not in skills:
|
||
skills.append(lbl)
|
||
elif isinstance(ks, list):
|
||
skills = [str(s) for s in ks if s]
|
||
lpa, pay = None, "Not disclosed"
|
||
if not it.get("salaryHidden") and it.get("salaryMax"):
|
||
lpa, pay = _salary_from_amount(int(it["salaryMax"]), per_year=True)
|
||
redirect = it.get("applyRedirectUrl")
|
||
education = "; ".join(it.get("educationUG") or []) or it.get("degreeCombination") or None
|
||
details = _compact({
|
||
"description": _plain(it.get("descriptionMarkdown") or it.get("description") or it.get("shortDescription")),
|
||
"experience": it.get("experienceText"),
|
||
"employment_type": it.get("employmentType"),
|
||
"education": education,
|
||
"industry": it.get("industry"),
|
||
"role_category": it.get("roleCategory"),
|
||
"skills": skills,
|
||
"company_about": _plain(it.get("companyDescription"), 700),
|
||
"vacancies": it.get("vacancy"),
|
||
})
|
||
return {
|
||
"id": f"naukri-{it.get('jobId')}",
|
||
"title": title.title() if title.isupper() else title,
|
||
"organization": org,
|
||
"logo": _logo(org, _logo_url(it)),
|
||
"posted_date": _posted_date(it),
|
||
"location_city": city,
|
||
"location_country": "India",
|
||
"location_mode": _WORKMODE.get((it.get("wfhType") or "").lower()),
|
||
"matchScore": 0,
|
||
"salary_lpa": lpa,
|
||
"payLabel": pay,
|
||
"seniority_level": _seniority_from_years(it.get("minimumExperience")),
|
||
"required_skills": skills[:6],
|
||
"tags": skills[:3],
|
||
"applicants": it.get("applyCount"),
|
||
"note": f"Live from {board}",
|
||
"apply_url": redirect or it.get("staticUrl") or it.get("portalUrl"),
|
||
"offsite_apply": bool(redirect), # True = real 3rd-party employer/ATS link (auto-applyable)
|
||
"details": details,
|
||
"qx": 10,
|
||
}
|
||
|
||
|
||
def foundit_to_scoutjob(it: dict, board: str = "Foundit") -> dict | None:
|
||
title = (it.get("title") or "").strip()
|
||
org = (it.get("company") or "").strip()
|
||
if not title or not org:
|
||
return None
|
||
loc = it.get("location") or ""
|
||
city = (loc.split("/")[0].split(",")[0].strip() or None) if isinstance(loc, str) else None
|
||
skills = it.get("skills") or []
|
||
if isinstance(skills, str):
|
||
skills = [s.strip() for s in skills.split(",") if s.strip()]
|
||
skills = [str(s) for s in skills if s]
|
||
seniority = None
|
||
m = re.search(r"\d+", str(it.get("experience") or ""))
|
||
if m:
|
||
seniority = _seniority_from_years(int(m.group()))
|
||
redirect = it.get("apply_url")
|
||
details = _compact({
|
||
"description": _plain(it.get("description_text") or it.get("description_html")),
|
||
"experience": it.get("experience"),
|
||
"employment_type": it.get("employment_type"),
|
||
"industry": it.get("industry"),
|
||
"role_category": it.get("function"),
|
||
"skills": skills,
|
||
})
|
||
return {
|
||
"id": f"foundit-{it.get('job_id')}",
|
||
"title": title,
|
||
"organization": org,
|
||
"logo": _logo(org, _logo_url(it)),
|
||
"posted_date": _posted_date(it),
|
||
"location_city": city,
|
||
"location_country": "India",
|
||
"matchScore": 0,
|
||
"salary_lpa": None,
|
||
"payLabel": "Not disclosed",
|
||
"seniority_level": seniority,
|
||
"required_skills": skills[:6],
|
||
"tags": [it.get("function")][:1] if it.get("function") else skills[:3],
|
||
"applicants": None,
|
||
"note": f"Live from {board}",
|
||
"apply_url": redirect or it.get("url"),
|
||
"offsite_apply": bool(redirect),
|
||
"details": details,
|
||
"qx": 10,
|
||
}
|
||
|
||
|
||
# LinkedIn seniorityLevel → our band (often "Not Applicable" for India → None, then title-inferred centrally).
|
||
_LI_SENIORITY = {"internship": "junior", "entry level": "junior", "associate": "mid",
|
||
"mid-senior level": "senior", "director": "lead", "executive": "lead"}
|
||
|
||
|
||
def linkedin_cc_to_scoutjob(it: dict, board: str = "LinkedIn") -> dict | None:
|
||
"""curious_coder~linkedin-jobs-scraper — structured INDUSTRY + jobFunction (role) + applicantsCount.
|
||
(seniorityLevel is present but often 'Not Applicable' for India; we map it when real, else infer.)"""
|
||
title = (it.get("title") or "").strip()
|
||
org = (it.get("companyName") or "").strip()
|
||
if not title or not org:
|
||
return None
|
||
# location is "City, State, Country" — first part is the city, last is the country.
|
||
parts = [p.strip() for p in str(it.get("location") or "").split(",") if p.strip()]
|
||
city = parts[0] if parts else None
|
||
country = parts[-1] if len(parts) >= 2 else "India"
|
||
sen = _LI_SENIORITY.get(str(it.get("seniorityLevel") or "").strip().lower()) # None if "Not Applicable"
|
||
inds = [s.strip() for s in str(it.get("industries") or "").split(",") if s.strip()]
|
||
fn = (it.get("jobFunction") or "").strip() or None
|
||
etype = (it.get("employmentType") or "").strip()
|
||
mode = "remote" if "remote" in etype.lower() else None
|
||
ext = it.get("applyUrl")
|
||
offsite = bool(ext and "linkedin.com" not in ext) # applyUrl (when present) = offsite employer
|
||
apply_url = ext if offsite else it.get("link") # else the LinkedIn job page
|
||
details = _compact({
|
||
"description": _plain(it.get("descriptionText") or it.get("descriptionHtml")),
|
||
"industry": inds[0] if inds else None,
|
||
"role_category": fn,
|
||
"employment_type": etype or None,
|
||
})
|
||
return {
|
||
"id": f"linkedin-{it.get('id')}",
|
||
"title": title,
|
||
"organization": org,
|
||
"logo": _logo(org, it.get("companyLogo")),
|
||
"posted_date": it.get("postedAt"),
|
||
"location_city": city,
|
||
"location_country": country,
|
||
"location_mode": mode,
|
||
"matchScore": 0,
|
||
"salary_lpa": None,
|
||
"payLabel": "Not disclosed",
|
||
"seniority_level": sen,
|
||
"required_skills": [], # LinkedIn exposes no skill tags; description + jobFunction carry it
|
||
"industry": inds[0] if inds else None,
|
||
"tags": [fn] if fn else [],
|
||
"applicants": it.get("applicantsCount"),
|
||
"note": f"Live from {board}",
|
||
"apply_url": apply_url,
|
||
"offsite_apply": offsite,
|
||
"details": details,
|
||
"qx": 10,
|
||
}
|
||
|
||
|
||
def timesjobs_to_scoutjob(it: dict, board: str = "TimesJobs") -> dict | None:
|
||
"""shahidirfan/Timesjobs-Scraper — the richest India board: skills + salary + experience +
|
||
description + an OFFSITE employer link (external_job_url). Clean structured fields."""
|
||
title = (it.get("title") or "").strip()
|
||
title = re.sub(r"^title:\s*", "", title, flags=re.I).strip() # some rows carry a stray "Title:" prefix
|
||
org = (it.get("company") or "").strip()
|
||
if not title or not org:
|
||
return None
|
||
if org.islower():
|
||
org = org.title()
|
||
# location is "City" or "City, City2" — take the first concrete city.
|
||
city = next((p.strip() for p in str(it.get("location") or "").split(",") if p.strip()), None)
|
||
skills = it.get("skills") or []
|
||
if isinstance(skills, str):
|
||
skills = [s.strip() for s in skills.split(",") if s.strip()]
|
||
skills = [str(s) for s in skills if s]
|
||
# experience "6 - 10 Yrs" → seniority from the LOW end of the band.
|
||
seniority = None
|
||
m = re.search(r"\d+", str(it.get("experience") or ""))
|
||
if m:
|
||
seniority = _seniority_from_years(int(m.group()))
|
||
# salary is usually the text "As per industry standard"; surface a ₹L band only if a real number is there.
|
||
sal_raw = str(it.get("salary") or "")
|
||
lpa, pay = (None, sal_raw or "Not disclosed")
|
||
if re.search(r"\d", sal_raw):
|
||
lpa, pay = _salary_from_amount(_digits(sal_raw), per_year=True)
|
||
types = [t.strip() for t in str(it.get("job_type") or "").split(",") if t.strip()]
|
||
mode = "remote" if any("remote" in t.lower() for t in types) else None
|
||
ext = it.get("external_job_url")
|
||
offsite = bool(ext and "timesjobs.com" not in (ext or "")) # external_job_url = real employer site
|
||
apply_url = ext if offsite else it.get("url")
|
||
details = _compact({
|
||
"description": _plain(it.get("description_text") or it.get("description_html")),
|
||
"experience": it.get("experience"),
|
||
"employment_type": ", ".join(types) or None,
|
||
"company_about": _plain(it.get("company_description"), limit=400),
|
||
"skills": skills,
|
||
})
|
||
return {
|
||
"id": f"timesjobs-{it.get('job_id')}",
|
||
"title": title,
|
||
"organization": org,
|
||
"logo": _logo(org),
|
||
"posted_date": it.get("date_posted"),
|
||
"location_city": city,
|
||
"location_country": "India",
|
||
"location_mode": mode,
|
||
"matchScore": 0,
|
||
"salary_lpa": lpa,
|
||
"payLabel": pay,
|
||
"seniority_level": seniority,
|
||
"required_skills": skills[:8],
|
||
"tags": skills[:3],
|
||
"applicants": None,
|
||
"note": f"Live from {board}",
|
||
"apply_url": apply_url,
|
||
"offsite_apply": offsite,
|
||
"details": details,
|
||
"qx": 10,
|
||
}
|
||
|
||
|
||
def workindia_to_scoutjob(it: dict, board: str = "WorkIndia") -> dict | None:
|
||
"""shahidirfan/workindia-jobs-scraper (includeDetails=True) — India blue/grey-collar non-tech.
|
||
Verified live (Phase 0): returns real `skills` + `profile_job_description`. Apply is on-platform."""
|
||
title = (it.get("profile_job_title") or "").strip()
|
||
org = (it.get("branch_company_name") or "").strip()
|
||
if not title or not org:
|
||
return None
|
||
city = (str(it.get("branch_location_city_name") or "").strip() or None)
|
||
skills = it.get("skills") or []
|
||
if isinstance(skills, str):
|
||
skills = [s.strip() for s in skills.split(",") if s.strip()]
|
||
skills = [str(s) for s in skills if s]
|
||
seniority = None
|
||
m = re.search(r"\d+", str(it.get("job_experience") or it.get("experience") or ""))
|
||
if m:
|
||
seniority = _seniority_from_years(int(m.group()))
|
||
# Blue/grey-collar pay is usually monthly ₹ and noisy — show the structure text, don't risk the ₹L band.
|
||
details = _compact({
|
||
"description": _plain(it.get("profile_job_description") or it.get("profile_html_description")
|
||
or it.get("profile_short_description")),
|
||
"experience": it.get("job_experience") or it.get("experience"),
|
||
"employment_type": it.get("employment_type"),
|
||
"education": it.get("degree") or it.get("profile_qualification_required"),
|
||
"industry": it.get("profile_industry_display_name"),
|
||
"skills": skills,
|
||
})
|
||
return {
|
||
"id": f"workindia-{it.get('job_id') or it.get('id')}",
|
||
"title": title.title() if title.isupper() else title,
|
||
"organization": org,
|
||
"logo": _logo(org, _logo_url(it)),
|
||
"posted_date": it.get("job_published_on_platform") or it.get("created_at"),
|
||
"location_city": city,
|
||
"location_country": "India",
|
||
"matchScore": 0,
|
||
"salary_lpa": None,
|
||
"payLabel": it.get("profile_salary_structure") or "Not disclosed",
|
||
"seniority_level": seniority,
|
||
"required_skills": skills[:6],
|
||
"tags": skills[:3],
|
||
"applicants": None,
|
||
"note": f"Live from {board}",
|
||
# source_url is a GENERIC listing page (same for every job) → would dedup-collapse the deck.
|
||
# Build a per-job URL from job_id so each is unique AND clickable.
|
||
"apply_url": (f"https://www.workindia.in/jobs/{it.get('job_id')}/"
|
||
if it.get("job_id") else it.get("source_url")),
|
||
"offsite_apply": False, # WorkIndia is a platform — apply on WorkIndia
|
||
"details": details,
|
||
"qx": 10,
|
||
}
|
||
|
||
|
||
def wellfound_to_scoutjob(it: dict, board: str = "Wellfound") -> dict | None:
|
||
"""blackfalcondata/wellfound-scraper (enrichDetail=True) — startup roles with skills, equity,
|
||
funding stage. Apply is on-platform (Wellfound), so apply_url = the listing (offsite_apply=False)."""
|
||
title = (it.get("title") or "").strip()
|
||
org = (it.get("companyName") or "").strip()
|
||
if not title or not org:
|
||
return None
|
||
locs = it.get("locationNames") or []
|
||
city = (str(locs[0]).split(",")[0].strip() or None) if isinstance(locs, list) and locs else None
|
||
skills = [str(s) for s in (it.get("skills") or []) if s]
|
||
# Salary → LPA only when the listing is in INR (don't pollute the India band with USD); else the
|
||
# raw compensation string is the honest label.
|
||
lpa, pay = None, "Not disclosed"
|
||
cur = (it.get("salaryCurrency") or "").upper()
|
||
if it.get("salaryMax") and cur == "INR":
|
||
lpa, pay = _salary_from_amount(int(it["salaryMax"]), per_year=True)
|
||
elif it.get("compensation"):
|
||
pay = str(it["compensation"])
|
||
elif it.get("salaryMin") or it.get("salaryMax"):
|
||
pay = f"{cur or '$'} {it.get('salaryMin') or ''}–{it.get('salaryMax') or ''}".strip()
|
||
details = _compact({
|
||
"description": _plain(it.get("description")),
|
||
"employment_type": (it.get("jobType") or "").replace("_", " ").title() or None,
|
||
"skills": skills,
|
||
"funding_stage": it.get("companyFundingStage"),
|
||
"equity": it.get("equityMin") or it.get("equity"),
|
||
"visa": it.get("visaSponsorship"),
|
||
})
|
||
return {
|
||
"id": f"wellfound-{it.get('id')}",
|
||
"title": title,
|
||
"organization": org,
|
||
"logo": _logo(org, _logo_url(it)),
|
||
"posted_date": _posted_date(it),
|
||
"location_city": city,
|
||
"location_country": None,
|
||
"location_mode": "remote" if it.get("remote") else None,
|
||
"matchScore": 0,
|
||
"salary_lpa": lpa,
|
||
"payLabel": pay,
|
||
"required_skills": skills[:6],
|
||
"tags": skills[:3],
|
||
"applicants": None,
|
||
"note": f"Live from {board}",
|
||
"apply_url": it.get("portalUrl"),
|
||
"offsite_apply": False, # apply happens on Wellfound (on-platform), not an offsite employer redirect
|
||
"details": details,
|
||
"qx": 10,
|
||
}
|
||
|
||
|
||
# board key → normalizer
|
||
NORMALIZERS = {
|
||
"naukri": naukri_feed_to_scoutjob, # blackfalcondata (primary)
|
||
"foundit": foundit_to_scoutjob,
|
||
"linkedin": linkedin_cc_to_scoutjob, # curious_coder (structured industry+jobFunction+applicants)
|
||
"linkedin_v1": linkedin_to_scoutjob, # harvestapi (fallback)
|
||
"naukri_v1": naukri_to_scoutjob, # muhammetakkurtt (fallback)
|
||
"indeed": indeed_to_scoutjob,
|
||
"ats": ats_to_scoutjob,
|
||
"wellfound": wellfound_to_scoutjob, # blackfalcondata (startup roles)
|
||
"workindia": workindia_to_scoutjob, # shahidirfan (India blue/grey-collar non-tech)
|
||
"timesjobs": timesjobs_to_scoutjob, # shahidirfan (India — full signal: skills+salary+experience)
|
||
}
|