538d2fc5f3
RCA of a prod "Naukri returned 0" on an AI-Architect search: NOT a cursor issue (cursors removed; naukri-feed is a date feed with no page param), NOT a city-name or keyword bug — the EXACT same input flaked 0 then returned 15 on retry. The scrape-based actors (Naukri-feed, curious_coder LinkedIn) intermittently return 0 under the site's anti-scrape rate-limiting, then recover. Fix: _fetch_board retries a board ONCE (fresh) when it returns 0. One guard covers BOTH the Naukri and LinkedIn intermittency (cleaner than per-board fallbacks). A genuinely-empty query just returns 0 again (cheap). 67 tests pass (3 new: recovers transient 0, no retry on success, no infinite loop).
52 lines
2.1 KiB
Python
52 lines
2.1 KiB
Python
"""Retry-on-empty guard — scrape-flaky boards (Naukri-feed, curious_coder LinkedIn) intermittently
|
|
return 0; one fresh retry rescues the transient case. RCA: the same input flaked 0 then returned 15."""
|
|
import asyncio
|
|
|
|
from app.engine import search
|
|
|
|
|
|
def _run(coro):
|
|
return asyncio.run(coro)
|
|
|
|
|
|
def test_retry_on_empty_recovers_transient_zero(monkeypatch):
|
|
calls = {"n": 0}
|
|
|
|
async def fake_run_actor(actor, inp, cache_mode=None):
|
|
calls["n"] += 1
|
|
# first call flakes (0), retry returns a real job
|
|
if calls["n"] == 1:
|
|
return []
|
|
return [{"profile_job_title": "Data Architect", "branch_company_name": "Acme",
|
|
"branch_location_city_name": "Delhi", "skills": ["python"], "job_id": "1"}]
|
|
|
|
monkeypatch.setattr(search, "run_actor", fake_run_actor)
|
|
key, jobs = _run(search._fetch_board("workindia", {"title": "Data Architect", "location": ["Delhi · India"]}))
|
|
assert calls["n"] == 2, "a board returning 0 must be retried exactly once"
|
|
assert len(jobs) == 1, "the retry's jobs are used"
|
|
|
|
|
|
def test_no_retry_when_first_call_succeeds(monkeypatch):
|
|
calls = {"n": 0}
|
|
|
|
async def fake_run_actor(actor, inp, cache_mode=None):
|
|
calls["n"] += 1
|
|
return [{"profile_job_title": "X", "branch_company_name": "Y",
|
|
"branch_location_city_name": "Delhi", "skills": [], "job_id": "1"}]
|
|
|
|
monkeypatch.setattr(search, "run_actor", fake_run_actor)
|
|
_run(search._fetch_board("workindia", {"title": "X", "location": ["Delhi · India"]}))
|
|
assert calls["n"] == 1, "a board that returns jobs is NOT retried (no extra cost)"
|
|
|
|
|
|
def test_genuinely_empty_returns_empty_after_one_retry(monkeypatch):
|
|
calls = {"n": 0}
|
|
|
|
async def fake_run_actor(actor, inp, cache_mode=None):
|
|
calls["n"] += 1
|
|
return [] # genuinely empty (e.g. WorkIndia for a tech role)
|
|
|
|
monkeypatch.setattr(search, "run_actor", fake_run_actor)
|
|
key, jobs = _run(search._fetch_board("workindia", {"title": "AI Architect", "location": ["Delhi · India"]}))
|
|
assert calls["n"] == 2 and jobs == [], "retry once, then accept the empty result (no infinite loop)"
|