From 538d2fc5f3bc7a6328827aba9676230bbfb1c189 Mon Sep 17 00:00:00 2001
From: raulgupta <rahul.gupta2608@yahoo.com>
Date: Fri, 26 Jun 2026 17:05:30 +0530
Subject: [PATCH] Retry-on-empty guard for scrape-flaky boards
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

RCA of a prod "Naukri returned 0" on an AI-Architect search: NOT a cursor issue (cursors removed;
naukri-feed is a date feed with no page param), NOT a city-name or keyword bug — the EXACT same input
flaked 0 then returned 15 on retry. The scrape-based actors (Naukri-feed, curious_coder LinkedIn)
intermittently return 0 under the site's anti-scrape rate-limiting, then recover.

Fix: _fetch_board retries a board ONCE (fresh) when it returns 0. One guard covers BOTH the Naukri and
LinkedIn intermittency (cleaner than per-board fallbacks). A genuinely-empty query just returns 0 again
(cheap). 67 tests pass (3 new: recovers transient 0, no retry on success, no infinite loop).
---
 app/engine/search.py       |  9 ++++++-
 tests/test_search_retry.py | 51 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 59 insertions(+), 1 deletion(-)
 create mode 100644 tests/test_search_retry.py

diff --git a/app/engine/search.py b/app/engine/search.py
index 71f74db..94fd675 100644
--- a/app/engine/search.py
+++ b/app/engine/search.py
@@ -93,7 +93,14 @@ BOARDS = {
 
 async def _fetch_board(key: str, prefs: dict, cache_mode: str | None = None):
     actor, build, norm = BOARDS[key]
-    items = await run_actor(actor, build(prefs), cache_mode=cache_mode)
+    inp = build(prefs)
+    items = await run_actor(actor, inp, cache_mode=cache_mode)
+    if not items:
+        # Scrape-based boards (Naukri-feed, curious_coder LinkedIn) intermittently return 0 under the
+        # site's anti-scrape rate-limiting, then recover. One fresh retry rescues the transient case
+        # (a genuinely-empty query just returns 0 again — cheap). RCA'd: same input flakes 0 then 15.
+        logger.info("board %s returned 0 — retrying once (transient scrape flake)", key)
+        items = await run_actor(actor, inp, cache_mode="refresh")
     jobs = []
     for it in items:
         sj = norm(it)
diff --git a/tests/test_search_retry.py b/tests/test_search_retry.py
new file mode 100644
index 0000000..623a11a
--- /dev/null
+++ b/tests/test_search_retry.py
@@ -0,0 +1,51 @@
+"""Retry-on-empty guard — scrape-flaky boards (Naukri-feed, curious_coder LinkedIn) intermittently
+return 0; one fresh retry rescues the transient case. RCA: the same input flaked 0 then returned 15."""
+import asyncio
+
+from app.engine import search
+
+
+def _run(coro):
+    return asyncio.run(coro)
+
+
+def test_retry_on_empty_recovers_transient_zero(monkeypatch):
+    calls = {"n": 0}
+
+    async def fake_run_actor(actor, inp, cache_mode=None):
+        calls["n"] += 1
+        # first call flakes (0), retry returns a real job
+        if calls["n"] == 1:
+            return []
+        return [{"profile_job_title": "Data Architect", "branch_company_name": "Acme",
+                 "branch_location_city_name": "Delhi", "skills": ["python"], "job_id": "1"}]
+
+    monkeypatch.setattr(search, "run_actor", fake_run_actor)
+    key, jobs = _run(search._fetch_board("workindia", {"title": "Data Architect", "location": ["Delhi · India"]}))
+    assert calls["n"] == 2, "a board returning 0 must be retried exactly once"
+    assert len(jobs) == 1, "the retry's jobs are used"
+
+
+def test_no_retry_when_first_call_succeeds(monkeypatch):
+    calls = {"n": 0}
+
+    async def fake_run_actor(actor, inp, cache_mode=None):
+        calls["n"] += 1
+        return [{"profile_job_title": "X", "branch_company_name": "Y",
+                 "branch_location_city_name": "Delhi", "skills": [], "job_id": "1"}]
+
+    monkeypatch.setattr(search, "run_actor", fake_run_actor)
+    _run(search._fetch_board("workindia", {"title": "X", "location": ["Delhi · India"]}))
+    assert calls["n"] == 1, "a board that returns jobs is NOT retried (no extra cost)"
+
+
+def test_genuinely_empty_returns_empty_after_one_retry(monkeypatch):
+    calls = {"n": 0}
+
+    async def fake_run_actor(actor, inp, cache_mode=None):
+        calls["n"] += 1
+        return []   # genuinely empty (e.g. WorkIndia for a tech role)
+
+    monkeypatch.setattr(search, "run_actor", fake_run_actor)
+    key, jobs = _run(search._fetch_board("workindia", {"title": "AI Architect", "location": ["Delhi · India"]}))
+    assert calls["n"] == 2 and jobs == [], "retry once, then accept the empty result (no infinite loop)"