fix(staging): unify matchmaking-v2 network topology + redact Apify token
Root cause: the VPS ran two Compose projects from the same file — api under -p matchmaking-v2-staging, postgres/redis under -p matchmaking-v2 — creating two disjoint <project>_default bridges. Every dependency hostname (postgres, redis) was NXDOMAIN from the api container, so feed store init, pool_load, save_feed, and RedisStreamWorker all failed with [Errno -2]. Fix: standardize on a single project (matchmaking-v2) for all three services with an external shared bridge (matchmaking-v2_default) and inline DB/Redis host pinning. The deploy script (sync-staging.sh) now brings up all three services, pre-creates the external network, and has a public_health_url case. Apify: moved the token from query params (4 sites) to an Authorization header so no request URL can leak it. All raise_for_status() paths now raise token-free _ApifyHTTPError. The live 403 (platform-feature-disabled: monthly usage cap exceeded) is an external blocker — no code fix. Tests: 15 focused regression tests covering network topology, volume config, deploy-script alignment, and Apify token redaction (with call-counters).
This commit is contained in:
@@ -51,13 +51,31 @@ _RUNS_BASE = "https://api.apify.com/v2"
|
||||
_POLL = 3.0 # async-run poll interval (s)
|
||||
|
||||
|
||||
def _auth_headers(token: str) -> dict:
|
||||
"""Bearer auth — keeps the token OUT of request URLs (and thus out of error/log strings).
|
||||
|
||||
Passing the token as a query param puts it in resp.request.url, which httpx embeds in
|
||||
HTTPStatusError.__str__ — so any logged error would leak the credential. The Apify v2 API
|
||||
accepts a Bearer header for every endpoint, so this is the single-point fix.
|
||||
"""
|
||||
return {"Authorization": f"Bearer {token}"}
|
||||
|
||||
|
||||
class _ApifyHTTPError(RuntimeError):
|
||||
"""Token-free HTTP error raised in place of httpx.HTTPStatusError (whose str includes the URL)."""
|
||||
|
||||
|
||||
async def _run_actor_async(actor_id: str, run_input: dict, *, timeout: float, token: str) -> list[dict]:
|
||||
"""Start an ASYNC run, poll to completion, return its dataset items. Needed for incremental /
|
||||
stateful actors (state persisted across runs via `stateKey`) whose crawl exceeds the run-sync
|
||||
window — that window aborts the run mid-crawl. No cache: the actor itself is the dedup."""
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
r = await client.post(f"{_BASE}/{actor_id}/runs", params={"token": token}, json=run_input)
|
||||
r.raise_for_status()
|
||||
headers = _auth_headers(token)
|
||||
async with httpx.AsyncClient(timeout=60.0, headers=headers) as client:
|
||||
r = await client.post(f"{_BASE}/{actor_id}/runs", json=run_input)
|
||||
try:
|
||||
r.raise_for_status()
|
||||
except httpx.HTTPStatusError:
|
||||
raise _ApifyHTTPError(f"apify async run-start {actor_id}: HTTP {r.status_code}") from None
|
||||
run = r.json().get("data", {})
|
||||
run_id = run.get("id")
|
||||
ds_id = run.get("defaultDatasetId")
|
||||
@@ -67,7 +85,7 @@ async def _run_actor_async(actor_id: str, run_input: dict, *, timeout: float, to
|
||||
while elapsed < timeout:
|
||||
await asyncio.sleep(_POLL)
|
||||
elapsed += _POLL
|
||||
s = await client.get(f"{_RUNS_BASE}/actor-runs/{run_id}", params={"token": token})
|
||||
s = await client.get(f"{_RUNS_BASE}/actor-runs/{run_id}")
|
||||
data = s.json().get("data", {})
|
||||
status = data.get("status")
|
||||
ds_id = data.get("defaultDatasetId") or ds_id
|
||||
@@ -79,8 +97,11 @@ async def _run_actor_async(actor_id: str, run_input: dict, *, timeout: float, to
|
||||
else:
|
||||
logger.warning("apify async run %s did not finish within %ss", actor_id, timeout)
|
||||
return []
|
||||
d = await client.get(f"{_RUNS_BASE}/datasets/{ds_id}/items", params={"token": token, "format": "json"})
|
||||
d.raise_for_status()
|
||||
d = await client.get(f"{_RUNS_BASE}/datasets/{ds_id}/items", params={"format": "json"})
|
||||
try:
|
||||
d.raise_for_status()
|
||||
except httpx.HTTPStatusError:
|
||||
raise _ApifyHTTPError(f"apify async dataset {actor_id}: HTTP {d.status_code}") from None
|
||||
items = d.json()
|
||||
n = len(items) if isinstance(items, list) else 0
|
||||
logger.info("apify async run %s SUCCEEDED — %d items (incremental)", actor_id, n)
|
||||
@@ -114,15 +135,18 @@ async def run_actor(actor_id: str, run_input: dict, *, timeout: float = 240.0,
|
||||
return await _run_actor_async(actor_id, run_input, timeout=timeout, token=token)
|
||||
|
||||
url = f"{_BASE}/{actor_id}/run-sync-get-dataset-items"
|
||||
headers = _auth_headers(token)
|
||||
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
async with httpx.AsyncClient(timeout=timeout, headers=headers) as client:
|
||||
for attempt in range(1, _MAX_ATTEMPTS + 1):
|
||||
try:
|
||||
resp = await client.post(url, params={"token": token}, json=run_input)
|
||||
resp = await client.post(url, json=run_input)
|
||||
if resp.status_code >= 500:
|
||||
raise httpx.HTTPStatusError(
|
||||
f"{resp.status_code} from Apify", request=resp.request, response=resp)
|
||||
resp.raise_for_status()
|
||||
raise _ApifyHTTPError(f"{resp.status_code} from Apify")
|
||||
try:
|
||||
resp.raise_for_status()
|
||||
except httpx.HTTPStatusError:
|
||||
raise _ApifyHTTPError(f"apify {actor_id}: HTTP {resp.status_code}") from None
|
||||
items = resp.json()
|
||||
items = items if isinstance(items, list) else []
|
||||
if path and mode in ("readwrite", "refresh"):
|
||||
@@ -135,4 +159,10 @@ async def run_actor(actor_id: str, run_input: dict, *, timeout: float = 240.0,
|
||||
await asyncio.sleep(_BACKOFF)
|
||||
continue
|
||||
raise
|
||||
except _ApifyHTTPError as e:
|
||||
if attempt < _MAX_ATTEMPTS and "from Apify" in str(e):
|
||||
logger.warning("apify %s: %s — retrying once", actor_id, e)
|
||||
await asyncio.sleep(_BACKOFF)
|
||||
continue
|
||||
raise
|
||||
return [] # pragma: no cover
|
||||
|
||||
Reference in New Issue
Block a user