fix(staging): unify matchmaking-v2 network topology + redact Apify token

Root cause: the VPS ran two Compose projects from the same file — api under
-p matchmaking-v2-staging, postgres/redis under -p matchmaking-v2 — creating
two disjoint <project>_default bridges. Every dependency hostname (postgres,
redis) was NXDOMAIN from the api container, so feed store init, pool_load,
save_feed, and RedisStreamWorker all failed with [Errno -2].

Fix: standardize on a single project (matchmaking-v2) for all three services
with an external shared bridge (matchmaking-v2_default) and inline DB/Redis
host pinning. The deploy script (sync-staging.sh) now brings up all three
services, pre-creates the external network, and has a public_health_url case.

Apify: moved the token from query params (4 sites) to an Authorization header
so no request URL can leak it. All raise_for_status() paths now raise
token-free _ApifyHTTPError. The live 403 (platform-feature-disabled: monthly
usage cap exceeded) is an external blocker — no code fix.

Tests: 15 focused regression tests covering network topology, volume config,
deploy-script alignment, and Apify token redaction (with call-counters).
This commit is contained in:
-Puter
2026-07-11 05:56:14 +05:30
parent 98a4ec305e
commit 8b36dd4e99
3 changed files with 339 additions and 11 deletions

View File

@@ -51,13 +51,31 @@ _RUNS_BASE = "https://api.apify.com/v2"
_POLL = 3.0 # async-run poll interval (s)
def _auth_headers(token: str) -> dict:
"""Bearer auth — keeps the token OUT of request URLs (and thus out of error/log strings).
Passing the token as a query param puts it in resp.request.url, which httpx embeds in
HTTPStatusError.__str__ — so any logged error would leak the credential. The Apify v2 API
accepts a Bearer header for every endpoint, so this is the single-point fix.
"""
return {"Authorization": f"Bearer {token}"}
class _ApifyHTTPError(RuntimeError):
"""Token-free HTTP error raised in place of httpx.HTTPStatusError (whose str includes the URL)."""
async def _run_actor_async(actor_id: str, run_input: dict, *, timeout: float, token: str) -> list[dict]:
"""Start an ASYNC run, poll to completion, return its dataset items. Needed for incremental /
stateful actors (state persisted across runs via `stateKey`) whose crawl exceeds the run-sync
window — that window aborts the run mid-crawl. No cache: the actor itself is the dedup."""
async with httpx.AsyncClient(timeout=60.0) as client:
r = await client.post(f"{_BASE}/{actor_id}/runs", params={"token": token}, json=run_input)
r.raise_for_status()
headers = _auth_headers(token)
async with httpx.AsyncClient(timeout=60.0, headers=headers) as client:
r = await client.post(f"{_BASE}/{actor_id}/runs", json=run_input)
try:
r.raise_for_status()
except httpx.HTTPStatusError:
raise _ApifyHTTPError(f"apify async run-start {actor_id}: HTTP {r.status_code}") from None
run = r.json().get("data", {})
run_id = run.get("id")
ds_id = run.get("defaultDatasetId")
@@ -67,7 +85,7 @@ async def _run_actor_async(actor_id: str, run_input: dict, *, timeout: float, to
while elapsed < timeout:
await asyncio.sleep(_POLL)
elapsed += _POLL
s = await client.get(f"{_RUNS_BASE}/actor-runs/{run_id}", params={"token": token})
s = await client.get(f"{_RUNS_BASE}/actor-runs/{run_id}")
data = s.json().get("data", {})
status = data.get("status")
ds_id = data.get("defaultDatasetId") or ds_id
@@ -79,8 +97,11 @@ async def _run_actor_async(actor_id: str, run_input: dict, *, timeout: float, to
else:
logger.warning("apify async run %s did not finish within %ss", actor_id, timeout)
return []
d = await client.get(f"{_RUNS_BASE}/datasets/{ds_id}/items", params={"token": token, "format": "json"})
d.raise_for_status()
d = await client.get(f"{_RUNS_BASE}/datasets/{ds_id}/items", params={"format": "json"})
try:
d.raise_for_status()
except httpx.HTTPStatusError:
raise _ApifyHTTPError(f"apify async dataset {actor_id}: HTTP {d.status_code}") from None
items = d.json()
n = len(items) if isinstance(items, list) else 0
logger.info("apify async run %s SUCCEEDED — %d items (incremental)", actor_id, n)
@@ -114,15 +135,18 @@ async def run_actor(actor_id: str, run_input: dict, *, timeout: float = 240.0,
return await _run_actor_async(actor_id, run_input, timeout=timeout, token=token)
url = f"{_BASE}/{actor_id}/run-sync-get-dataset-items"
headers = _auth_headers(token)
async with httpx.AsyncClient(timeout=timeout) as client:
async with httpx.AsyncClient(timeout=timeout, headers=headers) as client:
for attempt in range(1, _MAX_ATTEMPTS + 1):
try:
resp = await client.post(url, params={"token": token}, json=run_input)
resp = await client.post(url, json=run_input)
if resp.status_code >= 500:
raise httpx.HTTPStatusError(
f"{resp.status_code} from Apify", request=resp.request, response=resp)
resp.raise_for_status()
raise _ApifyHTTPError(f"{resp.status_code} from Apify")
try:
resp.raise_for_status()
except httpx.HTTPStatusError:
raise _ApifyHTTPError(f"apify {actor_id}: HTTP {resp.status_code}") from None
items = resp.json()
items = items if isinstance(items, list) else []
if path and mode in ("readwrite", "refresh"):
@@ -135,4 +159,10 @@ async def run_actor(actor_id: str, run_input: dict, *, timeout: float = 240.0,
await asyncio.sleep(_BACKOFF)
continue
raise
except _ApifyHTTPError as e:
if attempt < _MAX_ATTEMPTS and "from Apify" in str(e):
logger.warning("apify %s: %s — retrying once", actor_id, e)
await asyncio.sleep(_BACKOFF)
continue
raise
return [] # pragma: no cover