fix: run cache refresh in the background, return 202

The 4-hourly /admin/refresh-cache fans out across ~130 ATS boards and
runs for minutes — past the upstream gateway's ~100s timeout — so the
gateway logged a misleading 524 even though the refresh completed
server-side every time.

The endpoint now hands refresh_cached_jobs to a FastAPI BackgroundTasks
job and returns 202 Accepted immediately (the decouple-the-response
shape /analyze-jobs already uses). The Sentry cached-jobs-refresh cron
check-in moved into the background job so its in_progress -> ok/error
window still spans the real refresh duration; the background task
never raises (failures log at ERROR + error the check-in). pg_cron now
records a fast 202 instead of a 524.

See DEVLOG Day 78.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: LEANDERANTONY

backend/routers/jobs.py

@@ -1,6 +1,7 @@
+import logging
 import secrets
 
-from fastapi import APIRouter, Depends, Header, HTTPException, Request
+from fastapi import APIRouter, BackgroundTasks, Depends, Header, HTTPException, Request
 
 from backend import quota
 from backend.models import (
@@ -26,8 +27,11 @@
 from backend.tiers import resolve_user_tier
 from src.config import REFRESH_CACHE_SECRET
 from src.errors import AppError
+from src.logging_utils import get_logger, log_event
 
 
+LOGGER = get_logger(__name__)
+
 router = APIRouter(prefix="/jobs", tags=["jobs"])
 
 
@@ -152,32 +156,69 @@ def _verify_refresh_secret(authorization: str | None = Header(default=None)) ->
         raise HTTPException(status_code=401, detail="Invalid refresh-cache token.")
 
 
-@admin_router.post("/refresh-cache")
-def refresh_cache(
-    request: Request,
-    _: None = Depends(_verify_refresh_secret),
-):
-    """Refresh cached_jobs from all configured providers.
+def _run_cache_refresh_job() -> None:
+    """Background-task body for ``POST /admin/refresh-cache``.
 
-    This is what Supabase pg_cron hits every 4 hours (six times a
-    day on the `0 */4 * * *` schedule). Returns the
-    structured refresh report (see `refresh_cached_jobs`) so cron
-    output can be inspected when something goes wrong.
+    The refresh fans out across ~130 ATS boards and runs for minutes —
+    far longer than the upstream gateway will wait — so it runs here,
+    after the endpoint has already returned 202. Running it inline made
+    the gateway time the *response* out at ~100s and record a
+    misleading ``524`` in the cron logs even though the refresh
+    completed server-side.
 
-    Wrapped in a Sentry cron check-in (`cached-jobs-refresh`): a
-    missed or errored check-in is how monitoring learns the 4-hourly
-    refresh stopped firing — pg_cron failing silently is otherwise
-    invisible.
+    The Sentry ``cached-jobs-refresh`` check-in wraps THIS function,
+    not the request, so its in_progress -> ok/error window reflects the
+    real refresh duration. Never raises: a background task has no
+    caller to surface an exception to, so any failure is recorded as an
+    errored cron check-in plus an ERROR-level log (a Sentry issue via
+    the LoggingIntegration) and swallowed. ``refresh_cached_jobs`` is
+    itself crash-safe and idempotent — a failed run is recovered by the
+    next 4-hourly tick.
     """
     try:
         with sentry_cron_monitor(
             CACHED_JOBS_REFRESH_MONITOR_SLUG,
             CACHED_JOBS_REFRESH_MONITOR_CONFIG,
         ):
-            report = refresh_cached_jobs()
-    except RuntimeError as exc:
-        raise HTTPException(status_code=503, detail=str(exc)) from exc
-    return report
+            refresh_cached_jobs()
+    except Exception as exc:  # noqa: BLE001 — background-task boundary
+        log_event(
+            LOGGER,
+            logging.ERROR,
+            "cached_jobs_refresh_background_failed",
+            f"Background cached_jobs refresh failed: "
+            f"{type(exc).__name__}: {exc}",
+            error=f"{type(exc).__name__}: {exc}",
+        )
+
+
+@admin_router.post("/refresh-cache", status_code=202)
+def refresh_cache(
+    request: Request,
+    background_tasks: BackgroundTasks,
+    _: None = Depends(_verify_refresh_secret),
+):
+    """Trigger a cached_jobs refresh from all configured providers.
+
+    Supabase pg_cron hits this every 4 hours (`0 */4 * * *`). The
+    refresh fans out across ~130 ATS boards and runs for minutes —
+    longer than the upstream gateway will wait — so the work is handed
+    to a background task and the endpoint returns **202 Accepted**
+    immediately. Running it synchronously made the gateway time the
+    response out at ~100s and record a misleading `524` in the cron
+    logs, even though the refresh completed server-side.
+
+    The refresh outcome, its structured report, and the Sentry
+    `cached-jobs-refresh` cron check-in are all produced by the
+    background job — see `_run_cache_refresh_job`. A failed run
+    surfaces as an errored Sentry check-in plus an ERROR-level issue;
+    the daily `/admin/refresh-healthcheck` is the backstop.
+    """
+    background_tasks.add_task(_run_cache_refresh_job)
+    return {
+        "status": "accepted",
+        "detail": "cached_jobs refresh started in the background.",
+    }
 
 
 @admin_router.post("/refresh-healthcheck")

docs/DEVLOG.md

@@ -3957,3 +3957,29 @@ on both raise paths, `job_searched`, `resume_uploaded`,
 `analysis_started`, `artifact_exported` — added to the existing quota,
 app, and workspace suites. Also removed a pre-existing unused `Form`
 import flagged by ruff in `routers/workspace.py`.
+
+## Day 78: Async cache refresh — return 202, run in the background
+
+Verifying the Day 77 cron work surfaced an incidental finding in the
+pg_net response log: the 4-hourly `/admin/refresh-cache` call returns
+`524`. The refresh fans out across ~130 ATS boards and runs for
+minutes — past the upstream gateway's ~100s timeout — so the gateway
+gave up on the *response* and logged a 524, even though the refresh
+completed server-side every time (the healthcheck confirmed the cache
+was fresh, only ~2% stale). A misleading failure signal, not a real
+failure.
+
+The fix: `/admin/refresh-cache` now hands `refresh_cached_jobs` to a
+FastAPI `BackgroundTasks` job and returns **202 Accepted**
+immediately — the same decouple-the-response shape the `/analyze-jobs`
+route already uses. The Sentry `cached-jobs-refresh` cron check-in
+moved with the work: it now wraps the background
+`_run_cache_refresh_job`, not the request, so its
+in_progress -> ok/error window still spans the real refresh duration.
+The background task never raises — a failure becomes an errored
+check-in plus an ERROR-level log (a Sentry issue via the
+LoggingIntegration), and the daily healthcheck is the backstop.
+pg_cron now records a fast 202 instead of a 524; nothing else changes.
+
+Verification: test suite green; the `/admin/refresh-cache` endpoint
+test was updated to assert 202 + a scheduled background worker.

tests/test_backend_app.py

@@ -278,21 +278,18 @@ def test_admin_refresh_cache_returns_503_when_secret_unconfigured(monkeypatch):
     assert response.status_code == 503
 
 
-def test_admin_refresh_cache_accepts_correct_bearer_then_runs_worker(monkeypatch):
-    """Right secret + worker stub → endpoint returns the worker's
-    report. Pins that the auth → worker handoff actually works."""
+def test_admin_refresh_cache_accepts_correct_bearer_then_schedules_worker(monkeypatch):
+    """Right secret → 202 immediately, with the refresh handed to a
+    background task. The endpoint no longer blocks on the multi-minute
+    refresh (which used to time the gateway out into a 524)."""
     monkeypatch.setattr(
         "backend.routers.jobs.REFRESH_CACHE_SECRET", "test-secret"
     )
+    ran = []
 
     def _fake_refresh():
-        return {
-            "started_at": "2026-05-07T18:00:00Z",
-            "finished_at": "2026-05-07T18:00:05Z",
-            "duration_seconds": 5.0,
-            "providers": {"greenhouse": {"status": "ok"}},
-            "total_active_after": 42,
-        }
+        ran.append(True)
+        return {"total_active_after": 42}
 
     monkeypatch.setattr(
         "backend.routers.jobs.refresh_cached_jobs", _fake_refresh
@@ -302,10 +299,11 @@ def _fake_refresh():
         "/api/admin/refresh-cache",
         headers={"Authorization": "Bearer test-secret"},
     )
-    assert response.status_code == 200
-    payload = response.json()
-    assert payload["total_active_after"] == 42
-    assert payload["providers"]["greenhouse"]["status"] == "ok"
+    assert response.status_code == 202
+    assert response.json()["status"] == "accepted"
+    # Starlette's TestClient runs background tasks before returning, so
+    # the worker has executed by the time the response is in hand.
+    assert ran == [True]
 
 
 def test_admin_refresh_healthcheck_runs_service_with_correct_bearer(monkeypatch):