Load gateway auth secret from Secret Manager

Author: anth-volk

.github/scripts/validate_gateway_auth_env.sh

@@ -6,7 +6,7 @@ required=(
   GATEWAY_AUTH_ISSUER
   GATEWAY_AUTH_AUDIENCE
   GATEWAY_AUTH_CLIENT_ID
-  GATEWAY_AUTH_CLIENT_SECRET
+  GATEWAY_AUTH_CLIENT_SECRET_RESOURCE
 )
 
 missing=()
@@ -18,6 +18,6 @@ for name in "${required[@]}"; do
 done
 
 if [[ "${#missing[@]}" -gt 0 ]]; then
-  echo "Missing required gateway auth secrets: ${missing[*]}" >&2
+  echo "Missing required gateway auth configuration: ${missing[*]}" >&2
   exit 1
 fi

.github/workflows/push.yml

@@ -110,7 +110,7 @@ jobs:
           GATEWAY_AUTH_ISSUER: ${{ secrets.GATEWAY_AUTH_ISSUER }}
           GATEWAY_AUTH_AUDIENCE: ${{ secrets.GATEWAY_AUTH_AUDIENCE }}
           GATEWAY_AUTH_CLIENT_ID: ${{ secrets.GATEWAY_AUTH_CLIENT_ID }}
-          GATEWAY_AUTH_CLIENT_SECRET: ${{ secrets.GATEWAY_AUTH_CLIENT_SECRET }}
+          GATEWAY_AUTH_CLIENT_SECRET_RESOURCE: ${{ secrets.GATEWAY_AUTH_CLIENT_SECRET_RESOURCE }}
       - name: Deploy
         run: make deploy
         env:
@@ -123,7 +123,7 @@ jobs:
           GATEWAY_AUTH_ISSUER: ${{ secrets.GATEWAY_AUTH_ISSUER }}
           GATEWAY_AUTH_AUDIENCE: ${{ secrets.GATEWAY_AUTH_AUDIENCE }}
           GATEWAY_AUTH_CLIENT_ID: ${{ secrets.GATEWAY_AUTH_CLIENT_ID }}
-          GATEWAY_AUTH_CLIENT_SECRET: ${{ secrets.GATEWAY_AUTH_CLIENT_SECRET }}
+          GATEWAY_AUTH_CLIENT_SECRET_RESOURCE: ${{ secrets.GATEWAY_AUTH_CLIENT_SECRET_RESOURCE }}
   docker:
     name: Docker
     runs-on: ubuntu-latest

changelog.d/3496.added.md

@@ -1 +1 @@
-Add Auth0 client-credentials bearer auth for outbound simulation-gateway calls in API v1, including runtime env plumbing for the four `GATEWAY_AUTH_*` values and startup validation for partial auth configuration.
+Add Auth0 client-credentials bearer auth for outbound simulation-gateway calls in API v1, including `GATEWAY_AUTH_REQUIRED`, startup validation for partial auth configuration, and Google Secret Manager support for the gateway client secret via `GATEWAY_AUTH_CLIENT_SECRET_RESOURCE`.

gcp/export.py

@@ -7,7 +7,7 @@
     Path(GAE).resolve(strict=True)
     with open(GAE, "r") as f:
         GAE = f.read()
-except Exception as e:
+except Exception:
     pass
 DB_PD = os.environ["POLICYENGINE_DB_PASSWORD"]
 GITHUB_MICRODATA_TOKEN = os.environ["POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN"]
@@ -17,7 +17,7 @@
 GATEWAY_AUTH_ISSUER = os.environ["GATEWAY_AUTH_ISSUER"]
 GATEWAY_AUTH_AUDIENCE = os.environ["GATEWAY_AUTH_AUDIENCE"]
 GATEWAY_AUTH_CLIENT_ID = os.environ["GATEWAY_AUTH_CLIENT_ID"]
-GATEWAY_AUTH_CLIENT_SECRET = os.environ["GATEWAY_AUTH_CLIENT_SECRET"]
+GATEWAY_AUTH_CLIENT_SECRET_RESOURCE = os.environ["GATEWAY_AUTH_CLIENT_SECRET_RESOURCE"]
 
 # Export GAE to to .gac.json and DB_PD to .dbpw in the current directory
 
@@ -45,7 +45,8 @@
             ".gateway_auth_client_id", GATEWAY_AUTH_CLIENT_ID
         )
         dockerfile = dockerfile.replace(
-            ".gateway_auth_client_secret", GATEWAY_AUTH_CLIENT_SECRET
+            ".gateway_auth_client_secret_resource",
+            GATEWAY_AUTH_CLIENT_SECRET_RESOURCE,
         )
 
     with open(dockerfile_location, "w") as f:

gcp/policyengine_api/Dockerfile

@@ -9,10 +9,11 @@ ENV ANTHROPIC_API_KEY .anthropic_api_key
 ENV OPENAI_API_KEY .openai_api_key
 ENV HUGGING_FACE_TOKEN .hugging_face_token
 ENV CREDENTIALS_JSON_API_V2 .credentials_json_api_v2
+ENV GATEWAY_AUTH_REQUIRED 1
 ENV GATEWAY_AUTH_ISSUER .gateway_auth_issuer
 ENV GATEWAY_AUTH_AUDIENCE .gateway_auth_audience
 ENV GATEWAY_AUTH_CLIENT_ID .gateway_auth_client_id
-ENV GATEWAY_AUTH_CLIENT_SECRET .gateway_auth_client_secret
+ENV GATEWAY_AUTH_CLIENT_SECRET_RESOURCE .gateway_auth_client_secret_resource
 
 WORKDIR /app
 

policyengine_api/libs/gateway_auth.py

@@ -12,6 +12,7 @@
 import os
 import threading
 import time
+from functools import lru_cache
 from typing import Optional
 
 import httpx
@@ -21,29 +22,85 @@
 GATEWAY_AUTH_AUDIENCE_ENV = "GATEWAY_AUTH_AUDIENCE"
 GATEWAY_AUTH_CLIENT_ID_ENV = "GATEWAY_AUTH_CLIENT_ID"
 GATEWAY_AUTH_CLIENT_SECRET_ENV = "GATEWAY_AUTH_CLIENT_SECRET"
+GATEWAY_AUTH_CLIENT_SECRET_RESOURCE_ENV = "GATEWAY_AUTH_CLIENT_SECRET_RESOURCE"
+GATEWAY_AUTH_REQUIRED_ENV = "GATEWAY_AUTH_REQUIRED"
 
-GATEWAY_AUTH_ENV_VARS = (
+GATEWAY_AUTH_CORE_ENV_VARS = (
     GATEWAY_AUTH_ISSUER_ENV,
     GATEWAY_AUTH_AUDIENCE_ENV,
     GATEWAY_AUTH_CLIENT_ID_ENV,
+)
+
+GATEWAY_AUTH_SECRET_SOURCE_ENV_VARS = (
     GATEWAY_AUTH_CLIENT_SECRET_ENV,
+    GATEWAY_AUTH_CLIENT_SECRET_RESOURCE_ENV,
+)
+
+GATEWAY_AUTH_ENV_VARS = (
+    *GATEWAY_AUTH_CORE_ENV_VARS,
+    *GATEWAY_AUTH_SECRET_SOURCE_ENV_VARS,
 )
 
 
 class GatewayAuthError(RuntimeError):
     """Raised when the gateway auth config is missing or the token fetch fails."""
 
 
+def gateway_auth_required() -> bool:
+    """True iff this runtime requires gateway auth to be configured."""
+    return os.environ.get(GATEWAY_AUTH_REQUIRED_ENV, "").lower() in (
+        "1",
+        "true",
+        "yes",
+        "on",
+    )
+
+
+@lru_cache(maxsize=None)
+def _load_secret_from_secret_manager(resource_name: str) -> str:
+    """Fetch one secret payload from Google Secret Manager."""
+    from google.cloud import secretmanager
+
+    client = secretmanager.SecretManagerServiceClient()
+    response = client.access_secret_version(request={"name": resource_name})
+    return response.payload.data.decode("utf-8")
+
+
 def _require_all_or_none_gateway_auth_env() -> None:
-    """Refuse startup when the four GATEWAY_AUTH_* env vars are partially set."""
-    present = [name for name in GATEWAY_AUTH_ENV_VARS if os.environ.get(name)]
-    if present and len(present) != len(GATEWAY_AUTH_ENV_VARS):
-        missing = [name for name in GATEWAY_AUTH_ENV_VARS if not os.environ.get(name)]
+    """Refuse startup when gateway auth is partially or ambiguously configured."""
+    present_core = [name for name in GATEWAY_AUTH_CORE_ENV_VARS if os.environ.get(name)]
+    present_secret_sources = [
+        name for name in GATEWAY_AUTH_SECRET_SOURCE_ENV_VARS if os.environ.get(name)
+    ]
+    if len(present_secret_sources) > 1:
         raise GatewayAuthError(
-            "Gateway auth is partially configured: "
-            f"{', '.join(present)} set but {', '.join(missing)} missing. "
-            "Set all four or none."
+            "Gateway auth is ambiguously configured: both "
+            f"{GATEWAY_AUTH_CLIENT_SECRET_ENV} and "
+            f"{GATEWAY_AUTH_CLIENT_SECRET_RESOURCE_ENV} are set. "
+            "Set exactly one secret source."
         )
+    if present_core or present_secret_sources:
+        missing_core = [
+            name for name in GATEWAY_AUTH_CORE_ENV_VARS if not os.environ.get(name)
+        ]
+        if missing_core or not present_secret_sources:
+            missing = [
+                *missing_core,
+                *(
+                    []
+                    if present_secret_sources
+                    else [
+                        f"{GATEWAY_AUTH_CLIENT_SECRET_ENV} or "
+                        f"{GATEWAY_AUTH_CLIENT_SECRET_RESOURCE_ENV}"
+                    ]
+                ),
+            ]
+            present = [*present_core, *present_secret_sources]
+            raise GatewayAuthError(
+                "Gateway auth is partially configured: "
+                f"{', '.join(present)} set but {', '.join(missing)} missing. "
+                "Set issuer, audience, client ID, and exactly one secret source."
+            )
 
 
 class GatewayAuthTokenProvider:
@@ -57,6 +114,7 @@ def __init__(
         audience: Optional[str] = None,
         client_id: Optional[str] = None,
         client_secret: Optional[str] = None,
+        client_secret_resource: Optional[str] = None,
         *,
         http_timeout: float = 10.0,
     ):
@@ -80,6 +138,11 @@ def __init__(
             if client_secret is not None
             else os.environ.get(GATEWAY_AUTH_CLIENT_SECRET_ENV, "")
         )
+        self._client_secret_resource = (
+            client_secret_resource
+            if client_secret_resource is not None
+            else os.environ.get(GATEWAY_AUTH_CLIENT_SECRET_RESOURCE_ENV, "")
+        )
         self._http_timeout = http_timeout
         self._token: Optional[str] = None
         self._expires_at: float = 0.0
@@ -93,7 +156,7 @@ def configured(self) -> bool:
                 self._issuer,
                 self._audience,
                 self._client_id,
-                self._client_secret,
+                self._client_secret or self._client_secret_resource,
             )
         )
 
@@ -103,8 +166,9 @@ def get_token(self) -> str:
             raise GatewayAuthError(
                 "Gateway auth not configured: set "
                 f"{GATEWAY_AUTH_ISSUER_ENV}, {GATEWAY_AUTH_AUDIENCE_ENV}, "
-                f"{GATEWAY_AUTH_CLIENT_ID_ENV}, and "
-                f"{GATEWAY_AUTH_CLIENT_SECRET_ENV}."
+                f"{GATEWAY_AUTH_CLIENT_ID_ENV}, and either "
+                f"{GATEWAY_AUTH_CLIENT_SECRET_ENV} or "
+                f"{GATEWAY_AUTH_CLIENT_SECRET_RESOURCE_ENV}."
             )
 
         with self._lock:
@@ -118,12 +182,13 @@ def get_token(self) -> str:
 
     def _fetch_locked(self) -> None:
         """Call Auth0's /oauth/token. Caller must hold _lock."""
+        client_secret = self._get_client_secret_locked()
         try:
             response = httpx.post(
                 f"{self._issuer}/oauth/token",
                 json={
                     "client_id": self._client_id,
-                    "client_secret": self._client_secret,
+                    "client_secret": client_secret,
                     "audience": self._audience,
                     "grant_type": "client_credentials",
                 },
@@ -153,6 +218,31 @@ def _fetch_locked(self) -> None:
         self._token = token
         self._expires_at = time.time() + expires_in
 
+    def _get_client_secret_locked(self) -> str:
+        """Resolve the client secret from env or Secret Manager."""
+        if self._client_secret:
+            return self._client_secret
+        if not self._client_secret_resource:
+            raise GatewayAuthError(
+                "Gateway auth client secret not configured: set "
+                f"{GATEWAY_AUTH_CLIENT_SECRET_ENV} or "
+                f"{GATEWAY_AUTH_CLIENT_SECRET_RESOURCE_ENV}."
+            )
+        try:
+            self._client_secret = _load_secret_from_secret_manager(
+                self._client_secret_resource
+            )
+        except Exception as exc:
+            raise GatewayAuthError(
+                "Failed to load gateway auth client secret from Secret Manager "
+                f"resource {self._client_secret_resource}: {exc}"
+            ) from exc
+        if not self._client_secret:
+            raise GatewayAuthError(
+                "Secret Manager returned an empty gateway auth client secret."
+            )
+        return self._client_secret
+
     def invalidate(self) -> None:
         """Drop the cached token so the next call refetches it."""
         with self._lock:

policyengine_api/libs/simulation_api_modal.py

@@ -18,6 +18,7 @@
     GatewayAuthTokenProvider,
     GatewayBearerAuth,
     _require_all_or_none_gateway_auth_env,
+    gateway_auth_required,
 )
 
 
@@ -62,20 +63,20 @@ def __init__(self):
             else None
         )
         if auth is None:
-            if os.environ.get("FLASK_DEBUG") == "1":
-                print(
-                    "SimulationAPIModal initialised without gateway auth; "
-                    "all GATEWAY_AUTH_* env vars are unset.",
-                    file=sys.stderr,
-                    flush=True,
-                )
-            else:
+            if gateway_auth_required():
                 raise GatewayAuthError(
-                    "Gateway auth is required outside local debug mode: set "
+                    "Gateway auth is required in this runtime: set "
                     "GATEWAY_AUTH_ISSUER, GATEWAY_AUTH_AUDIENCE, "
                     "GATEWAY_AUTH_CLIENT_ID, and "
                     "GATEWAY_AUTH_CLIENT_SECRET."
                 )
+            print(
+                "SimulationAPIModal initialised without gateway auth; "
+                "all GATEWAY_AUTH_* env vars are unset and "
+                "GATEWAY_AUTH_REQUIRED is not enabled.",
+                file=sys.stderr,
+                flush=True,
+            )
         self.client = httpx.Client(timeout=30.0, auth=auth)
 
     def run(self, payload: dict) -> ModalSimulationExecution:

pyproject.toml

@@ -30,6 +30,7 @@ dependencies = [
     "flask-cors>=5,<6",
     "Flask-Caching>=2,<3",
     "google-cloud-logging>=3,<4",
+    "google-cloud-secret-manager>=2,<3",
     "gunicorn",
     "httpx>=0.27.0",
     "markupsafe>=3,<4",

scripts/smoke_test_local_boot.py

@@ -22,6 +22,7 @@
 
 REPO_ROOT = Path(__file__).resolve().parents[1]
 HEALTH_PATHS = ("/liveness-check", "/readiness-check")
+BOOT_TIMEOUT_SECONDS = 600
 HTTP_TIMEOUT_SECONDS = 2
 
 
@@ -111,8 +112,9 @@ def main() -> int:
         )
 
     try:
+        deadline = time.time() + BOOT_TIMEOUT_SECONDS
         ready = False
-        while True:
+        while time.time() < deadline:
             if process.poll() is not None:
                 break