Verify sensitive requests actually came through Traefik (#125)

agents.md

@@ -81,6 +81,26 @@ Postgres data is not part of the rclone backup set (which only syncs `core/`/`us
 - `/internal/*` — HTTP Message Signature verification, `RSA_PSS_SHA512` (for app-to-shard and peer-to-shard calls)
 - `/management/*` — Management API auth (hosted shards only)
 
+Because every app container shares the `portal` docker network with shard_core, any
+app could reach `/protected` or `/management` directly and bypass the Traefik
+forwardAuth gate. To raise the bar, both routers additionally require a shared secret
+header (`X-Ptl-Traefik-Secret`) injected by the `verify-traefik` Traefik middleware and
+verified by `service/traefik_secret.py`; a request that did not traverse Traefik lacks
+it and is rejected with 403. The secret is generated once and stored in `kv_store`
+(`traefik_secret`), and compiled into the dynamic Traefik config. `/internal` routes are
+deliberately NOT gated: the forwardAuth endpoints are called by Traefik directly (they
+never pass through the middleware chain), and `call_backend`/`call_peer` are called by
+apps directly by design.
+
+This is the short-term half of issue #125 and is intentionally partial. Because
+Postgres (`kv_store`, default creds) and the Traefik dashboard (`api.insecure: true`)
+sit on the same `portal` network, an app that actively targets shard_core can still
+read the secret and forge the header — so the gate stops accidental/naive direct calls
+and any app not specifically hunting the secret, but not a determined one. Closing that
+requires the network-isolation work tracked as the long-term half (isolate Postgres from
+`portal`, disable the unauthenticated Traefik API); until then, do not treat `/protected`
+and `/management` as fully sealed against a hostile app.
+
 ### Background Tasks
 Started at app lifespan startup, stopped at shutdown:
 - `InstallationWorker` — async task queue for app install/uninstall

shard_core/service/app_installation/util.py

@@ -16,6 +16,7 @@
 from shard_core.service.app_tools import get_installed_apps_path, get_app_metadata
 from shard_core.settings import settings
 from shard_core.service.traefik_dynamic_config import AppInfo, compile_config
+from shard_core.service.traefik_secret import ensure_traefik_secret
 from shard_core.util import signals
 
 log = logging.getLogger(__name__)
@@ -131,10 +132,14 @@ async def write_traefik_dyn_config():
     default_identity = Identity(**default_row)
     portal = SafeIdentity.from_identity(default_identity)
 
+    traefik_secret = await ensure_traefik_secret()
+
     traefik_dyn_filename = (
         Path(settings().path_root) / "core" / "traefik_dyn" / "traefik_dyn.yml"
     )
-    await _write_to_yaml(compile_config(app_infos, portal), traefik_dyn_filename)
+    await _write_to_yaml(
+        compile_config(app_infos, portal, traefik_secret), traefik_dyn_filename
+    )
 
 
 async def _write_to_yaml(spec: pydantic.BaseModel, output_path: Path):

shard_core/service/traefik_dynamic_config.py

@@ -9,6 +9,7 @@
     AppMeta,
 )
 from shard_core.data_model.identity import SafeIdentity
+from shard_core.service.traefik_secret import HEADER_NAME as TRAEFIK_SECRET_HEADER
 from shard_core.settings import settings
 
 
@@ -18,9 +19,11 @@ class AppInfo:
     installed_app: InstalledApp
 
 
-def compile_config(apps: List[AppInfo], portal: SafeIdentity) -> t.Model:
+def compile_config(
+    apps: List[AppInfo], portal: SafeIdentity, traefik_secret: str
+) -> t.Model:
     model = t.Model()
-    _add_http_section(model, portal)
+    _add_http_section(model, portal, traefik_secret)
     _add_tcp_section(model, portal)
     for app_info in apps:
         for ep in app_info.app_meta.entrypoints:
@@ -38,7 +41,7 @@ def compile_config(apps: List[AppInfo], portal: SafeIdentity) -> t.Model:
     return model
 
 
-def _add_http_section(model: t.Model, portal: SafeIdentity):
+def _add_http_section(model: t.Model, portal: SafeIdentity, traefik_secret: str):
     http_entrypoint = "http" if _disable_ssl() else "https"
     _routers = {
         "shard_core_public": t.HttpRouter(
@@ -52,14 +55,14 @@ def _add_http_section(model: t.Model, portal: SafeIdentity):
             rule="PathPrefix(`/core/protected`)",
             entryPoints=[http_entrypoint],
             service="shard_core",
-            middlewares=["strip", "auth-private"],
+            middlewares=["strip", "auth-private", "verify-traefik"],
             tls=make_http_cert_resolver(portal),
         ),
         "shard_core_management": t.HttpRouter(
             rule="PathPrefix(`/core/management`)",
             entryPoints=[http_entrypoint],
             service="shard_core",
-            middlewares=["strip", "auth-management"],
+            middlewares=["strip", "auth-management", "verify-traefik"],
             tls=make_http_cert_resolver(portal),
         ),
         "web-terminal": t.HttpRouter(
@@ -114,6 +117,13 @@ def _add_http_section(model: t.Model, portal: SafeIdentity):
                 )
             )
         ),
+        "verify-traefik": t.HttpMiddleware(
+            root=t.HttpMiddlewareItem10(
+                headers=t.HeadersMiddleware(
+                    customRequestHeaders={TRAEFIK_SECRET_HEADER: traefik_secret}
+                )
+            )
+        ),
         "auth": t.HttpMiddleware(
             root=t.HttpMiddlewareItem9(
                 forwardAuth=t.ForwardAuthMiddleware(

shard_core/service/traefik_secret.py

@@ -0,0 +1,40 @@
+import logging
+import secrets
+
+from fastapi import Header, HTTPException, status
+
+from shard_core.database import database
+
+log = logging.getLogger(__name__)
+
+STORE_KEY_TRAEFIK_SECRET = "traefik_secret"
+HEADER_NAME = "X-Ptl-Traefik-Secret"
+_SECRET_LENGTH = 64
+
+
+async def ensure_traefik_secret() -> str:
+    try:
+        return await database.get_value(STORE_KEY_TRAEFIK_SECRET)
+    except KeyError:
+        secret = secrets.token_urlsafe(_SECRET_LENGTH)
+        await database.set_value(STORE_KEY_TRAEFIK_SECRET, secret)
+        log.info("Generated new Traefik verification secret")
+        return secret
+
+
+async def get_traefik_secret() -> str:
+    return await database.get_value(STORE_KEY_TRAEFIK_SECRET)
+
+
+async def verify_traefik_secret(x_ptl_traefik_secret: str = Header(None)):
+    try:
+        expected = await get_traefik_secret()
+    except KeyError:
+        log.error("Traefik verification secret missing from kv_store")
+        raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+    if not x_ptl_traefik_secret or not secrets.compare_digest(
+        x_ptl_traefik_secret, expected
+    ):
+        log.warning("rejected request that did not traverse Traefik")
+        raise HTTPException(status.HTTP_403_FORBIDDEN)

shard_core/web/management/__init__.py

@@ -1,10 +1,13 @@
-from fastapi import APIRouter
+from fastapi import APIRouter, Depends
+
+from shard_core.service.traefik_secret import verify_traefik_secret
 
 from . import apps, notify, pairing_code
 
 router = APIRouter(
     prefix="/management",
     tags=["/management"],
+    dependencies=[Depends(verify_traefik_secret)],
 )
 
 router.include_router(apps.router)

shard_core/web/protected/__init__.py

@@ -1,4 +1,6 @@
-from fastapi import APIRouter
+from fastapi import APIRouter, Depends
+
+from shard_core.service.traefik_secret import verify_traefik_secret
 
 from . import (
     apps,
@@ -17,6 +19,7 @@
 router = APIRouter(
     prefix="/protected",
     tags=["/protected"],
+    dependencies=[Depends(verify_traefik_secret)],
 )
 
 router.include_router(apps.router)

tests/conftest.py

@@ -34,7 +34,7 @@
 from shard_core.data_model.identity import OutputIdentity, Identity
 from shard_core.data_model.profile import Profile
 from shard_core.database import database
-from shard_core.service import websocket, app_installation, telemetry
+from shard_core.service import websocket, app_installation, telemetry, traefik_secret
 from shard_core.service.app_tools import get_installed_apps_path
 from shard_core.settings import Settings, set_settings, reset_settings
 from shard_core.web.internal.call_peer import _get_app_for_ip_address
@@ -201,6 +201,9 @@ async def noop():
         ):
             whoareyou = (await client.get("/public/meta/whoareyou")).json()
             client.base_url = f'https://{whoareyou["domain"]}'
+            client.headers[traefik_secret.HEADER_NAME] = (
+                await traefik_secret.ensure_traefik_secret()
+            )
             await wait_until_all_apps_installed(client)
             yield client
 
@@ -229,6 +232,9 @@ async def app_client(mocker) -> AsyncGenerator[AsyncClient]:
         ) as client:
             whoareyou = (await client.get("/public/meta/whoareyou")).json()
             client.base_url = f'https://{whoareyou["domain"]}'
+            client.headers[traefik_secret.HEADER_NAME] = (
+                await traefik_secret.ensure_traefik_secret()
+            )
             yield client
     finally:
         await database.shutdown_database()

tests/test_traefik_dyn_spec.py

@@ -50,6 +50,7 @@ async def test_template_is_written(api_client):
             "auth-public",
             "auth-private",
             "auth-management",
+            "verify-traefik",
         }
         assert "authResponseHeadersRegex" in out_middlewares["auth"]["forwardAuth"]
 

tests/test_traefik_secret.py

@@ -0,0 +1,146 @@
+from pathlib import Path
+
+import yaml
+from httpx import AsyncClient
+from starlette import status
+
+from shard_core.database import database
+from shard_core.service import traefik_secret
+from shard_core.service.app_installation.util import write_traefik_dyn_config
+from shard_core.settings import settings
+
+
+async def _drive_ws_handshake(app_client: AsyncClient, path: str) -> list[str]:
+    """Open a websocket against the ASGI app behind app_client using the client's
+    current headers, and return the ASGI message types the app sent back."""
+    app = app_client._transport.app
+    sent: list[dict] = []
+    received = {"n": 0}
+
+    async def receive():
+        received["n"] += 1
+        if received["n"] == 1:
+            return {"type": "websocket.connect"}
+        return {"type": "websocket.disconnect", "code": 1000}
+
+    async def send(message):
+        sent.append(message)
+
+    scope = {
+        "type": "websocket",
+        "path": path,
+        "raw_path": path.encode(),
+        "headers": [(k.encode(), v.encode()) for k, v in app_client.headers.items()],
+        "query_string": b"",
+        "scheme": "ws",
+        "subprotocols": [],
+        "client": ("10.0.0.9", 1234),
+        "server": ("shard_core", 80),
+    }
+    await app(scope, receive, send)
+    return [m["type"] for m in sent]
+
+
+def _read_traefik_dyn() -> dict:
+    with open(
+        Path(settings().path_root) / "core" / "traefik_dyn" / "traefik_dyn.yml", "r"
+    ) as f:
+        return yaml.safe_load(f)
+
+
+async def test_protected_without_secret_is_rejected(app_client: AsyncClient):
+    del app_client.headers[traefik_secret.HEADER_NAME]
+
+    response = await app_client.get("protected/terminals")
+
+    assert response.status_code == status.HTTP_403_FORBIDDEN
+
+
+async def test_protected_with_wrong_secret_is_rejected(app_client: AsyncClient):
+    app_client.headers[traefik_secret.HEADER_NAME] = "not-the-real-secret"
+
+    response = await app_client.get("protected/terminals")
+
+    assert response.status_code == status.HTTP_403_FORBIDDEN
+
+
+async def test_protected_with_secret_is_allowed(app_client: AsyncClient):
+    response = await app_client.get("protected/terminals")
+
+    assert response.status_code == status.HTTP_200_OK
+
+
+async def test_management_without_secret_is_rejected(app_client: AsyncClient):
+    del app_client.headers[traefik_secret.HEADER_NAME]
+
+    response = await app_client.get("management/pairing_code")
+
+    assert response.status_code == status.HTTP_403_FORBIDDEN
+
+
+async def test_management_with_secret_is_allowed(app_client: AsyncClient):
+    response = await app_client.get("management/pairing_code")
+
+    assert response.status_code == status.HTTP_200_OK
+
+
+async def test_public_needs_no_secret(app_client: AsyncClient):
+    del app_client.headers[traefik_secret.HEADER_NAME]
+
+    response = await app_client.get("public/meta/whoareyou")
+
+    assert response.status_code == status.HTTP_200_OK
+
+
+async def test_forward_auth_endpoint_is_not_gated_by_secret(app_client: AsyncClient):
+    # Traefik calls the forwardAuth endpoints directly (no verify-traefik middleware
+    # in the chain), so they must reject on their own auth (401), not the secret gate.
+    del app_client.headers[traefik_secret.HEADER_NAME]
+
+    response = await app_client.get("internal/authenticate_terminal")
+
+    assert response.status_code == status.HTTP_401_UNAUTHORIZED
+
+
+async def test_missing_secret_fails_closed_with_500(app_client: AsyncClient):
+    # A shard whose secret somehow never got stored must fail closed, not open.
+    await database.remove_value(traefik_secret.STORE_KEY_TRAEFIK_SECRET)
+
+    response = await app_client.get("protected/terminals")
+
+    assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR
+
+
+async def test_protected_websocket_rejected_without_secret(app_client: AsyncClient):
+    # The /protected/ws/updates stream must not be openable by an app connecting
+    # directly (bypassing Traefik). Without the secret the handshake is denied, so the
+    # app never sees a websocket.accept.
+    del app_client.headers[traefik_secret.HEADER_NAME]
+
+    sent_types = await _drive_ws_handshake(app_client, "/protected/ws/updates")
+
+    assert "websocket.accept" not in sent_types
+
+
+async def test_protected_websocket_accepts_with_secret(app_client: AsyncClient):
+    # Positive control: with the Traefik-injected secret the same handshake is accepted.
+    # This is what makes the rejection test above meaningful rather than decorative.
+    sent_types = await _drive_ws_handshake(app_client, "/protected/ws/updates")
+
+    assert "websocket.accept" in sent_types
+
+
+async def test_traefik_config_injects_secret_on_sensitive_routers(app_client):
+    await write_traefik_dyn_config()
+    stored_secret = await traefik_secret.get_traefik_secret()
+    config = _read_traefik_dyn()
+
+    middleware = config["http"]["middlewares"]["verify-traefik"]
+    assert middleware["headers"]["customRequestHeaders"] == {
+        traefik_secret.HEADER_NAME: stored_secret
+    }
+
+    routers = config["http"]["routers"]
+    assert "verify-traefik" in routers["shard_core_private"]["middlewares"]
+    assert "verify-traefik" in routers["shard_core_management"]["middlewares"]
+    assert "verify-traefik" not in routers["shard_core_public"]["middlewares"]