feat: SafeTrust integration, dedicated regex ETS table, rate limiter

Wire proven/SafeTrust.idr verified trust hierarchy into the gateway,
replacing the ad-hoc evaluate_access/2 function. All access decisions
now flow through SafeTrust.evaluate/2 with structured audit trails.
Trust levels are atoms from a fixed set (no String.to_atom on input).

Split policy ETS storage into dual tables: main table for O(1) exact
path and global rule lookups, dedicated regex table for Tier 2 scans.
Atomic reload swaps both tables as a pair for zero-downtime reloads.

Add token bucket rate limiter as a Plug with per-trust-level quotas
(untrusted=10/s, authenticated=100/s, internal=unlimited). Uses ETS
for O(1) bucket state. Returns 429 + Retry-After on rate limit hit.
Client key derived from X-Forwarded-For first entry or peer IP.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: hyperpolymath

.machine_readable/STATE.scm

@@ -19,21 +19,23 @@
       (OTP "27+")
       (Plug "HTTP interface")
       (Cowboy "HTTP server")
-      (ETS "Policy storage")
+      (ETS "Policy storage + rate limiter buckets")
       (Prometheus "Metrics export")))
 
   (current-position
     (phase "production-ready")
-    (overall-completion 97)
+    (overall-completion 98)
     (components
-      (policy-pipeline "100% - DSL v1 loader, validator, compiler, tiered lookup")
-      (http-gateway "100% - Verb enforcement, proxy, stealth, security headers")
-      (health-checks "100% - /health, /ready endpoints")
+      (policy-pipeline "100% - DSL v1 loader, validator, compiler, tiered lookup with dedicated regex ETS table")
+      (http-gateway "100% - Verb enforcement, proxy, stealth, security headers, SafeTrust integration")
+      (health-checks "100% - /health, /ready endpoints with dual-table and rate limiter stats")
       (metrics "100% - Prometheus /metrics endpoint")
       (mtls "100% - Certificate-based trust extraction")
       (containerization "100% - Containerfile, docker-compose")
-      (performance "100% - Tiered ETS lookup (exact→regex→global), O(1) literal paths")
-      (security-hardening "100% - OWASP headers, safe verb allowlist, trust header spoofing protection, atomic policy reload, specific rescue clauses")
+      (performance "100% - Tiered ETS lookup (exact->regex->global), dedicated regex table, O(1) literal paths")
+      (security-hardening "100% - OWASP headers, safe verb allowlist, trust header spoofing protection, atomic dual-table reload, specific rescue clauses")
+      (safe-trust "100% - Verified trust hierarchy from proven/SafeTrust.idr, parse_trust/parse_exposure, evaluate/2")
+      (rate-limiter "100% - Token bucket per trust level, ETS-backed, 429+Retry-After, X-Forwarded-For client key")
       (documentation "80% - ExDoc, README, TOPOLOGY, missing deployment guide"))
     (working-features
       "Policy loading and validation"
@@ -44,10 +46,13 @@
       "Prometheus metrics export"
       "mTLS trust level extraction"
       "Container deployment"
-      "Tiered O(1)/O(r)/O(1) policy lookup"
+      "Tiered O(1)/O(r)/O(1) policy lookup with dedicated regex ETS table"
       "Security headers (OWASP hardened)"
       "Trust header spoofing protection (strip_untrusted_headers plug)"
-      "Atomic policy reload (zero-downtime ETS swap)"))
+      "Atomic dual-table policy reload (zero-downtime ETS swap)"
+      "SafeTrust verified trust hierarchy (replaces ad-hoc evaluate_access)"
+      "Token bucket rate limiting per trust level (10/100/unlimited rps)"
+      "Rate limiter wired into plug pipeline after trust extraction"))
 
   (route-to-mvp
     (milestones
@@ -61,8 +66,8 @@
         (completed "2026-02-07"))
       (v1.0.0
         (status "in-progress")
-        (description "Production ready - health checks, metrics, mTLS, containers")
-        (progress 95)
+        (description "Production ready - health checks, metrics, mTLS, containers, rate limiting")
+        (progress 98)
         (remaining
           "Deployment guide documentation"
           "Policy DSL reference documentation"))))
@@ -75,7 +80,8 @@
       "Property tests need DSL v1 format updates")
     (low
       "Example policy file uses old format (needs DSL v1 update)"
-      "Benchmark tiered lookup vs flat scan for different policy sizes"))
+      "Benchmark tiered lookup vs flat scan for different policy sizes"
+      "Rate limiter bucket cleanup for stale entries (low-priority, minimal memory)"))
 
   (critical-next-actions
     (immediate
@@ -86,10 +92,29 @@
       "Write policy DSL reference (docs/POLICY-DSL.md)"
       "Update performance tests for DSL v1")
     (this-month
-      "Add rate limiting support"
-      "Add request/response logging"))
+      "Add request/response logging"
+      "Add rate limiter bucket cleanup (periodic sweep of stale entries)"))
 
   (session-history
+    (session
+      (date "2026-02-28")
+      (focus "SafeTrust integration, dedicated regex ETS table, rate limiter")
+      (accomplishments
+        "Feature 1: Wired SafeTrust.evaluate/2 into gateway.ex replacing ad-hoc evaluate_access/2"
+        "Feature 1: Trust levels now atoms via SafeTrust.parse_trust/1 (fail-safe to :untrusted)"
+        "Feature 1: Exposure levels parsed via SafeTrust.parse_exposure/1 (fail-open to :public)"
+        "Feature 1: extract_trust plug in pipeline stores trust level in conn.assigns"
+        "Feature 2: Dedicated regex ETS table (policy_regex_table) for Tier 2 lookups"
+        "Feature 2: Regex routes stored in separate table, no filtering needed during scans"
+        "Feature 2: Atomic dual-table reload: both main and regex tables swapped as a pair"
+        "Feature 2: Updated stats/1 to count from both main and regex tables"
+        "Feature 3: Created RateLimiter plug with token bucket algorithm (ETS-backed)"
+        "Feature 3: Per-trust-level quotas: untrusted=10/s, authenticated=100/s, internal=unlimited"
+        "Feature 3: Client key from X-Forwarded-For first entry or peer IP"
+        "Feature 3: 429 Too Many Requests with Retry-After header on rate limit exceeded"
+        "Feature 3: Wired into gateway.ex plug pipeline after trust extraction, before routing"
+        "Updated readiness check to report dual-table stats and rate limiter bucket count")
+      (notes "Three major features shipped: verified trust hierarchy, optimized regex lookup, and rate limiting. All access decisions now flow through the formally verified SafeTrust module."))
     (session
       (date "2026-02-28")
       (focus "Critical security hardening - 5 fixes")
@@ -104,7 +129,7 @@
       (date "2026-02-28")
       (focus "Performance + security hardening")
       (accomplishments
-        "Tiered ETS lookup: O(1) exact path → O(r) regex routes → O(1) global rules"
+        "Tiered ETS lookup: O(1) exact path -> O(r) regex routes -> O(1) global rules"
         "Literal path detection: routes without regex metacharacters stored with {:exact, path, verb} key"
         "Security headers plug: X-Content-Type-Options, X-Frame-Options, Referrer-Policy, Cache-Control, Connection"
         "Updated stats/1 to report exact_routes vs regex_routes separately"

lib/http_capability_gateway/gateway.ex

@@ -39,6 +39,8 @@ defmodule HttpCapabilityGateway.Gateway do
 
   alias HttpCapabilityGateway.PolicyCompiler
   alias HttpCapabilityGateway.Proxy
+  alias HttpCapabilityGateway.RateLimiter
+  alias HttpCapabilityGateway.SafeTrust
 
   # Safe HTTP verb conversion with allowlist.
   #
@@ -68,6 +70,8 @@ defmodule HttpCapabilityGateway.Gateway do
   plug(Plug.Logger)
   plug(:security_headers)
   plug(:strip_untrusted_headers)
+  plug(:extract_trust)
+  plug(RateLimiter)
   plug(:match)
   plug(:dispatch)
 
@@ -160,6 +164,22 @@ defmodule HttpCapabilityGateway.Gateway do
     end
   end
 
+  # Extract trust level from headers/mTLS and store in conn.assigns.
+  #
+  # This plug runs BEFORE the RateLimiter plug in the pipeline so that
+  # rate limiting decisions can be based on the authenticated trust level.
+  # The trust level is parsed through SafeTrust.parse_trust/1 which
+  # safely maps strings to atoms from a fixed set (no String.to_atom).
+  #
+  # The trust level is stored in conn.assigns[:trust_level] and reused
+  # by both the rate limiter and the request handler, avoiding duplicate
+  # extraction work.
+  defp extract_trust(conn, _opts) do
+    trust_level_str = extract_trust_level(conn)
+    trust_level = SafeTrust.parse_trust(trust_level_str)
+    Plug.Conn.assign(conn, :trust_level, trust_level)
+  end
+
   # Health check endpoint - doesn't require policy
   get "/health" do
     handle_health_check(conn)
@@ -265,7 +285,12 @@ defmodule HttpCapabilityGateway.Gateway do
 
       verb ->
         # Valid HTTP method -- proceed with policy evaluation.
-        trust_level = extract_trust_level(conn)
+        #
+        # Trust level was already extracted and parsed by the :extract_trust
+        # plug earlier in the pipeline (stored in conn.assigns[:trust_level]).
+        # This avoids duplicate header parsing and ensures the rate limiter
+        # and request handler see the same trust level.
+        trust_level = Map.get(conn.assigns, :trust_level, :untrusted)
 
         Logger.info("Processing request",
           path: path,
@@ -291,20 +316,26 @@ defmodule HttpCapabilityGateway.Gateway do
         else
           # Lookup policy rule using tiered strategy:
           # Tier 1: O(1) exact literal path match
-          # Tier 2: O(r) regex route pattern scan
+          # Tier 2: O(r) regex route pattern scan (dedicated regex table)
           # Tier 3: O(1) global verb rule fallback
           case PolicyCompiler.lookup(policy_table, path, verb) do
             {:ok, rule} ->
-              # Evaluate access decision based on trust level vs exposure requirement
-              case evaluate_access(trust_level, rule.exposure) do
-                :allow ->
+              # Evaluate access decision using SafeTrust.evaluate/2.
+              # This replaces the ad-hoc evaluate_access/2 function with the
+              # formally verified trust hierarchy from proven/SafeTrust.idr.
+              # SafeTrust.evaluate/2 returns {:allow, t, e} or {:deny, t, e}
+              # providing a structured audit trail for every decision.
+              exposure = SafeTrust.parse_exposure(rule.exposure)
+
+              case SafeTrust.evaluate(trust_level, exposure) do
+                {:allow, _t, _e} ->
                   # Forward to backend -- trust level satisfies exposure requirement
                   duration_us = System.monotonic_time() - start_time
                   log_decision(request_id, path, verb, trust_level, :allow, rule, duration_us)
 
                   Proxy.forward(conn, rule)
 
-                :deny ->
+                {:deny, _t, _e} ->
                   # Access denied -- apply stealth profile if configured, otherwise 403
                   duration_us = System.monotonic_time() - start_time
                   log_decision(request_id, path, verb, trust_level, :deny, rule, duration_us)
@@ -506,43 +537,38 @@ defmodule HttpCapabilityGateway.Gateway do
     end
   end
 
-  # Evaluate if trust level satisfies exposure requirement
-  @spec evaluate_access(trust_level :: String.t(), exposure :: String.t()) :: :allow | :deny
-  defp evaluate_access(trust_level, exposure) do
-    case {trust_level, exposure} do
-      # Public endpoints - anyone can access
-      {_, "public"} -> :allow
-
-      # Authenticated endpoints - authenticated or internal only
-      {"authenticated", "authenticated"} -> :allow
-      {"internal", "authenticated"} -> :allow
-
-      # Internal endpoints - internal only
-      {"internal", "internal"} -> :allow
+  # NOTE: The previous ad-hoc evaluate_access/2 function has been removed.
+  # All access decisions now go through SafeTrust.evaluate/2 which implements
+  # the formally verified trust hierarchy from proven/SafeTrust.idr.
+  # The access decision is: rank(trust) >= rank(exposure), where ranks are
+  # untrusted=0, authenticated=1, internal=2 for trust, and
+  # public=0, authenticated=1, internal=2 for exposure.
+  # See HttpCapabilityGateway.SafeTrust for the single source of truth.
 
-      # All other combinations - deny
-      _ -> :deny
-    end
-  end
-
-  # Handle denied requests - apply stealth profile if configured
+  # Handle denied requests - apply stealth profile if configured.
+  #
+  # trust_level is now a SafeTrust atom (:untrusted, :authenticated, :internal).
+  # We convert to string for stealth profile map lookups and JSON responses,
+  # since stealth profiles use string keys matching the DSL v1 format.
   defp handle_denial(conn, rule, trust_level) do
     stealth_profile = get_stealth_profile(rule.stealth_profile)
+    trust_str = Atom.to_string(trust_level)
 
     {status_code, response_body} =
       case stealth_profile do
         nil ->
-          # No stealth - return clear error
+          # No stealth - return clear error with trust/exposure atoms as strings
           {403, %{
             error: "Forbidden",
             message: "Insufficient trust level for this operation",
             required: rule.exposure,
-            provided: trust_level
+            provided: trust_str
           }}
 
         profile when is_map(profile) ->
-          # Apply stealth - return configured status for trust level
-          code = get_stealth_code(profile, trust_level, rule.exposure)
+          # Apply stealth - return configured status for trust level.
+          # Stealth profile keys are strings matching DSL v1 format.
+          code = get_stealth_code(profile, trust_str, rule.exposure)
           message = get_stealth_message(code)
           {code, %{error: message}}
       end
@@ -689,13 +715,26 @@ defmodule HttpCapabilityGateway.Gateway do
         |> send_resp(503, Jason.encode!(response))
 
       true ->
-        # Ready to serve traffic
-        rule_count = :ets.info(policy_table, :size)
+        # Ready to serve traffic.
+        # Report rule counts from both main and regex tables.
+        regex_table = Application.get_env(:http_capability_gateway, :policy_regex_table)
+
+        main_count = :ets.info(policy_table, :size)
+
+        regex_count =
+          if regex_table && :ets.whereis(regex_table) != :undefined,
+            do: :ets.info(regex_table, :size),
+            else: 0
+
+        rate_limiter_buckets = RateLimiter.bucket_count()
 
         response = %{
           status: "ready",
           service: "http-capability-gateway",
-          policy_rules: rule_count
+          policy_rules: main_count + regex_count,
+          main_table_rules: main_count,
+          regex_table_rules: regex_count,
+          rate_limiter_buckets: rate_limiter_buckets
         }
 
         conn