feat: add health checks and Prometheus metrics endpoints

Production readiness features:

Health Endpoints:
- GET /health - Liveness check (returns service status, version, uptime)
- GET /ready - Readiness check (verifies policy loaded and ETS tables exist)
- GET /metrics - Prometheus metrics export

Observability:
- Added prometheus_telemetry dependency
- Configured TelemetryMetricsPrometheus.Core for metrics collection
- Metrics include request counts, durations, policy stats
- All endpoints return JSON (except /metrics which returns Prometheus format)

Implementation:
- Health checks bypass policy enforcement
- Readiness returns 503 if policy not loaded
- Metrics endpoint scrapes telemetry data

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>

Author: Jonathan D.A. Jewell

lib/http_capability_gateway/application.ex

@@ -23,6 +23,9 @@ defmodule HttpCapabilityGateway.Application do
         port = Application.get_env(:http_capability_gateway, :port, 4000)
 
         children = [
+          # Prometheus metrics exporter
+          {TelemetryMetricsPrometheus.Core, metrics: telemetry_metrics()},
+
           # HTTP server with our Gateway router
           {Plug.Cowboy, scheme: :http, plug: HttpCapabilityGateway.Gateway, options: [port: port]}
         ]

lib/http_capability_gateway/gateway.ex

@@ -44,6 +44,21 @@ defmodule HttpCapabilityGateway.Gateway do
   plug(:match)
   plug(:dispatch)
 
+  # Health check endpoint - doesn't require policy
+  get "/health" do
+    handle_health_check(conn)
+  end
+
+  # Readiness check endpoint - verifies policy is loaded
+  get "/ready" do
+    handle_readiness_check(conn)
+  end
+
+  # Prometheus metrics endpoint
+  get "/metrics" do
+    handle_metrics(conn)
+  end
+
   # Catch-all route - enforce policy on all requests
   match _ do
     handle_request(conn)
@@ -258,4 +273,104 @@ defmodule HttpCapabilityGateway.Gateway do
       }
     )
   end
+
+  @doc """
+  Health check endpoint - returns 200 OK if service is running.
+
+  Does not check policy loading or backend connectivity - use /ready for that.
+  """
+  def handle_health_check(conn) do
+    uptime_seconds = div(System.monotonic_time(:second) - get_start_time(), 1)
+
+    response = %{
+      status: "healthy",
+      service: "http-capability-gateway",
+      version: Application.spec(:http_capability_gateway, :vsn) |> to_string(),
+      uptime_seconds: uptime_seconds
+    }
+
+    conn
+    |> put_resp_content_type("application/json")
+    |> send_resp(200, Jason.encode!(response))
+  end
+
+  @doc """
+  Readiness check endpoint - returns 200 OK if service is ready to handle traffic.
+
+  Checks:
+  - Policy is loaded
+  - ETS tables exist
+  """
+  def handle_readiness_check(conn) do
+    policy_table = Application.get_env(:http_capability_gateway, :policy_table)
+
+    cond do
+      is_nil(policy_table) ->
+        # Policy not loaded
+        response = %{
+          status: "not_ready",
+          reason: "Policy not loaded",
+          service: "http-capability-gateway"
+        }
+
+        conn
+        |> put_resp_content_type("application/json")
+        |> send_resp(503, Jason.encode!(response))
+
+      :ets.whereis(policy_table) == :undefined ->
+        # ETS table doesn't exist
+        response = %{
+          status: "not_ready",
+          reason: "Policy table not found",
+          service: "http-capability-gateway"
+        }
+
+        conn
+        |> put_resp_content_type("application/json")
+        |> send_resp(503, Jason.encode!(response))
+
+      true ->
+        # Ready to serve traffic
+        rule_count = :ets.info(policy_table, :size)
+
+        response = %{
+          status: "ready",
+          service: "http-capability-gateway",
+          policy_rules: rule_count
+        }
+
+        conn
+        |> put_resp_content_type("application/json")
+        |> send_resp(200, Jason.encode!(response))
+    end
+  end
+
+  @doc """
+  Prometheus metrics endpoint - exports metrics in Prometheus format.
+
+  Metrics include:
+  - Request counts by decision (allow/deny)
+  - Request duration histograms
+  - Policy rule counts
+  """
+  def handle_metrics(conn) do
+    metrics = TelemetryMetricsPrometheus.Core.scrape()
+
+    conn
+    |> put_resp_content_type("text/plain")
+    |> send_resp(200, metrics)
+  end
+
+  # Get application start time (monotonic time when app started)
+  defp get_start_time do
+    case :persistent_term.get({__MODULE__, :start_time}, nil) do
+      nil ->
+        start_time = System.monotonic_time(:second)
+        :persistent_term.put({__MODULE__, :start_time}, start_time)
+        start_time
+
+      start_time ->
+        start_time
+    end
+  end
 end

mix.exs

@@ -40,6 +40,7 @@ defmodule HttpCapabilityGateway.MixProject do
       {:telemetry, "~> 1.2"},
       {:telemetry_metrics, "~> 1.0"},
       {:telemetry_poller, "~> 1.1"},
+      {:prometheus_telemetry, "~> 0.4"},
 
       # Testing
       {:stream_data, "~> 1.0", only: :test}

mix.lock

@@ -14,11 +14,13 @@
   "plug": {:hex, :plug, "1.19.1", "09bac17ae7a001a68ae393658aa23c7e38782be5c5c00c80be82901262c394c0", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "560a0017a8f6d5d30146916862aaf9300b7280063651dd7e532b8be168511e62"},
   "plug_cowboy": {:hex, :plug_cowboy, "2.7.5", "261f21b67aea8162239b2d6d3b4c31efde4daa22a20d80b19c2c0f21b34b270e", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "20884bf58a90ff5a5663420f5d2c368e9e15ed1ad5e911daf0916ea3c57f77ac"},
   "plug_crypto": {:hex, :plug_crypto, "2.1.1", "19bda8184399cb24afa10be734f84a16ea0a2bc65054e23a62bb10f06bc89491", [:mix], [], "hexpm", "6470bce6ffe41c8bd497612ffde1a7e4af67f36a15eea5f921af71cf3e11247c"},
+  "prometheus_telemetry": {:hex, :prometheus_telemetry, "0.4.13", "873215e360a7a460732df63f10d9f55741ed9939b66d5ca9d4ac2fafe74d7b26", [:mix], [{:absinthe, ">= 0.0.0", [hex: :absinthe, repo: "hexpm", optional: true]}, {:ecto, ">= 0.0.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:finch, ">= 0.12.0", [hex: :finch, repo: "hexpm", optional: true]}, {:hackney, ">= 0.0.0", [hex: :hackney, repo: "hexpm", optional: true]}, {:nimble_options, "~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:oban, ">= 0.0.0", [hex: :oban, repo: "hexpm", optional: true]}, {:phoenix, ">= 0.0.0", [hex: :phoenix, repo: "hexpm", optional: true]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.5", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:swoosh, ">= 0.0.0", [hex: :swoosh, repo: "hexpm", optional: true]}, {:telemetry_metrics, "~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}, {:telemetry_metrics_prometheus_core, "~> 1.2", [hex: :telemetry_metrics_prometheus_core, repo: "hexpm", optional: false]}, {:telemetry_poller, "~> 1.0", [hex: :telemetry_poller, repo: "hexpm", optional: false]}], "hexpm", "cd0a14ba4ad3287ec77aa2df84b2f6c782ec704ff0989224ddf9959593f9621f"},
   "ranch": {:hex, :ranch, "2.2.0", "25528f82bc8d7c6152c57666ca99ec716510fe0925cb188172f41ce93117b1b0", [:make, :rebar3], [], "hexpm", "fa0b99a1780c80218a4197a59ea8d3bdae32fbff7e88527d7d8a4787eff4f8e7"},
   "req": {:hex, :req, "0.5.17", "0096ddd5b0ed6f576a03dde4b158a0c727215b15d2795e59e0916c6971066ede", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "0b8bc6ffdfebbc07968e59d3ff96d52f2202d0536f10fef4dc11dc02a2a43e39"},
   "stream_data": {:hex, :stream_data, "1.2.0", "58dd3f9e88afe27dc38bef26fce0c84a9e7a96772b2925c7b32cd2435697a52b", [:mix], [], "hexpm", "eb5c546ee3466920314643edf68943a5b14b32d1da9fe01698dc92b73f89a9ed"},
   "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"},
   "telemetry_metrics": {:hex, :telemetry_metrics, "1.1.0", "5bd5f3b5637e0abea0426b947e3ce5dd304f8b3bc6617039e2b5a008adc02f8f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e7b79e8ddfde70adb6db8a6623d1778ec66401f366e9a8f5dd0955c56bc8ce67"},
+  "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.2.1", "c9755987d7b959b557084e6990990cb96a50d6482c683fb9622a63837f3cd3d8", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "5e2c599da4983c4f88a33e9571f1458bf98b0cf6ba930f1dc3a6e8cf45d5afb6"},
   "telemetry_poller": {:hex, :telemetry_poller, "1.3.0", "d5c46420126b5ac2d72bc6580fb4f537d35e851cc0f8dbd571acf6d6e10f5ec7", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "51f18bed7128544a50f75897db9974436ea9bfba560420b646af27a9a9b35211"},
   "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"},
   "yaml_elixir": {:hex, :yaml_elixir, "2.12.0", "30343ff5018637a64b1b7de1ed2a3ca03bc641410c1f311a4dbdc1ffbbf449c7", [:mix], [{:yamerl, "~> 0.10", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "ca6bacae7bac917a7155dca0ab6149088aa7bc800c94d0fe18c5238f53b313c6"},