add: produce OpenTelemetry traces

* Introduces producing OpenTelemetry traces with hs-opentelemetry.
* Adds OTel spans over the whole application
  loop and over each request processing phase
* Preliminary OTel tracing support in spec tests
* Disables tracing in load and memory tests
fix deps

Author: develop7

CHANGELOG.md

@@ -53,6 +53,8 @@ All notable changes to this project will be documented in this file. From versio
 ### Added
 
 - Log when the pool is released during schema cache reload on `log-level=debug` by @mkleczek in #4668
+- Introduced producing OpenTelemetry traces by @develop7 in #3140
+  + Requires a new `otel-enabled` config parameter to be enabled first.
 
 ### Fixed
 

cabal.project

@@ -1,4 +1,4 @@
 packages: postgrest.cabal
 tests: true
 allow-newer:
-  hasql:postgresql-libpq
+    hasql:postgresql-libpq

cabal.project.freeze

@@ -1 +1 @@
-index-state: hackage.haskell.org 2026-04-18T18:42:36Z
+index-state: hackage.haskell.org 2026-05-15T10:51:09Z

docs/postgrest.dict

@@ -109,6 +109,9 @@ Observability
 Okta
 OpenAPI
 openapi
+OpenTelemetry
+otel
+OTLP
 ov
 parametrized
 passphrase

docs/references/configuration.rst

@@ -840,6 +840,21 @@ openapi-server-proxy-uri
       ]
     }
 
+.. _otel-enabled:
+
+otel-enabled
+-------------------
+
+  =============== =================================
+  **Type**        Boolean
+  **Default**     False
+  **Reloadable**  Y
+  **Environment** PGRST_OTEL_ENABLED
+  **In-Database** `pgrst.otel_enabled`
+  =============== =================================
+
+  When this is set to :code:`true`, OpenTelemetry tracing is enabled. See :ref:`opentelemetry` for details and settings.
+
 .. _server_cors_allowed_origins:
 
 server-cors-allowed-origins

docs/references/observability.rst

@@ -312,6 +312,73 @@ You can enable tracing HTTP requests by setting :ref:`server-trace-header`. Spec
   HTTP/1.1 200 OK
   X-Request-Id: 123
 
+.. _opentelemetry:
+
+OpenTelemetry
+-------------
+
+PostgREST is able to produce OpenTelemetry traces. OpenTelemetry is configured
+using ``OTEL_*`` environment variables, per the `OpenTelemetry specification`_.
+
+The OpenTelemetry support is currently both experimental and in early stages of development, so expect some rough edges
+or lack of functionality, such as metrics or logs. Since current OpenTelemetry implementation incurs a small performance
+hit, it is gated behind the :ref:`otel-enabled` configuration option, disabled by default.
+
+Example configuration:
+
+.. code-block:: shell
+
+    OTEL_EXPORTER_OTLP_ENDPOINT='https://api.honeycomb.io/' \
+    OTEL_EXPORTER_OTLP_HEADERS="x-honeycomb-team=<honeycomb_api_key>"  \
+    OTEL_SERVICE_NAME='PostgREST'\
+    OTEL_LOG_LEVEL='debug'\
+    OTEL_TRACES_SAMPLER='always_on' \
+    postgrest
+
+Prometheus metrics through the OpenTelemetry Collector
+------------------------------------------------------
+
+PostgREST currently only exports traces through OpenTelemetry,-. To get metrics,
+it's possible to scrape the :ref:`metrics` endpoint exposed by the
+:ref:`admin_server` and relay it through the OpenTelemetry Collector.
+
+Example collector configuration:
+
+.. code-block:: yaml
+
+    receivers:
+      prometheus:
+        config:
+          scrape_configs:
+            - job_name: postgrest
+              scrape_interval: 15s
+              metrics_path: /metrics
+              static_configs:
+                - targets: ["127.0.0.1:3001"]
+
+    processors:
+      batch:
+
+    exporters:
+      otlp:
+        endpoint: otel-collector:4317
+        tls:
+          insecure: true
+
+    service:
+      pipelines:
+        metrics:
+          receivers: [prometheus]
+          processors: [batch]
+          exporters: [otlp]
+
+This assumes the PostgREST :ref:`admin_server` is listening on port ``3001`` and that
+its ``/metrics`` endpoint is reachable by the collector. Adjust the target address and
+OTLP exporter settings for your deployment.
+
+.. _`OpenTelemetry specification`: https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/
+
+
 Proxy-Status Header
 -------------------
 

nix/overlays/haskell-packages.nix

@@ -76,6 +76,10 @@ let
           }
           { };
 
+      # Support GHC 9.12
+      proto-lens = lib.doJailbreak prev.proto-lens;
+      proto-lens-runtime = lib.doJailbreak prev.proto-lens-runtime;
+
       warp =
         lib.dontCheck (prev.callHackageDirect
           {

nix/tools/tests.nix

@@ -10,6 +10,7 @@
 , jq
 , lib
 , nginx
+, opentelemetry-collector
 , postgrest
 , python3
 , runtimeShell
@@ -40,6 +41,7 @@ let
         args = [ "ARG_LEFTOVERS([hspec arguments])" ];
         workingDir = "/";
         withEnv = postgrest.env;
+        withPath = [ opentelemetry-collector ];
       }
       ''
         ${withTools.withPg} -f test/observability/fixtures/load.sql \

postgrest.cabal

@@ -80,6 +80,7 @@ library
                       PostgREST.Query.QueryBuilder
                       PostgREST.Query.SqlFragment
                       PostgREST.Query.Statements
+                      PostgREST.OpenTelemetry
                       PostgREST.Plan
                       PostgREST.Plan.CallPlan
                       PostgREST.Plan.MutatePlan
@@ -116,6 +117,7 @@ library
                     , crypton                   < 1.1.0
                     , directory                 >= 1.2.6 && < 1.4
                     , either                    >= 4.4.1 && < 5.1
+                    , exceptions                >= 0.10 && < 0.12
                     , extra                     >= 1.7.0 && < 2.0
                     , fast-logger               >= 3.2.0 && < 3.3
                     , fuzzyset                  >= 0.2.4 && < 0.3
@@ -126,6 +128,12 @@ library
                     , hasql-transaction         >= 1.0.1 && <= 1.2.1
                     , http-client               >= 0.7.19 && < 0.8
                     , http-types                >= 0.12.2 && < 0.13
+                    , hs-opentelemetry-sdk      >= 0.1.0.0 && < 0.2.0.0
+                    , hs-opentelemetry-instrumentation-wai
+                    , hs-opentelemetry-api      >= 0.3.1.0
+                    -- ^ this is due to hs-otel-sdk is not reexporting getTracerTracerProvider
+                    -- needed to initialize OpenTelemetry.middleware
+                    , hs-opentelemetry-utils-exceptions
                     , insert-ordered-containers >= 0.2.2 && < 0.3
                     , jose-jwt                  >= 0.9.6 && < 0.11
                     , lens                      >= 4.14 && < 5.4
@@ -276,6 +284,8 @@ test-suite spec
                     , hasql-pool        >= 1.0.1 && <= 1.3.0.4
                     , hasql-transaction >= 1.0.1 && <= 1.2.1
                     , heredoc           >= 0.2 && < 0.3
+                    , hs-opentelemetry-sdk      >= 0.1.0.0 && < 0.2.0.0
+                    , hs-opentelemetry-instrumentation-hspec
                     , hspec             >= 2.3 && < 2.12
                     , hspec-expectations >= 0.8.4 && < 0.9
                     , hspec-wai         >= 0.10 && < 0.12
@@ -311,12 +321,18 @@ test-suite observability
   hs-source-dirs:     test/observability
   main-is:            Main.hs
   other-modules:      ObsHelper
+                      OTelHelper
                       Observation.JwtCache
                       Observation.MetricsSpec
+                      Observation.OpenTelemetry
                       Observation.SchemaCacheSpec
   build-depends:      base              >= 4.9 && < 4.22
+                    , aeson             >= 2.0.3 && < 2.3
                     , base64-bytestring >= 1 && < 1.3
                     , bytestring        >= 0.10.8 && < 0.13
+                    , containers        >= 0.5.7 && < 0.8
+                    , directory         >= 1.2.6 && < 1.4
+                    , filepath          >= 1.4.1 && < 1.6
                     , hasql-pool        >= 1.0.1 && <= 1.3.0.4
                     , hasql-transaction >= 1.0.1 && <= 1.2.1
                     , hspec             >= 2.3 && < 2.12
@@ -325,10 +341,15 @@ test-suite observability
                     , hspec-wai-json    >= 0.10 && < 0.12
                     , http-types        >= 0.12.3 && < 0.13
                     , jose-jwt          >= 0.9.6 && < 0.11
+                    , network           >= 2.6 && < 3.3
                     , postgrest
                     , prometheus-client >= 1.1.1 && < 1.2.0
+                    , process           >= 1.4.2 && < 1.7
                     , protolude         >= 0.3.1 && < 0.4
                     , text              >= 1.2.2 && < 2.2
+                    , retry             >= 0.7.4 && < 0.10
+                    , streaming-commons
+                    , temporary         >= 1.3 && < 1.4
                     , wai               >= 3.2.1 && < 3.3
   ghc-options:        -threaded -O0 -Werror -Wall -fwarn-identities
                       -fno-spec-constr -optP-Wno-nonportable-include-path

src/PostgREST/App.hs

@@ -35,21 +35,22 @@ import qualified Network.Wai              as Wai
 import qualified Network.Wai.Handler.Warp as Warp
 import qualified Network.Wai.Header       as WaiHeader
 
-import qualified PostgREST.Admin      as Admin
-import qualified PostgREST.ApiRequest as ApiRequest
-import qualified PostgREST.AppState   as AppState
-import qualified PostgREST.Auth       as Auth
-import qualified PostgREST.Cors       as Cors
-import qualified PostgREST.Error      as Error
-import qualified PostgREST.Listener   as Listener
-import qualified PostgREST.MainTx     as MainTx
-import qualified PostgREST.Plan       as Plan
-import qualified PostgREST.Query      as Query
-import qualified PostgREST.Response   as Response
-import qualified PostgREST.Unix       as Unix (installSignalHandlers)
+import qualified PostgREST.Admin         as Admin
+import qualified PostgREST.ApiRequest    as ApiRequest
+import qualified PostgREST.AppState      as AppState
+import qualified PostgREST.Auth          as Auth
+import qualified PostgREST.Cors          as Cors
+import qualified PostgREST.Error         as Error
+import qualified PostgREST.Listener      as Listener
+import qualified PostgREST.MainTx        as MainTx
+import qualified PostgREST.OpenTelemetry as OTel
+import qualified PostgREST.Plan          as Plan
+import qualified PostgREST.Query         as Query
+import qualified PostgREST.Response      as Response
+import qualified PostgREST.Unix          as Unix (installSignalHandlers)
 
 import PostgREST.ApiRequest           (ApiRequest (..))
-import PostgREST.AppState             (AppState)
+import PostgREST.AppState             (AppState, getOTelMaybeTracer)
 import PostgREST.Auth.Types           (AuthResult (..))
 import PostgREST.Config               (AppConfig (..))
 import PostgREST.Error                (Error)
@@ -71,7 +72,7 @@ import qualified Network.Socket            as NS
 import           PostgREST.Unix            (createAndBindDomainSocket)
 import           Protolude                 hiding (Handler)
 
-run :: AppState -> IO ()
+run :: HasCallStack => AppState -> IO ()
 run appState = do
   conf <- AppState.getConfig appState
 
@@ -119,11 +120,12 @@ serverSettings AppConfig{..} =
     & setServerName ("postgrest/" <> prettyVersion)
 
 -- | PostgREST application
-postgrest :: AppState.AppState -> IO () -> Wai.Application
+postgrest :: HasCallStack => AppState.AppState -> IO () -> Wai.Application
 postgrest appState connWorker =
+  (OTel.oTelMiddleware . getOTelMaybeTracer) appState .
   traceHeaderMiddleware appState .
   Cors.middleware appState $
-    \req respond -> do
+    \req respond -> OTel.inSpanMDefault (getOTelMaybeTracer appState) "request" $ do
       appConf@AppConfig{..} <- AppState.getConfig appState -- the config must be read again because it can reload
       maybeSchemaCache <- AppState.getSchemaCache appState
 
@@ -164,7 +166,8 @@ postgrest appState connWorker =
       ResponseObs user req (Wai.responseStatus resp) (WaiHeader.contentLength $ Wai.responseHeaders resp)
 
 postgrestResponse
-  :: AppState.AppState
+  :: HasCallStack
+  => AppState.AppState
   -> AppConfig
   -> Maybe SchemaCache
   -> Maybe Double
@@ -186,14 +189,14 @@ postgrestResponse appState conf@AppConfig{..} maybeSchemaCache jwtTime authResul
 
   body <- lift $ Wai.strictRequestBody req
 
-  (parseTime, apiReq@ApiRequest{..}) <- withTiming conf $ liftEither . mapLeft Error.ApiRequestErr $ ApiRequest.userApiRequest conf prefs req body
-  (planTime, plan)                   <- withTiming conf $ liftEither $ Plan.actionPlan iAction conf apiReq sCache
+  (parseTime, apiReq@ApiRequest{..}) <- withRequestPhase "parse" $ liftEither . mapLeft Error.ApiRequestErr $ ApiRequest.userApiRequest conf prefs req body
+  (planTime, plan)                   <- withRequestPhase "plan" $ liftEither $ Plan.actionPlan iAction conf apiReq sCache
 
   let mainQ = Query.mainQuery plan conf apiReq authResult configDbPreRequest
       tx = MainTx.mainTx mainQ conf authResult apiReq plan sCache
       obsQuery s = when configLogQuery $ observer $ QueryObs mainQ s
 
-  (txTime, txResult) <- withTiming conf $ do
+  (txTime, txResult) <- withRequestPhase "query" $ do
     case tx of
       MainTx.NoDbTx r -> pure r
       MainTx.DbTx{..} -> do
@@ -206,7 +209,7 @@ postgrestResponse appState conf@AppConfig{..} maybeSchemaCache jwtTime authResul
         lift $ whenLeft eitherResp $ obsQuery . Error.status
         liftEither eitherResp
 
-  (respTime, resp) <- withTiming conf $ do
+  (respTime, resp) <- withRequestPhase "response" $ do
     let response = Response.actionResponse txResult apiReq (T.decodeUtf8 prettyVersion, docsVersion) conf sCache
         status' = either Error.status Response.pgrstStatus response
 
@@ -230,6 +233,13 @@ postgrestResponse appState conf@AppConfig{..} maybeSchemaCache jwtTime authResul
     varyHeaderPresent :: [HTTP.Header] -> Bool
     varyHeaderPresent = any (\(h, _v) -> h == HTTP.hVary)
 
+    withOTel :: HasCallStack => Text -> ExceptT e IO a -> ExceptT e  IO a
+    withOTel label = do
+      OTel.inSpanMDefault (getOTelMaybeTracer appState) label
+
+    withRequestPhase :: HasCallStack => Text -> ExceptT e IO a -> ExceptT e IO (Maybe Double, a)
+    withRequestPhase label = withOTel label . withTiming conf
+
 withTiming :: AppConfig -> ExceptT e IO a -> ExceptT e IO (Maybe Double, a)
 withTiming AppConfig{configServerTimingEnabled} f = if configServerTimingEnabled
     then do