test: add toxiproxy in front of postgresql in withTmpDb

Having toxiproxy between PostgREST and PostgreSQL in tests opens possibilities of validating behavior of PostgREST under various network conditions: temporary network partitioning, delays etc.

This change adds toxiproxy startup and configuration to withTmpDb function in withTools.nix, so that all communication goes through it in tests.

Author: mkleczek

nix/tools/withTools.nix

@@ -9,6 +9,7 @@
 , slocat
 , writeText
 , writers
+, toxiproxy
 }:
 let
   withTmpDb =
@@ -35,7 +36,7 @@ let
         positionalCompletion = "_command";
         workingDir = "/";
         redirectTixFiles = false;
-        withPath = [ postgresql ];
+        withPath = [ postgresql toxiproxy ];
         withTmpDir = true;
       }
       ''
@@ -55,6 +56,7 @@ let
 
           export PGDATA="$tmpdir/db"
           export PGHOST="$tmpdir/socket"
+          export PGPORT=6432
           export PGUSER
           export PGDATABASE
           export PGRST_DB_SCHEMAS
@@ -63,8 +65,17 @@ let
 
           HBA_FILE="$tmpdir/pg_hba.conf"
           echo "local $PGDATABASE some_protected_user password" > "$HBA_FILE"
-          echo "local $PGDATABASE all trust" >> "$HBA_FILE"
-          echo "local replication all trust" >> "$HBA_FILE"
+          {
+            echo "local $PGDATABASE all trust"
+            echo "local replication all trust"
+            echo "host $PGDATABASE some_protected_user localhost scram-sha-256"
+            echo "host $PGDATABASE all localhost trust"
+          } >> "$HBA_FILE"
+
+          UNIX_PGHOST="$PGHOST"
+          export TCP_PGHOST="localhost"
+          REAL_PGPORT="$PGPORT"
+          export TOXI_PGPORT=7432
 
           log "Initializing database cluster..."
           # We try to make the database cluster as independent as possible from the host
@@ -81,9 +92,14 @@ let
           # On MacOS, it's 104 chars
           # See: https://serverfault.com/questions/641347/check-if-a-path-exceeds-maximum-for-unix-domain-socket
 
-          pg_ctl -l "$tmpdir/db.log" -w start -o "-F -c listen_addresses=\"\" -c hba_file=$HBA_FILE -k $PGHOST -c log_statement=\"all\" " \
+          pg_ctl -l "$tmpdir/db.log" -w start -o "-F -c listen_addresses=\"$TCP_PGHOST\" -c port=$REAL_PGPORT -c hba_file=$HBA_FILE -k $UNIX_PGHOST -c log_statement=\"all\" " \
             >> "$setuplog"
 
+          LOG_LEVEL=error toxiproxy-server&
+          TOXIPROXY_PID=$!
+          sleep 1 # give the server a moment to start
+          toxiproxy-cli create -l "$TCP_PGHOST:$TOXI_PGPORT" -u "$TCP_PGHOST:$REAL_PGPORT" pg
+
           log "Creating a minimally privileged $PGUSER connection role..."
           createuser "$PGUSER" -U postgres --host="$tmpdir/socket" --no-createdb --no-inherit --no-superuser --no-createrole --no-replication --login
 
@@ -94,6 +110,8 @@ let
             replica_slot="replica_$RANDOM"
             replica_dir="$tmpdir/$replica_slot"
             replica_host="$tmpdir/socket_$replica_slot"
+            replica_port=6433
+            export TOXI_REPLICA_PGPORT=7433
 
             mkdir -p "$replica_host"
 
@@ -106,15 +124,17 @@ let
 
             log "Starting replica on $replica_host"
 
-            pg_ctl -D "$replica_dir" -l "$replica_dblog" -w start -o "-F -c listen_addresses=\"\" -c hba_file=$HBA_FILE -k $replica_host -c log_statement=\"all\" " \
+            pg_ctl -D "$replica_dir" -l "$replica_dblog" -w start -o "-F -c listen_addresses=\"$TCP_PGHOST\" -c port=$replica_port -c hba_file=$HBA_FILE -k $replica_host -c log_statement=\"all\" " \
               >> "$setuplog"
+            toxiproxy-cli create -l $TCP_PGHOST:$TOXI_REPLICA_PGPORT -u $TCP_PGHOST:$replica_port pg_replica
 
             >&2 echo "${commandName}: Replica enabled. You can connect to it with: psql 'postgres:///$PGDATABASE?host=$replica_host' -U postgres"
             >&2 echo "${commandName}: You can tail the replica logs with: tail -f $replica_dblog"
 
-            export PGREPLICAHOST="$replica_host"
+            export PGREPLICAHOST="$TCP_PGHOST"
+            export PGREPLICAPORT="$TOXI_REPLICA_PGPORT"
             export PGREPLICASLOT="$replica_slot"
-            export PGRST_DB_URI="postgres:///$PGDATABASE?host=$PGREPLICAHOST,$PGHOST"
+            export PGRST_DB_URI="postgres:///$PGDATABASE?host=$PGREPLICAHOST,$TCP_PGHOST&port=$PGREPLICAPORT,$TOXI_PGPORT"
           fi
 
           # shellcheck disable=SC2317
@@ -127,6 +147,8 @@ let
               pg_ctl -D "$replica_dir" stop --mode=immediate >> "$setuplog"
               rm -rf "$replica_dir"
             fi
+            kill "$TOXIPROXY_PID" || true
+            wait "$TOXIPROXY_PID" || true
           }
           trap stop EXIT
         fi
@@ -140,6 +162,7 @@ let
         fi
 
         ("$_arg_command" "''${_arg_leftovers[@]}")
+        #(PGHOST="$TCP_PGHOST" PGPORT="$TOXI_PGPORT" "$_arg_command" "''${_arg_leftovers[@]}")
       '';
 
   # Helper script for running a command against all PostgreSQL versions.

postgrest.cabal

@@ -259,7 +259,9 @@ test-suite spec
                       Feature.Query.UpsertSpec
                       Feature.RollbackSpec
                       Feature.RpcPreRequestGucsSpec
+                      Feature.ToxiSpec
                       SpecHelper
+                      Toxiproxy
   build-depends:      base              >= 4.9 && < 4.20
                     , aeson             >= 2.0.3 && < 2.3
                     , aeson-qq          >= 0.8.1 && < 0.9
@@ -290,6 +292,9 @@ test-suite spec
                     , transformers-base >= 0.4.4 && < 0.5
                     , wai               >= 3.2.1 && < 3.3
                     , wai-extra         >= 3.0.19 && < 3.2
+                    , servant-client    >= 0.20.3.0 && < 0.21
+                    , servant           >= 0.20.3.0 && < 0.21
+                    , http-client       >= 0.7.19 && < 0.8
   ghc-options:        -threaded -O0 -Werror -Wall -fwarn-identities
                       -fno-spec-constr -optP-Wno-nonportable-include-path
                       -fno-warn-missing-signatures

src/PostgREST/AppState.hs

@@ -5,6 +5,7 @@
 module PostgREST.AppState
   ( AppState
   , destroy
+  , flushPool
   , getConfig
   , getSchemaCache
   , getMainThreadId

test/io/conftest.py

@@ -9,8 +9,9 @@ def dburi():
     "Postgres database connection URI."
     dbname = os.environ["PGDATABASE"]
     host = os.environ["PGHOST"]
+    port = os.environ["PGPORT"]
     user = os.environ["PGUSER"]
-    return f"postgresql://?dbname={dbname}&host={host}&user={user}".encode()
+    return f"postgresql://?dbname={dbname}&host={host}&port={port}&user={user}".encode()
 
 
 @pytest.fixture
@@ -19,6 +20,7 @@ def baseenv():
     return {
         "PGDATABASE": os.environ["PGDATABASE"],
         "PGHOST": os.environ["PGHOST"],
+        "PGPORT": os.environ["PGPORT"],
         "PGUSER": os.environ["PGUSER"],
     }
 
@@ -76,6 +78,7 @@ def metapostgrest():
     env = {
         "PGDATABASE": os.environ["PGDATABASE"],
         "PGHOST": os.environ["PGHOST"],
+        "PGPORT": os.environ["PGPORT"],
         "PGUSER": role,
         "PGRST_DB_ANON_ROLE": role,
         "PGRST_DB_CONFIG": "true",

test/io/test_auth.py

@@ -165,7 +165,7 @@ def relativeSeconds(sec):
 
 def test_fail_with_invalid_password(defaultenv):
     "Connecting with an invalid password should fail without retries."
-    uri = f'postgresql://?dbname={defaultenv["PGDATABASE"]}&host={defaultenv["PGHOST"]}&user=some_protected_user&password=invalid_pass'
+    uri = f'postgresql://?dbname={defaultenv["PGDATABASE"]}&host={defaultenv["PGHOST"]}&port={defaultenv["PGPORT"]}&user=some_protected_user&password=invalid_pass'
     env = {**defaultenv, "PGRST_DB_URI": uri}
     with run(env=env, wait_for_readiness=False) as postgrest:
         exitCode = wait_until_exit(postgrest)

test/io/test_io.py

@@ -1653,7 +1653,7 @@ def test_log_listener_connection_start(defaultenv):
         # Check for the listener start message containing host and port
         # Do not check if pg version is displayed properly as it is tricky to test it
         assert any(
-            f'"{defaultenv["PGHOST"]}:5432" and listening for database notifications on the "pgrst" channel'
+            f'"{defaultenv["PGHOST"]}:{defaultenv["PGPORT"]}" and listening for database notifications on the "pgrst" channel'
             in line
             for line in output
         )

test/spec/Feature/ConcurrentSpec.hs

@@ -7,15 +7,14 @@ module Feature.ConcurrentSpec where
 import Control.Concurrent.Async (mapConcurrently)
 import Network.Wai              (Application)
 
-import Control.Monad.Base
 import Control.Monad.Trans.Control
 
-import Network.Wai.Test        (Session)
 import Test.Hspec
 import Test.Hspec.Wai
-import Test.Hspec.Wai.Internal
 import Test.Hspec.Wai.JSON
 
+import SpecHelper ()
+
 import Protolude hiding (get)
 
 spec :: SpecWith ((), Application)
@@ -34,15 +33,3 @@ raceTest :: Int -> WaiExpectation st -> WaiExpectation st
 raceTest times = liftBaseDiscard go
  where
   go test = void $ mapConcurrently (const test) [1..times]
-
-instance MonadBaseControl IO (WaiSession st) where
-  type StM (WaiSession st) a = StM Session a
-  liftBaseWith f = WaiSession $
-    liftBaseWith $ \runInBase ->
-      f $ \k -> runInBase (unWaiSession k)
-  restoreM = WaiSession . restoreM
-  {-# INLINE liftBaseWith #-}
-  {-# INLINE restoreM #-}
-
-instance MonadBase IO (WaiSession st) where
-  liftBase = liftIO

test/spec/Feature/MetricsSpec.hs

@@ -1,6 +1,7 @@
 {-# LANGUAGE DataKinds           #-}
 {-# LANGUAGE FlexibleContexts    #-}
 {-# LANGUAGE MonadComprehensions #-}
+{-# LANGUAGE NamedFieldPuns      #-}
 {-# LANGUAGE TypeApplications    #-}
 
 module Feature.MetricsSpec where
@@ -17,11 +18,11 @@ import           SpecHelper
 import           Test.Hspec            (SpecWith, describe, it)
 import           Test.Hspec.Wai        (getState)
 
-spec :: SpecWith ((AppState.AppState, Metrics.MetricsState, ObsChan), Application)
+spec :: SpecWith (SpecState, Application)
 spec = describe "Server started with metrics enabled" $ do
   it "Should update pgrst_schema_cache_loads_total[SUCCESS]" $ do
-    (appState, metrics, obsChan) <- getState
-    let waitFor = waitForObs obsChan
+    SpecState{specAppState = appState, specMetrics = metrics, specObsChan} <- getState
+    let waitFor = waitForObs specObsChan
 
     liftIO $ checkState' metrics [
         schemaCacheLoads "SUCCESS" (+1)
@@ -30,8 +31,8 @@ spec = describe "Server started with metrics enabled" $ do
         waitFor (1 * sec) "SchemaCacheLoadedObs" $ \x -> [ o | o@(SchemaCacheLoadedObs{}) <- pure x]
 
   it "Should update pgrst_schema_cache_loads_total[ERROR]" $ do
-    (appState, metrics, obsChan) <- getState
-    let waitFor = waitForObs obsChan
+    SpecState{specAppState = appState, specMetrics = metrics, specObsChan} <- getState
+    let waitFor = waitForObs specObsChan
 
     liftIO $ checkState' metrics [
         schemaCacheLoads "FAIL" (+1),

test/spec/Feature/ToxiSpec.hs

@@ -0,0 +1,33 @@
+{-# LANGUAGE DataKinds           #-}
+{-# LANGUAGE FlexibleContexts    #-}
+{-# LANGUAGE MonadComprehensions #-}
+{-# LANGUAGE NamedFieldPuns      #-}
+module Feature.ToxiSpec where
+
+import           Control.Monad.Trans.Control (liftBaseDiscard)
+import           Network.Wai                 (Application)
+import qualified PostgREST.AppState          as AppState
+import           Protolude                   hiding (get)
+import           SpecHelper
+import           Test.Hspec                  (SpecWith, describe, it)
+import           Test.Hspec.Wai
+import           Toxiproxy                   (withDisabled)
+
+spec :: SpecWith (SpecState, Application)
+spec = describe "Tests using Toxiproxy" $ do
+  it "Should return 503 on temporary database server unavailability" $ do
+    SpecState{specAppState, specToxiProxy} <- getState
+
+    -- make sure there are no open connections
+    liftIO $ AppState.flushPool specAppState
+
+    liftBaseDiscard (withDisabled specToxiProxy) $ do
+        void $ get "/items?id=eq.5"
+            `shouldRespondWith` 503
+
+    void $ get "/items?id=eq.5"
+        `shouldRespondWith` 200
+
+    liftBaseDiscard (withDisabled specToxiProxy) $ do
+        void $ get "/items?id=eq.5"
+            `shouldRespondWith` 503

test/spec/Main.hs

@@ -69,6 +69,7 @@ import qualified Feature.Query.UpdateSpec
 import qualified Feature.Query.UpsertSpec
 import qualified Feature.RollbackSpec
 import qualified Feature.RpcPreRequestGucsSpec
+import qualified Feature.ToxiSpec
 import           PostgREST.Observation                 (Observation (HasqlPoolObs))
 
 
@@ -91,6 +92,16 @@ main = do
     , P.observationHandler $ (writeChan poolChan <> Metrics.observationMetrics metricsState) . HasqlPoolObs
     ]
 
+  toxicPool <- P.acquire $ P.settings
+    [ P.size 3
+    , P.acquisitionTimeout 10
+    , P.agingTimeout 60
+    , P.idlenessTimeout 60
+    , P.staticConnectionSettings (toUtf8 $ configDbUri toxicCfg)
+    -- make sure metrics are updated and pool observations published to poolChan
+    , P.observationHandler $ (writeChan poolChan <> Metrics.observationMetrics metricsState) . HasqlPoolObs
+    ]
+
   actualPgVersion <- either (panic . show) id <$> P.use pool (queryPgVersion False)
 
   -- cached schema cache so most tests run fast
@@ -109,10 +120,10 @@ main = do
       -- duplicate poolChan as a starting point
       obsChan <- dupChan poolChan
       stateObsChan <- newObsChan obsChan
-      appState <- AppState.initWithPool sockets pool config loggerState metricsState (Metrics.observationMetrics metricsState <> writeChan obsChan)
+      appState <- AppState.initWithPool sockets toxicPool config loggerState metricsState (Metrics.observationMetrics metricsState <> writeChan obsChan)
       AppState.putPgVersion appState actualPgVersion
       AppState.putSchemaCache appState (Just sCache)
-      return ((appState, metricsState, stateObsChan), postgrest (configLogLevel config) appState (pure ()))
+      return (SpecState appState metricsState stateObsChan testToxiProxy, postgrest (configLogLevel config) appState (pure ()))
 
     -- For tests that run with the same schema cache
     app = initApp baseSchemaCache ()
@@ -298,8 +309,10 @@ main = do
     before (initApp baseSchemaCache metricsState testCfgJwtCache) $
       describe "Feature.Auth.JwtCacheSpec" Feature.Auth.JwtCacheSpec.spec
 
-    before observationsApp $
-      describe "Feature.MetricsSpec" Feature.MetricsSpec.spec
+    traverse_ (before observationsApp . uncurry describe) [
+        ("Feature.MetricsSpec", Feature.MetricsSpec.spec)
+      , ("Feature.ToxiSpec", Feature.ToxiSpec.spec)
+      ]
 
   where
     loadSCache pool conf =