Skip to content

Commit 9d6712c

Browse files
mkleczekmkleczek
committed
test(refactor): Add Toxiproxy Hspec test infrastructure
2 parents ea4ed53 + 67cc56c commit 9d6712c

10 files changed

Lines changed: 623 additions & 29 deletions

File tree

‎nix/tools/tests.nix‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,8 @@ let
2929
}
3030
''
3131
${withTools.withPg} -f test/spec/fixtures/load.sql \
32-
${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec -- "''${_arg_leftovers[@]}"
32+
${withTools.withToxiproxyPgProxy} \
33+
${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec -- "''${_arg_leftovers[@]}"
3334
'';
3435

3536
testDoctests =

‎postgrest.cabal‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,7 @@ test-suite spec
219219
Feature.ConcurrentSpec
220220
Feature.CorsSpec
221221
Feature.ExtraSearchPathSpec
222+
Feature.MetricsSpec
222223
Feature.NoSuperuserSpec
223224
Feature.ObservabilitySpec
224225
Feature.OpenApi.DisabledOpenApiSpec
@@ -258,7 +259,9 @@ test-suite spec
258259
Feature.Query.UpsertSpec
259260
Feature.RollbackSpec
260261
Feature.RpcPreRequestGucsSpec
262+
Feature.ToxiSpec
261263
SpecHelper
264+
Toxiproxy
262265
build-depends: base >= 4.9 && < 4.20
263266
, aeson >= 2.0.3 && < 2.3
264267
, aeson-qq >= 0.8.1 && < 0.9
@@ -289,6 +292,9 @@ test-suite spec
289292
, transformers-base >= 0.4.4 && < 0.5
290293
, wai >= 3.2.1 && < 3.3
291294
, wai-extra >= 3.0.19 && < 3.2
295+
, servant-client >= 0.20.3.0 && < 0.21
296+
, servant >= 0.20.3.0 && < 0.21
297+
, http-client >= 0.7.19 && < 0.8
292298
ghc-options: -threaded -O0 -Werror -Wall -fwarn-identities
293299
-fno-spec-constr -optP-Wno-nonportable-include-path
294300
-fno-warn-missing-signatures

‎src/PostgREST/AppState.hs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
module PostgREST.AppState
66
( AppState
77
, destroy
8+
, flushPool
89
, getConfig
910
, getSchemaCache
1011
, getMainThreadId
@@ -15,6 +16,7 @@ module PostgREST.AppState
1516
, getJwtCacheState
1617
, init
1718
, initWithPool
19+
, putConfig -- For tests TODO refactoring
1820
, putNextListenerDelay
1921
, putSchemaCache
2022
, putPgVersion

‎src/PostgREST/Observation.hs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
{-# LANGUAGE DeriveGeneric #-}
12
{-|
23
Module : PostgREST.Observation
34
Description : This module holds an Observation type which is the core of Observability for PostgREST.
@@ -55,6 +56,7 @@ data Observation
5556
| JwtCacheLookup Bool
5657
| JwtCacheEviction
5758
| WarpErrorObs Text
59+
deriving (Generic)
5860

5961
data ObsFatalError = ServerAuthError | ServerPgrstBug | ServerError42P05 | ServerError08P01
6062

‎test/spec/Feature/ConcurrentSpec.hs‎

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,14 @@ module Feature.ConcurrentSpec where
77
import Control.Concurrent.Async (mapConcurrently)
88
import Network.Wai (Application)
99

10-
import Control.Monad.Base
1110
import Control.Monad.Trans.Control
1211

13-
import Network.Wai.Test (Session)
1412
import Test.Hspec
1513
import Test.Hspec.Wai
16-
import Test.Hspec.Wai.Internal
1714
import Test.Hspec.Wai.JSON
1815

16+
import SpecHelper ()
17+
1918
import Protolude hiding (get)
2019

2120
spec :: SpecWith ((), Application)
@@ -34,15 +33,3 @@ raceTest :: Int -> WaiExpectation st -> WaiExpectation st
3433
raceTest times = liftBaseDiscard go
3534
where
3635
go test = void $ mapConcurrently (const test) [1..times]
37-
38-
instance MonadBaseControl IO (WaiSession st) where
39-
type StM (WaiSession st) a = StM Session a
40-
liftBaseWith f = WaiSession $
41-
liftBaseWith $ \runInBase ->
42-
f $ \k -> runInBase (unWaiSession k)
43-
restoreM = WaiSession . restoreM
44-
{-# INLINE liftBaseWith #-}
45-
{-# INLINE restoreM #-}
46-
47-
instance MonadBase IO (WaiSession st) where
48-
liftBase = liftIO

‎test/spec/Feature/MetricsSpec.hs‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
{-# LANGUAGE DataKinds #-}
2+
{-# LANGUAGE FlexibleContexts #-}
3+
{-# LANGUAGE MonadComprehensions #-}
4+
{-# LANGUAGE NamedFieldPuns #-}
5+
{-# LANGUAGE TypeApplications #-}
6+
7+
module Feature.MetricsSpec where
8+
9+
import Data.List (lookup)
10+
import Network.Wai (Application)
11+
import qualified PostgREST.AppState as AppState
12+
import PostgREST.Config (AppConfig (configDbSchemas))
13+
import qualified PostgREST.Metrics as Metrics
14+
import PostgREST.Observation
15+
import Prometheus (getCounter, getVectorWith)
16+
import Protolude
17+
import SpecHelper
18+
import Test.Hspec (SpecWith, describe, it)
19+
import Test.Hspec.Wai (getState)
20+
21+
spec :: SpecWith (SpecState, Application)
22+
spec = describe "Server started with metrics enabled" $ do
23+
it "Should update pgrst_schema_cache_loads_total[SUCCESS]" $ do
24+
SpecState{specAppState = appState, specMetrics = metrics, specObsChan} <- getState
25+
let waitFor = waitForObs specObsChan
26+
27+
liftIO $ checkState' metrics [
28+
schemaCacheLoads "SUCCESS" (+1)
29+
] $ do
30+
AppState.schemaCacheLoader appState
31+
waitFor (1 * sec) "SchemaCacheLoadedObs" $ \x -> [ o | o@(SchemaCacheLoadedObs{}) <- pure x]
32+
33+
it "Should update pgrst_schema_cache_loads_total[ERROR]" $ do
34+
SpecState{specAppState = appState, specMetrics = metrics, specObsChan} <- getState
35+
let waitFor = waitForObs specObsChan
36+
37+
liftIO $ checkState' metrics [
38+
schemaCacheLoads "FAIL" (+1),
39+
schemaCacheLoads "SUCCESS" (+1)
40+
] $ do
41+
AppState.getConfig appState >>= \prev -> do
42+
AppState.putConfig appState $ prev { configDbSchemas = pure "bad_schema" }
43+
AppState.schemaCacheLoader appState
44+
waitFor (1 * sec) "SchemaCacheErrorObs" $ \x -> [ o | o@(SchemaCacheErrorObs{}) <- pure x]
45+
AppState.putConfig appState prev
46+
47+
-- wait up to 2 secs so that retry can happen
48+
waitFor (2 * sec) "SchemaCacheLoadedObs" $ \x -> [ o | o@(SchemaCacheLoadedObs{}) <- pure x]
49+
50+
where
51+
-- prometheus-client api to handle vectors is convoluted
52+
schemaCacheLoads label = expectField @"schemaCacheLoads" $
53+
fmap (maybe (0::Int) round . lookup label) . (`getVectorWith` getCounter)
54+
sec = 1000000

‎test/spec/Feature/ToxiSpec.hs‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
{-# LANGUAGE DataKinds #-}
2+
{-# LANGUAGE FlexibleContexts #-}
3+
{-# LANGUAGE MonadComprehensions #-}
4+
{-# LANGUAGE NamedFieldPuns #-}
5+
module Feature.ToxiSpec where
6+
7+
import Control.Monad.Trans.Control (liftBaseDiscard)
8+
import Network.Wai (Application)
9+
import qualified PostgREST.AppState as AppState
10+
import Protolude hiding (get)
11+
import SpecHelper
12+
import Test.Hspec (SpecWith, describe, it)
13+
import Test.Hspec.Wai
14+
import Toxiproxy (withDisabled)
15+
16+
spec :: SpecWith (SpecState, Application)
17+
spec = describe "Tests using Toxiproxy" $ do
18+
it "Should return 503 on temporary database server unavailability" $ do
19+
pendingWith "TODO fix"
20+
SpecState{specAppState, specToxiProxy} <- getState
21+
22+
-- make sure there are no open connections
23+
liftIO $ AppState.flushPool specAppState
24+
25+
liftBaseDiscard (withDisabled specToxiProxy) $ do
26+
void $ get "/items?id=eq.5"
27+
`shouldRespondWith` 503
28+
29+
void $ get "/items?id=eq.5"
30+
`shouldRespondWith` 200
31+
32+
liftBaseDiscard (withDisabled specToxiProxy) $ do
33+
void $ get "/items?id=eq.5"
34+
`shouldRespondWith` 503

‎test/spec/Main.hs‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import qualified PostgREST.AppState as AppState
1919
import qualified PostgREST.Logger as Logger
2020
import qualified PostgREST.Metrics as Metrics
2121

22+
import qualified Data.Text as T
2223
import qualified Feature.Auth.AsymmetricJwtSpec
2324
import qualified Feature.Auth.AudienceJwtSecretSpec
2425
import qualified Feature.Auth.AuthSpec
@@ -29,6 +30,7 @@ import qualified Feature.Auth.NoJwtSecretSpec
2930
import qualified Feature.ConcurrentSpec
3031
import qualified Feature.CorsSpec
3132
import qualified Feature.ExtraSearchPathSpec
33+
import qualified Feature.MetricsSpec
3234
import qualified Feature.NoSuperuserSpec
3335
import qualified Feature.ObservabilitySpec
3436
import qualified Feature.OpenApi.DisabledOpenApiSpec
@@ -68,24 +70,48 @@ import qualified Feature.Query.UpdateSpec
6870
import qualified Feature.Query.UpsertSpec
6971
import qualified Feature.RollbackSpec
7072
import qualified Feature.RpcPreRequestGucsSpec
73+
import qualified Feature.ToxiSpec
74+
import PostgREST.Observation (Observation (HasqlPoolObs))
75+
import qualified System.Environment as System
7176

7277

7378
main :: IO ()
7479
main = do
80+
poolChan <- newChan
81+
-- make sure poolChan is not growing indefinitely
82+
-- start a thread that drains the channel
83+
-- this is necessary because test cases operate on
84+
-- copies so poolChan is never read from
85+
void $ forkIO $ forever $ readChan poolChan
86+
metricsState <- Metrics.init (configDbPoolSize testCfg)
7587
pool <- P.acquire $ P.settings
7688
[ P.size 3
7789
, P.acquisitionTimeout 10
7890
, P.agingTimeout 60
7991
, P.idlenessTimeout 60
8092
, P.staticConnectionSettings (toUtf8 $ configDbUri testCfg)
93+
-- make sure metrics are updated and pool observations published to poolChan
94+
, P.observationHandler $ (writeChan poolChan <> Metrics.observationMetrics metricsState) . HasqlPoolObs
95+
]
96+
97+
toxiProxyName <- T.pack <$> System.getEnv "TOXI_PROXY_NAME"
98+
toxiPgPort <- T.pack <$> System.getEnv "TOXI_PGPORT"
99+
pgPort <- T.pack <$> System.getEnv "PGPORT"
100+
toxicPool <- P.acquire $ P.settings
101+
[ P.size 3
102+
, P.acquisitionTimeout 10
103+
, P.agingTimeout 60
104+
, P.idlenessTimeout 60
105+
, P.staticConnectionSettings (toUtf8 $ configDbUri (baseCfg { configDbUri = "postgresql://localhost:" <> toxiPgPort }))
106+
-- make sure metrics are updated and pool observations published to poolChan
107+
, P.observationHandler $ (writeChan poolChan <> Metrics.observationMetrics metricsState) . HasqlPoolObs
81108
]
82109

83110
actualPgVersion <- either (panic . show) id <$> P.use pool (queryPgVersion False)
84111

85112
-- cached schema cache so most tests run fast
86113
baseSchemaCache <- loadSCache pool testCfg
87114
loggerState <- Logger.init
88-
metricsState <- Metrics.init (configDbPoolSize testCfg)
89115

90116
let
91117
initApp sCache st config = do
@@ -94,6 +120,15 @@ main = do
94120
AppState.putSchemaCache appState (Just sCache)
95121
return (st, postgrest (configLogLevel config) appState (pure ()))
96122

123+
initObservationsApp sCache config = do
124+
-- duplicate poolChan as a starting point
125+
obsChan <- dupChan poolChan
126+
stateObsChan <- newObsChan obsChan
127+
appState <- AppState.initWithPool toxicPool config loggerState metricsState (Metrics.observationMetrics metricsState <> writeChan obsChan)
128+
AppState.putPgVersion appState actualPgVersion
129+
AppState.putSchemaCache appState (Just sCache)
130+
return (SpecState appState metricsState stateObsChan $ testToxiProxy toxiProxyName toxiPgPort pgPort, postgrest (configLogLevel config) appState (pure ()))
131+
97132
-- For tests that run with the same schema cache
98133
app = initApp baseSchemaCache ()
99134

@@ -122,6 +157,7 @@ main = do
122157
obsApp = app testObservabilityCfg
123158
serverTiming = app testCfgServerTiming
124159
aggregatesEnabled = app testCfgAggregatesEnabled
160+
observationsApp = initObservationsApp baseSchemaCache testCfg
125161

126162
extraSearchPathApp = appDbs testCfgExtraSearchPath
127163
unicodeApp = appDbs testUnicodeCfg
@@ -277,6 +313,11 @@ main = do
277313
before (initApp baseSchemaCache metricsState testCfgJwtCache) $
278314
describe "Feature.Auth.JwtCacheSpec" Feature.Auth.JwtCacheSpec.spec
279315

316+
traverse_ (before observationsApp . uncurry describe) [
317+
("Feature.MetricsSpec", Feature.MetricsSpec.spec)
318+
, ("Feature.ToxiSpec", Feature.ToxiSpec.spec)
319+
]
320+
280321
where
281322
loadSCache pool conf =
282323
either (panic.show) id <$> P.use pool (HT.transaction HT.ReadCommitted HT.Read $ querySchemaCache conf)

0 commit comments

Comments
 (0)