Skip to content

Commit 053ce13

Browse files
mkleczekmkleczek
committed
add: load schema cache dump from URI at startup
1 parent ab41649 commit 053ce13

13 files changed

Lines changed: 385 additions & 38 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ All notable changes to this project will be documented in this file. From versio
1515
- Log schema cache queries timings on `log-level=debug` by @steve-chavez in #4805
1616
- Add GHC runtime metrics to the metrics endpoint by @mkleczek in #4862
1717
- Support running the admin server on a unix socket by @wolfgangwalther in #5003
18+
- Add `--schema-cache-uri` to asynchronously load an initial schema cache dump from `http(s)` or local `file` URIs by @mkleczek in #5031
1819

1920
### Fixed
2021

‎docs/references/cli.rst‎

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ PostgREST provides a CLI with the options listed below:
88
.. code:: text
99
1010
Usage: postgrest [-v|--version] [-e|--example] [--dump-config | --dump-schema | --ready]
11-
[FILENAME]
11+
[--schema-cache-uri URI] [FILENAME]
1212
1313
PostgREST / create a REST API to an existing Postgres
1414
database
@@ -22,6 +22,8 @@ PostgREST provides a CLI with the options listed below:
2222
output structure is unstable)
2323
--ready Checks the health of PostgREST by doing a request on
2424
the admin server /ready endpoint
25+
--schema-cache-uri URI
26+
Try pre-loading schema cache from provided URI
2527
FILENAME Path to configuration file
2628
2729
FILENAME
@@ -74,6 +76,68 @@ Dump Schema
7476
7577
Dumps the schema cache in JSON format.
7678

79+
Schema Cache URI
80+
----------------
81+
82+
.. code:: bash
83+
84+
$ postgrest --schema-cache-uri http://localhost/schema-cache.json
85+
86+
Tries to load the initial :ref:`schema_cache` from the given URI when starting PostgREST.
87+
The load is asynchronous and does not block application startup. If the URI cannot be
88+
loaded, PostgREST logs the failure and continues with the regular database-backed
89+
schema cache load.
90+
91+
The supported URI schemes are:
92+
93+
``file:``
94+
Loads a schema cache dump from a local file. The URI must point to a local path, for
95+
example ``file:///var/lib/postgrest/schema-cache.json``. Remote file authorities are
96+
not supported.
97+
98+
``http:`` and ``https:``
99+
Loads a schema cache dump over HTTP(S). The response body must be a schema cache JSON
100+
dump, such as the output of ``postgrest --dump-schema`` or the runtime cache exposed
101+
by another PostgREST instance's :ref:`admin_server`.
102+
103+
Other schemes are rejected as unsupported.
104+
105+
Load From a Local File
106+
~~~~~~~~~~~~~~~~~~~~~~
107+
108+
First, write a schema cache dump to a file:
109+
110+
.. code:: bash
111+
112+
$ postgrest --dump-schema > /var/lib/postgrest/schema-cache.json
113+
114+
Then start PostgREST with a ``file:`` URI:
115+
116+
.. code:: bash
117+
118+
$ postgrest --schema-cache-uri file:///var/lib/postgrest/schema-cache.json
119+
120+
Load From Another PostgREST Instance
121+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
122+
123+
The :ref:`admin_server` exposes the current runtime schema cache on the
124+
``/schema_cache`` endpoint. If one PostgREST instance exposes its admin server on
125+
``localhost:3001``, a new instance can load from it with:
126+
127+
.. code:: bash
128+
129+
$ postgrest --schema-cache-uri http://localhost:3001/schema_cache
130+
131+
In Kubernetes, the same pattern can use an internal Service that points at ready
132+
PostgREST pods:
133+
134+
.. code:: bash
135+
136+
$ postgrest --schema-cache-uri http://postgrest-admin:3001/schema_cache
137+
138+
The admin endpoint should stay internal to your deployment or be protected by your
139+
platform's networking controls.
140+
77141
Ready Flag
78142
----------
79143

‎src/library/PostgREST/AppState.hs‎

Lines changed: 60 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
1-
{-# LANGUAGE LambdaCase #-}
2-
{-# LANGUAGE NamedFieldPuns #-}
3-
{-# LANGUAGE RecordWildCards #-}
4-
{-# LANGUAGE RecursiveDo #-}
1+
{-# LANGUAGE LambdaCase #-}
2+
{-# LANGUAGE NamedFieldPuns #-}
3+
{-# LANGUAGE RecordWildCards #-}
4+
{-# LANGUAGE RecursiveDo #-}
5+
{-# LANGUAGE TupleSections #-}
6+
{-# LANGUAGE TypeApplications #-}
57

68
module PostgREST.AppState
79
( AppState
@@ -48,13 +50,14 @@ import Control.AutoUpdate (defaultUpdateSettings, mkAutoUpdate,
4850
import Control.Retry (RetryPolicy, RetryStatus (..), capDelay,
4951
exponentialBackoff, retrying,
5052
rsPreviousDelay)
51-
import Data.IORef (IORef, atomicWriteIORef, newIORef,
52-
readIORef)
53+
import Data.IORef (IORef, atomicModifyIORef, atomicWriteIORef,
54+
newIORef, readIORef)
5355
import Data.Time.Clock (UTCTime, getCurrentTime)
5456

5557
import Control.Concurrent.STM (TMVar, newEmptyTMVarIO,
5658
putTMVar, readTMVar,
57-
tryReadTMVar, tryTakeTMVar)
59+
tryPutTMVar, tryReadTMVar,
60+
tryTakeTMVar)
5861
import PostgREST.Auth.JwtCache (JwtCacheState, update)
5962
import PostgREST.Config (AppConfig (..),
6063
readAppConfig,
@@ -70,6 +73,13 @@ import PostgREST.SchemaCache (SchemaCache (..),
7073
showSummary)
7174
import PostgREST.SchemaCache.Identifiers (quoteQi)
7275

76+
import qualified Data.Aeson as JSON
77+
import qualified Data.ByteString.Lazy as LBS
78+
import qualified Network.HTTP.Client as HC
79+
import Network.URI (URI (..), URIAuth (..),
80+
parseURI, unEscapeString)
81+
import System.IO.Error (userError)
82+
7383
import Protolude
7484

7585
data AppState = AppState
@@ -109,16 +119,56 @@ newtype SchemaCacheStatus = SchemaCacheStatus
109119
{ getSCStatusTMVar :: TMVar Bool
110120
}
111121

112-
init :: AppConfig -> IO () -> IO AppState
113-
init conf@AppConfig{configLogLevel, configDbPoolSize} appKiller = do
122+
init :: AppConfig -> IO () -> Maybe Text -> IO AppState
123+
init conf@AppConfig{configLogLevel, configDbPoolSize} appKiller schemaCacheLoadUri = do
114124
loggerState <- Logger.init
115125
metricsState <- Metrics.init configDbPoolSize
116126
let observer = liftA2 (>>) (Logger.observationLogger loggerState configLogLevel) (Metrics.observationMetrics metricsState)
117127

118128
observer $ AppStartObs prettyVersion
119129

120130
pool <- initPool conf observer
121-
initWithPool pool conf loggerState metricsState observer appKiller
131+
appState <- initWithPool pool conf loggerState metricsState observer appKiller
132+
133+
runInitialSchemaCacheLoader observer schemaCacheLoadUri appState
134+
135+
pure appState
136+
137+
runInitialSchemaCacheLoader :: ObservationHandler -> Maybe Text -> AppState -> IO ()
138+
runInitialSchemaCacheLoader observer schemaCacheLoadUri AppState{stateSchemaCache, stateSCacheStatus=SchemaCacheStatus{getSCStatusTMVar}} = do
139+
void $ forkIO $
140+
traverse (fetchInitialSchemaCache observer) schemaCacheLoadUri >>= foldMap setInitialSchemaCache . join
141+
where
142+
setInitialSchemaCache sc =
143+
whenM (atomically $ tryPutTMVar getSCStatusTMVar True) $
144+
atomicModifyIORef stateSchemaCache $ (, ()) . maybe (Just sc) Just
145+
146+
fetchInitialSchemaCache :: ObservationHandler -> Text -> IO (Maybe SchemaCache)
147+
fetchInitialSchemaCache observer uri = flip catches [
148+
Handler (handleError @IOException),
149+
Handler (handleError @HC.HttpException),
150+
Handler (handleError @JSON.AesonException)
151+
] $ maybe (throwIO $ userError "Invalid schema cache dump URI") pure =<< traverse fetchURI (parseURI $ toS uri)
152+
where
153+
handleError :: Show e => e -> IO (Maybe a)
154+
handleError = (Nothing <$) . observer . SchemaCacheInitialLoadFailureObs uri . show
155+
156+
fetchURI URI{uriScheme, ..}
157+
| uriScheme == "file:" = do
158+
path <- fileURIPath uriAuthority uriPath
159+
Just <$> (JSON.throwDecode =<< LBS.readFile path)
160+
| uriScheme `elem` ["http:", "https:"] = do
161+
request <- HC.parseUrlThrow $ toS uri
162+
manager <- HC.newManager HC.defaultManagerSettings
163+
HC.withResponse request manager $ \response ->
164+
Just <$> (JSON.throwDecode . LBS.fromChunks =<< HC.brConsume (HC.responseBody response))
165+
| otherwise =
166+
throwIO $ userError $ "Unsupported schema cache dump URI scheme: " <> uriScheme
167+
168+
fileURIPath Nothing path = pure $ unEscapeString path
169+
fileURIPath (Just URIAuth{uriRegName=""}) path = pure $ unEscapeString path
170+
fileURIPath (Just URIAuth{uriRegName="localhost"}) path = pure $ unEscapeString path
171+
fileURIPath _ _ = throwIO $ userError "Only local file URIs are supported"
122172

123173
initWithPool :: SQL.Pool -> AppConfig -> Logger.LoggerState -> Metrics.MetricsState -> ObservationHandler -> IO () -> IO AppState
124174
initWithPool pool conf loggerState metricsState observer appKiller = mdo

‎src/library/PostgREST/CLI.hs‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
1-
{-# LANGUAGE NamedFieldPuns #-}
2-
{-# LANGUAGE RecordWildCards #-}
1+
{-# LANGUAGE MonadComprehensions #-}
2+
{-# LANGUAGE NamedFieldPuns #-}
3+
{-# LANGUAGE RecordWildCards #-}
34
module PostgREST.CLI
45
( main
56
, CLI (..)
@@ -48,7 +49,7 @@ runAppCommand conf@AppConfig{..} runCmd = do
4849
-- explicitly close the connections to PostgreSQL on shutdown.
4950
-- 'AppState.destroy' takes care of that.
5051
bracket
51-
(AppState.init conf (killThread mainThreadId))
52+
(AppState.init conf (killThread mainThreadId) [schemaCacheLoadUri | CmdRun (Just schemaCacheLoadUri) <- Just runCmd])
5253
AppState.destroy
5354
(\appState -> case runCmd of
5455
CmdDumpConfig -> do
@@ -57,7 +58,7 @@ runAppCommand conf@AppConfig{..} runCmd = do
5758
CmdDumpSchema -> do
5859
when configDbConfig $ AppState.readInDbConfig True appState
5960
putStrLn =<< dumpSchema appState
60-
CmdRun -> App.run appState mainThreadIdRef)
61+
CmdRun _ -> App.run appState mainThreadIdRef)
6162

6263
-- | Dump SchemaCache schema to JSON
6364
dumpSchema :: AppState -> IO LBS.ByteString
@@ -86,7 +87,7 @@ data ClientCommand
8687
= CmdReady
8788

8889
data RunCommand
89-
= CmdRun
90+
= CmdRun (Maybe Text)
9091
| CmdDumpConfig
9192
| CmdDumpSchema
9293

@@ -120,25 +121,34 @@ readCLIShowHelp =
120121
cliParser :: O.Parser CLI
121122
cliParser =
122123
CLI
123-
<$> (dumpConfigFlag <|> dumpSchemaFlag <|> readyFlag)
124+
<$> (dumpConfigFlag <|> dumpSchemaFlag <|> readyFlag <|> runCommand)
124125
<*> O.optional configFileOption
125126

127+
runCommand =
128+
Run . CmdRun <$> O.optional schemaCacheUriOption
129+
130+
schemaCacheUriOption =
131+
O.strOption $
132+
O.long "schema-cache-uri"
133+
<> O.metavar "URI"
134+
<> O.help "Try pre-loading schema cache from provided URI"
135+
126136
configFileOption =
127137
O.strArgument $
128138
O.metavar "FILENAME"
129139
<> O.help "Path to configuration file"
130140

131141
dumpConfigFlag =
132-
O.flag (Run CmdRun) (Run CmdDumpConfig) $
142+
O.flag' (Run CmdDumpConfig) $
133143
O.long "dump-config"
134144
<> O.help "Dump loaded configuration and exit"
135145

136146
dumpSchemaFlag =
137-
O.flag (Run CmdRun) (Run CmdDumpSchema) $
147+
O.flag' (Run CmdDumpSchema) $
138148
O.long "dump-schema"
139149
<> O.help "Dump loaded schema as JSON and exit (for debugging, output structure is unstable)"
140150

141151
readyFlag =
142-
O.flag (Run CmdRun) (Client CmdReady) $
152+
O.flag' (Client CmdReady) $
143153
O.long "ready"
144154
<> O.help "Checks the health of PostgREST by doing a request on the admin server /ready endpoint"

‎src/library/PostgREST/Logger.hs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,8 @@ observationMessages = \case
160160
<> " and "
161161
<> "db-extra-search-path=" <> T.intercalate "," extraPaths
162162
<> ". " <> jsonMessage usageErr
163+
SchemaCacheInitialLoadFailureObs uri err ->
164+
pure $ "Failed to load schema cache dump from " <> uri <> ". " <> err
163165
SchemaCacheQueriedObs resultTime timings ->
164166
[ "Schema cache queried in " <> showMillis resultTime <> " milliseconds " ] <>
165167
let showTimings qt = [ T.intercalate ", " $ (\(l, v) -> T.decodeUtf8 l <> ": " <> v <> " ms") <$> queryTimingsWLabels qt ] in

‎src/library/PostgREST/MediaType.hs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,17 +43,17 @@ data MediaType
4343
| MTVndSingularJSON Bool
4444
-- TODO MTVndPlan should only have its options as [Text]. Its ResultAggregate should have the typed attributes.
4545
| MTVndPlan MediaType MTVndPlanFormat [MTVndPlanOption]
46-
deriving (Eq, Show, Generic, JSON.ToJSON)
46+
deriving (Eq, Show, Generic, JSON.FromJSON, JSON.ToJSON)
4747
instance Hashable MediaType
4848

4949
data MTVndPlanOption
5050
= PlanAnalyze | PlanVerbose | PlanSettings | PlanBuffers | PlanWAL
51-
deriving (Eq, Show, Generic, JSON.ToJSON)
51+
deriving (Eq, Show, Generic, JSON.FromJSON, JSON.ToJSON)
5252
instance Hashable MTVndPlanOption
5353

5454
data MTVndPlanFormat
5555
= PlanJSON | PlanText
56-
deriving (Eq, Show, Generic, JSON.ToJSON)
56+
deriving (Eq, Show, Generic, JSON.FromJSON, JSON.ToJSON)
5757
instance Hashable MTVndPlanFormat
5858

5959
-- | Convert MediaType to a Content-Type HTTP Header

‎src/library/PostgREST/Observation.hs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ data Observation
3434
| DBConnectedObs Text
3535
| SchemaCacheEmptyObs
3636
| SchemaCacheErrorObs (NonEmpty Text) [Text] SQL.UsageError
37+
| SchemaCacheInitialLoadFailureObs Text Text
3738
| SchemaCacheQueriedObs Double (Maybe QueryTimings)
3839
| SchemaCacheLoadedObs Double Text
3940
| ConnectionRetryObs Int

‎src/library/PostgREST/SchemaCache.hs‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ module PostgREST.SchemaCache
2828
, queryTimingsWLabels
2929
) where
3030

31-
import Data.Aeson ((.=))
31+
import Data.Aeson ((.:), (.=))
3232
import qualified Data.Aeson as JSON
3333

3434
import qualified Data.ByteString.Char8 as BS
@@ -101,6 +101,17 @@ instance JSON.ToJSON SchemaCache where
101101
, "dbTimezones" .= JSON.toJSON tzs
102102
]
103103

104+
instance JSON.FromJSON SchemaCache where
105+
parseJSON = JSON.withObject "SchemaCache" $ \o -> do
106+
tabs <- o .: "dbTables"
107+
SchemaCache tabs
108+
<$> o .: "dbRelationships"
109+
<*> o .: "dbRoutines"
110+
<*> o .: "dbRepresentations"
111+
<*> o .: "dbMediaHandlers"
112+
<*> o .: "dbTimezones"
113+
<*> pure (tablesFuzzyIndex tabs)
114+
104115
showSummary :: SchemaCache -> Text
105116
showSummary (SchemaCache tbls rels routs reps mediaHdlrs tzs _) =
106117
T.intercalate ", "

‎src/library/PostgREST/SchemaCache/Identifiers.hs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import qualified Data.Text as T
2020
import Protolude
2121

2222
data RelIdentifier = RelId QualifiedIdentifier | RelAnyElement
23-
deriving (Eq, Ord, Generic, JSON.ToJSON, JSON.ToJSONKey, Show)
23+
deriving (Eq, Ord, Generic, JSON.FromJSON, JSON.FromJSONKey, JSON.ToJSON, JSON.ToJSONKey, Show)
2424
instance Hashable RelIdentifier
2525

2626
-- | Represents a pg identifier with a prepended schema name "schema.table".
@@ -30,7 +30,7 @@ data QualifiedIdentifier = QualifiedIdentifier
3030
{ qiSchema :: Schema
3131
, qiName :: TableName
3232
}
33-
deriving (Eq, Show, Ord, Generic, JSON.ToJSON, JSON.ToJSONKey)
33+
deriving (Eq, Show, Ord, Generic, JSON.FromJSON, JSON.FromJSONKey, JSON.ToJSON, JSON.ToJSONKey)
3434

3535
instance Hashable QualifiedIdentifier
3636

‎src/library/PostgREST/SchemaCache/Relationship.hs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ data Relationship = Relationship
3535
, relToOne :: Bool
3636
, relIsSelf :: Bool
3737
}
38-
deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
38+
deriving (Eq, Show, Ord, Generic, JSON.FromJSON, JSON.ToJSON)
3939

4040
-- | The relationship cardinality
4141
-- | https://en.wikipedia.org/wiki/Cardinality_(data_modeling)
@@ -48,7 +48,7 @@ data Cardinality
4848
-- ^ one-to-one, this is a refinement over M2O, operating on it is pretty much the same as M2O when isParent == False
4949
| M2M Junction
5050
-- ^ many-to-many
51-
deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
51+
deriving (Eq, Show, Ord, Generic, JSON.FromJSON, JSON.ToJSON)
5252

5353
type FKConstraint = Text
5454

@@ -60,7 +60,7 @@ data Junction = Junction
6060
, junColsSource :: [(FieldName, FieldName)]
6161
, junColsTarget :: [(FieldName, FieldName)]
6262
}
63-
deriving (Eq, Show, Ord, Generic, JSON.ToJSON)
63+
deriving (Eq, Show, Ord, Generic, JSON.FromJSON, JSON.ToJSON)
6464

6565
-- | Key based on the source table and the foreign table schema
6666
type RelationshipsMap = HM.HashMap (QualifiedIdentifier, Schema) [Relationship]

0 commit comments

Comments
 (0)