fix: Admin server dies silently

This is a workaround for Warp crashing with "thread blocked indefinitely in an STM transaction" when receiving eMFILE from accept.

The fix is to provide Warp with a custom accept function that handles eMFILE by logging it and keeping accepting connections.

Author: mkleczek

src/PostgREST/Admin.hs

@@ -1,8 +1,11 @@
+{-# LANGUAGE MonadComprehensions #-}
 module PostgREST.Admin
   ( runAdmin
   ) where
 
 import qualified Data.Aeson                as JSON
+import           Foreign.C.Error           (Errno (..), eMFILE)
+import           GHC.IO.Exception
 import qualified Network.HTTP.Types.Status as HTTP
 import qualified Network.Wai               as Wai
 import qualified Network.Wai.Handler.Warp  as Warp
@@ -13,6 +16,7 @@ import Network.Socket.ByteString
 
 import PostgREST.AppState    (AppState, getConfig)
 import PostgREST.Config      (AppConfig (..))
+import PostgREST.Debounce    (makeDebouncer)
 import PostgREST.MediaType   (MediaType (..), toContentType)
 import PostgREST.Metrics     (metricsToText)
 import PostgREST.Network     (resolveSocketToAddress)
@@ -23,18 +27,46 @@ import qualified PostgREST.AppState as AppState
 import qualified Network.Socket as NS
 import           Protolude
 
-runAdmin :: AppState -> Maybe NS.Socket -> IO (Maybe NS.Socket) -> Warp.Settings -> IO ()
-runAdmin appState maybeAdminSocket getSocketREST settings = do
+runAdmin :: AppState -> Maybe NS.Socket -> IO (Maybe NS.Socket) -> Warp.ServerState -> Warp.Settings -> IO ()
+runAdmin appState maybeAdminSocket getSocketREST serverState settings = do
   conf <- getConfig appState
   whenJust maybeAdminSocket $ \adminSocket -> do
     address <- resolveSocketToAddress adminSocket
+    -- log EMFILE at most once every 10s
+    logEMFILE <- makeDebouncer $ observer AdminServerAcceptEMFILEFailure *> threadDelay 10_000_000
+    let
+      -- Keep accepting on eMFILE
+      safeAccept sock =
+        accept sock `catchEMFile`
+          do
+            logEMFILE
+
+            connCount <- Warp.currentOpenConnections serverState
+            join $ atomically $ do
+              if connCount > 0 then do
+                -- There are open connections
+                -- wait for closing some and restart accepting
+                nextConnCount <- Warp.currentOpenConnectionsSTM serverState
+                check (nextConnCount < connCount)
+                pure $ safeAccept sock
+              else
+                pure $ do
+                  -- Backoff for 1ms so that we don't enter busy accept loop
+                  -- this should let the system recover fd's once the pressure is gone
+                  -- then restart accepting
+                  threadDelay 1_000
+                  safeAccept sock
+
     void . forkIO $ handle (onError adminSocket) $
-      Warp.runSettingsSocket (adminServerSettings conf address) adminSocket adminApp
+      Warp.runSettingsSocket (adminServerSettings conf address safeAccept) adminSocket adminApp
   where
     adminApp = admin appState getSocketREST
     observer = AppState.getObserver appState
-    adminServerSettings config addr=
+    isEMFILE err = Just eMFILE == fmap Errno (ioe_errno err)
+    catchEMFile action handler = handleJust (\e -> [ e | isEMFILE e]) (const handler) action
+    adminServerSettings config addr safeAccept = do
       settings
+        & Warp.setAccept safeAccept
         & Warp.setBeforeMainLoop (observer $ AdminStartObs addr)
         & maybe identity Warp.setPort (configAdminServerPort config)
 

src/PostgREST/App.hs

@@ -89,7 +89,8 @@ run appState = do
         readIORef mainSocketRef >>= foldMap NS.close
   Unix.installSignalHandlers observer closeSockets (AppState.schemaCacheLoader appState) (AppState.readInDbConfig False appState)
 
-  Admin.runAdmin appState adminSocket (readIORef mainSocketRef) (serverSettings conf)
+  (adminState, adminSettings) <- fmap (serverSettings conf) <$> Warp.makeSettingsAndServerState
+  Admin.runAdmin appState adminSocket (readIORef mainSocketRef) adminState adminSettings
 
   Listener.runListener appState
 
@@ -106,7 +107,7 @@ run appState = do
   address <- resolveSocketToAddress mainSocket
 
   let
-    appServerSettings = serverSettings conf
+    appServerSettings = serverSettings conf defaultSettings
       & setPort (configServerPort conf)
       & setOnException onWarpException
       & setBeforeMainLoop (observer $ AppServerAddressObs address)
@@ -130,9 +131,9 @@ run appState = do
         | Just (ioeGetErrorType -> et) <- fromException se, et == ResourceVanished || et == InvalidArgument = False
         | otherwise = True
 
-serverSettings :: AppConfig -> Warp.Settings
-serverSettings AppConfig{..} =
-  defaultSettings
+serverSettings :: AppConfig -> Warp.Settings -> Warp.Settings
+serverSettings AppConfig{..} settings =
+  settings
     & setHost (fromString $ toS configServerHost)
     & setServerName ("postgrest/" <> prettyVersion)
 

src/PostgREST/Logger.hs

@@ -134,6 +134,8 @@ observationMessages = \case
     pure $ "Admin server listening on " <> address
   AdminServerCrashedObs ex ->
     pure $ "FAILURE: Admin server crashed unexpectedly: " <> (showOnSingleLine '\t' . show) ex
+  AdminServerAcceptEMFILEFailure ->
+    pure "Admin server failed to accept connection due to file descriptor exhaustion (EMFILE)"
   AppStartObs ver ->
     pure $ "Starting PostgREST " <> T.decodeUtf8 ver <> "..."
   AppServerAddressObs address ->

src/PostgREST/Observation.hs

@@ -26,6 +26,7 @@ import Protolude hiding (toList)
 data Observation
   = AdminStartObs Text
   | AdminServerCrashedObs SomeException
+  | AdminServerAcceptEMFILEFailure
   | AppStartObs ByteString
   | AppServerAddressObs Text
   | ExitUnsupportedPgVersion PgVersion PgVersion

test/io/test_io.py

@@ -846,10 +846,6 @@ def test_admin_live_dependent_on_main_app(defaultenv):
         assert response.status_code == 500
 
 
-@pytest.mark.xfail(
-    reason="Admin server crashes when the first admin accept hits EMFILE",
-    strict=True,
-)
 def test_admin_server_does_not_crash_when_file_limit_is_reached(defaultenv):
     "Admin server should keep running when accept reaches the open file limit."