fix: Admin server dies silently

This is a workaround for Warp crashing with "thread blocked indefinitely in an STM transaction" when receiving eMFILE from accept.

The fix is to provide Warp with a custom accept function that handles eMFILE by logging it and keeping accepting connections.

Author: mkleczek

src/PostgREST/Admin.hs

@@ -2,48 +2,70 @@ module PostgREST.Admin
   ( runAdmin
   ) where
 
+import           Control.Monad.Extra       (whenJust)
 import qualified Data.Aeson                as JSON
+import           Foreign.C.Error           (Errno (..), eMFILE)
+import           GHC.IO.Exception
 import qualified Network.HTTP.Types.Status as HTTP
+import           Network.Socket            hiding (addrFamily)
+import           Network.Socket.ByteString
 import qualified Network.Wai               as Wai
 import qualified Network.Wai.Handler.Warp  as Warp
 
-import Control.Monad.Extra       (whenJust)
-import Network.Socket            hiding (addrFamily)
-import Network.Socket.ByteString
-
 import PostgREST.AppState    (AppState, getConfig)
 import PostgREST.Config      (AppConfig (..))
+import PostgREST.Debounce    (makeDebouncer)
 import PostgREST.MediaType   (MediaType (..), toContentType)
 import PostgREST.Metrics     (metricsToText)
 import PostgREST.Network     (resolveSocketToAddress)
 import PostgREST.Observation (Observation (..))
 
 import qualified PostgREST.AppState as AppState
 
-import qualified Network.Socket as NS
-import           Protolude
+import Protolude
 
-runAdmin :: AppState -> Maybe NS.Socket -> IO (Maybe NS.Socket) -> Warp.Settings -> IO ()
+runAdmin :: AppState -> Maybe Socket -> IO (Maybe Socket) -> Warp.Settings -> IO ()
 runAdmin appState maybeAdminSocket getSocketREST settings = do
   conf <- getConfig appState
   whenJust maybeAdminSocket $ \adminSocket -> do
     address <- resolveSocketToAddress adminSocket
+    -- log EMFILE at most once every 10s
+    logEMFILE <- makeDebouncer $ observer AdminServerAcceptEMFILEFailure *> threadDelay 10_000_000
+    let
+      safeAccept sock =
+        -- TODO use catchNoPropagate and rethrowIO once
+        -- we stop supporting GHC versions < 9.12.1
+        accept sock `catch`
+          \e ->
+            -- Keep accepting on eMFILE
+            if isEMFILE e then do
+              logEMFILE
+              -- Backoff for 1ms so that we don't enter busy accept loop
+              -- this should let the system recover fd's once the pressure is gone
+              threadDelay 1_000
+              safeAccept sock
+            else
+              throwIO e
+      adminServerSettings config addr =
+        settings
+          & Warp.setAccept safeAccept
+          & Warp.setBeforeMainLoop (observer $ AdminStartObs addr)
+          & maybe identity Warp.setPort (configAdminServerPort config)
+
     void . forkIO $ handle (onError adminSocket) $
       Warp.runSettingsSocket (adminServerSettings conf address) adminSocket adminApp
   where
     adminApp = admin appState getSocketREST
     observer = AppState.getObserver appState
-    adminServerSettings config addr=
-      settings
-        & Warp.setBeforeMainLoop (observer $ AdminStartObs addr)
-        & maybe identity Warp.setPort (configAdminServerPort config)
+
+    isEMFILE = (Just eMFILE ==) . fmap Errno . ioe_errno
 
     onError adminSock ex = do
       observer $ AdminServerCrashedObs ex
-      NS.close adminSock -- we close the socket so request doesn't hang
+      close adminSock -- we close the socket so request doesn't hang
 
 -- | PostgREST admin application
-admin :: AppState.AppState -> IO (Maybe NS.Socket) -> Wai.Application
+admin :: AppState.AppState -> IO (Maybe Socket) -> Wai.Application
 admin appState getSocketREST req respond = do
   isMainAppReachable <- getSocketREST >>= maybe (pure False) (fmap isRight . reachMainApp)
   isLoaded <- AppState.isLoaded appState

src/PostgREST/Logger.hs

@@ -134,6 +134,8 @@ observationMessages = \case
     pure $ "Admin server listening on " <> address
   AdminServerCrashedObs ex ->
     pure $ "FAILURE: Admin server crashed unexpectedly: " <> (showOnSingleLine '\t' . show) ex
+  AdminServerAcceptEMFILEFailure ->
+    pure "Admin server failed to accept connection due to file descriptor exhaustion (EMFILE)"
   AppStartObs ver ->
     pure $ "Starting PostgREST " <> T.decodeUtf8 ver <> "..."
   AppServerAddressObs address ->

src/PostgREST/Observation.hs

@@ -26,6 +26,7 @@ import Protolude hiding (toList)
 data Observation
   = AdminStartObs Text
   | AdminServerCrashedObs SomeException
+  | AdminServerAcceptEMFILEFailure
   | AppStartObs ByteString
   | AppServerAddressObs Text
   | ExitUnsupportedPgVersion PgVersion PgVersion

test/io/test_io.py

@@ -846,10 +846,6 @@ def test_admin_live_dependent_on_main_app(defaultenv):
         assert response.status_code == 500
 
 
-@pytest.mark.xfail(
-    reason="Admin server crashes when the first admin accept hits EMFILE",
-    strict=True,
-)
 def test_admin_server_does_not_crash_when_file_limit_is_reached(defaultenv):
     "Admin server should keep running when accept reaches the open file limit."