Merge pull request #2104 from IntersectMBO/kderme/integrate-node-10.7-lsm

Integrate lsm

Author: kderme

cabal.project

@@ -39,11 +39,6 @@ package cardano-db-tool
 package cardano-smash-server
   ghc-options: -Wall -Werror -Wredundant-constraints -Wincomplete-uni-patterns -Wincomplete-record-updates -Wpartial-fields -Wunused-imports -Wunused-packages
 
-package blockio
-  -- Use serial block IO to avoid requiring liburing (not available in devx CI).
-  -- TODO: revert when CI provides liburing or switch to io_uring for better LSM perf.
-  flags: +serialblockio
-
 package cardano-node
    -- We are using cardano-node as a library and we never use the systemd scribe, so there
    -- is no benefit to linking against it

cardano-chain-gen/test/Test/Cardano/Db/Mock/Config.hs

@@ -281,8 +281,15 @@ withConfig staticDir mutableDir cmdLineArgs config action = do
 {-# ANN withConfig ("HLint: ignore Redundant pure" :: String) #-}
 
 mkSyncNodeConfig :: FilePath -> CommandLineArgs -> IO SyncNodeConfig
-mkSyncNodeConfig configFilePath cmdLineArgs =
-  readSyncNodeConfig $ mkConfigFile configDir configFilename
+mkSyncNodeConfig configFilePath cmdLineArgs = do
+  cfg <- readSyncNodeConfig $ mkConfigFile configDir configFilename
+  -- Allow env-var override of the ledger backend so CI can run the
+  -- full suite against both "inmemory" and "lsm" without per-test config changes.
+  mBackend <- lookupEnv "DB_SYNC_TEST_LEDGER_BACKEND"
+  pure $ case mBackend of
+    Just "lsm" -> cfg {dncLedgerBackend = LedgerBackendLSM Nothing}
+    Just "inmemory" -> cfg {dncLedgerBackend = LedgerBackendInMemory}
+    _ -> cfg
   where
     configFilename = claConfigFilename cmdLineArgs
     configDir = mkConfigDir configFilePath

cardano-db-sync/app/cardano-db-sync.hs

@@ -67,9 +67,10 @@ dbSyncMain = do
 
     stateDirErrorMsg :: [Char]
     stateDirErrorMsg =
-      "Error: If not using --state-dir then make sure to have ledger disabled. "
-        <> "For more details view https://github.com/IntersectMBO/cardano-db-sync/blob"
-        <> "/master/doc/syncing-and-rollbacks.md#ledger-state"
+      "Error: --state-dir is required when ledger is enabled. "
+        <> "Either provide --state-dir or set \"ledger\": \"disable\" in the config. "
+        <> "See https://github.com/IntersectMBO/cardano-db-sync/blob"
+        <> "/master/doc/configuration.md#ledger"
 
 ---------------------------------------------------------------------------------------------------
 -- Command Line Configurations

cardano-db-sync/cardano-db-sync.cabal

@@ -108,6 +108,7 @@ library
                         Cardano.DbSync.Era.Util
 
                         Cardano.DbSync.Ledger.Event
+                        Cardano.DbSync.Ledger.Snapshot
                         Cardano.DbSync.Ledger.State
                         Cardano.DbSync.Ledger.Types
 
@@ -190,6 +191,8 @@ library
                       , either
                       , extra
                       , filepath
+                      , fs-api
+                      , lsm-tree
                       , groups
                       , hasql
                       , http-client
@@ -204,13 +207,16 @@ library
                       , ouroboros-consensus
                       , ouroboros-consensus:cardano
                       , ouroboros-consensus:diffusion
+                      , ouroboros-consensus:lsm
+                      , resource-registry
                       , ouroboros-consensus:protocol
                       , ouroboros-network:api
                       , ouroboros-network:framework
                       , ouroboros-network:protocols
                       , plutus-ledger-api
                       , prometheus
                       , psqueues
+                      , random
                       , random-shuffle
                       , scientific
                       , serialise
@@ -250,7 +256,11 @@ executable cardano-db-sync
                         -Wno-unsafe
                         -threaded
                         -rtsopts
-                        "-with-rtsopts=-A16m -N3 --disable-delayed-os-memory-return"
+
+  if arch(arm)
+    ghc-options:        "-with-rtsopts=-T -I0 -A16m -N1 --disable-delayed-os-memory-return"
+  else
+    ghc-options:        "-with-rtsopts=-T -I0 -A16m -qg1 -qb1 -N2 --disable-delayed-os-memory-return"
 
   autogen-modules:      Paths_cardano_db_sync
                         MigrationValidations
@@ -289,7 +299,11 @@ executable http-get-json-metadata
                         -Wno-unsafe
                         -threaded
                         -rtsopts
-                        "-with-rtsopts=-A16m -N3 --disable-delayed-os-memory-return"
+
+  if arch(arm)
+    ghc-options:        "-with-rtsopts=-T -I0 -A16m -N1 --disable-delayed-os-memory-return"
+  else
+    ghc-options:        "-with-rtsopts=-T -I0 -A16m -qg1 -qb1 -N2 --disable-delayed-os-memory-return"
 
   build-depends:        base
                       , ansi-terminal
@@ -318,7 +332,11 @@ executable test-http-get-json-metadata
                         -Wno-unsafe
                         -threaded
                         -rtsopts
-                        "-with-rtsopts=-A16m -N3 --disable-delayed-os-memory-return"
+                        
+  if arch(arm)
+    ghc-options:        "-with-rtsopts=-T -I0 -A16m -N1 --disable-delayed-os-memory-return"
+  else
+    ghc-options:        "-with-rtsopts=-T -I0 -A16m -qg1 -qb1 -N2 --disable-delayed-os-memory-return"
 
   build-depends:        base
                       , bytestring

cardano-db-sync/src/Cardano/DbSync.hs

@@ -45,7 +45,7 @@ import Cardano.Slotting.Slot (EpochNo (..))
 
 import qualified Cardano.Db as DB
 import Cardano.DbSync.Api
-import Cardano.DbSync.Api.Types (InsertOptions (..), RunMigration, SyncEnv (..), SyncOptions (..), envLedgerEnv)
+import Cardano.DbSync.Api.Types (InsertOptions (..), LedgerEnv (..), RunMigration, SyncEnv (..), SyncOptions (..), envLedgerEnv)
 import Cardano.DbSync.Config (configureLogging)
 import Cardano.DbSync.Config.Cardano
 import Cardano.DbSync.Config.Types
@@ -54,6 +54,7 @@ import Cardano.DbSync.DbEvent
 import Cardano.DbSync.Era
 import Cardano.DbSync.Error
 import Cardano.DbSync.Ledger.State
+import Cardano.DbSync.Ledger.Types (HasLedgerEnv (..))
 import Cardano.DbSync.OffChain (runFetchOffChainPoolThread, runFetchOffChainVoteThread)
 import Cardano.DbSync.Rollback (handlePostRollbackSnapshots, unsafeRollback)
 import Cardano.DbSync.Sync (runSyncNodeClient)
@@ -248,6 +249,10 @@ runSyncNode metricsSetters trce iomgr dbConnSetting runNearTipMigrationFnc syncN
 
           -- communication channel between datalayer thread and chainsync-client thread
           threadChannels <- liftIO newThreadChannels
+          -- 'finally' on the worker pool ensures the LSM session (and any other
+          -- backend resources) are closed even when db-sync is cancelled or
+          -- crashes — important for tests that restart db-sync in the same
+          -- process and need the OS file lock to be released.
           liftIO $
             mapConcurrently_
               id
@@ -257,6 +262,7 @@ runSyncNode metricsSetters trce iomgr dbConnSetting runNearTipMigrationFnc syncN
               , runFetchOffChainVoteThread syncEnv
               , runLedgerStateWriteThread (getTrace syncEnv) (envLedgerEnv syncEnv)
               ]
+              `finally` closeLedgerEnv syncEnv
     )
   where
     useShelleyInit :: SyncNodeConfig -> Bool
@@ -353,3 +359,14 @@ txOutConfigToTableType config = case config of
   TxOutConsumed _ (UseTxOutAddress flag) -> if flag then DB.TxOutVariantAddress else DB.TxOutVariantCore
   TxOutConsumedPrune _ (UseTxOutAddress flag) -> if flag then DB.TxOutVariantAddress else DB.TxOutVariantCore
   TxOutConsumedBootstrap _ (UseTxOutAddress flag) -> if flag then DB.TxOutVariantAddress else DB.TxOutVariantCore
+
+-- | Release backend resources held by the ledger environment.
+-- Currently this closes the LSM session (no-op for InMemory and NoLedger).
+closeLedgerEnv :: SyncEnv -> IO ()
+closeLedgerEnv syncEnv = case envLedgerEnv syncEnv of
+  HasLedger le -> do
+    let trce = leTrace le
+    logInfo trce "closeLedgerEnv: closing LSM session..."
+    leClose le
+    logInfo trce "closeLedgerEnv: closed."
+  NoLedger _ -> pure ()

cardano-db-sync/src/Cardano/DbSync/Api.hs

@@ -85,14 +85,14 @@ import Cardano.DbSync.Error
 import Cardano.DbSync.Ledger.Event (LedgerEvent (..))
 import Cardano.DbSync.Ledger.State (
   getHeaderHash,
-  hashToAnnotation,
   listKnownSnapshots,
   mkHasLedgerEnv,
  )
-import Cardano.DbSync.Ledger.Types (HasLedgerEnv (..), LedgerStateFile (..), SnapshotPoint (..))
+import Cardano.DbSync.Ledger.Types (HasLedgerEnv (..), SnapshotPoint (..))
 import Cardano.DbSync.LocalStateQuery
 import Cardano.DbSync.Types
 import Cardano.DbSync.Util
+import Ouroboros.Consensus.Storage.LedgerDB.Snapshots (DiskSnapshot (..))
 
 setConsistentLevel :: SyncEnv -> ConsistentLevel -> IO ()
 setConsistentLevel env cst = do
@@ -341,7 +341,15 @@ mkSyncEnv metricSetters trce dbEnv syncOptions protoInfo nw maxLovelaceSupply nw
       else pure useNoCache
   consistentLevelVar <- newTVarIO Unchecked
   indexesVar <- newTVarIO $ enpForceIndexes syncNP
-  bts <- getBootstrapInProgress trce (isTxOutConsumedBootstrap' syncNodeConfigFromFile) dbEnv
+  let bootstrapFlag = isTxOutConsumedBootstrap' syncNodeConfigFromFile
+  case (bootstrapFlag, dncLedgerBackend syncNodeConfigFromFile) of
+    (True, LedgerBackendLSM _) ->
+      DB.logAndThrowIO trce $
+        "bootstrap-tx-out is not supported with ledger_backend=lsm. "
+          <> "The bootstrap path reads the full UTxO from the in-memory ledger state, "
+          <> "which is empty under LSM. Use ledger_backend=inmemory or disable bootstrap."
+    _ -> pure ()
+  bts <- getBootstrapInProgress trce bootstrapFlag dbEnv
   bootstrapVar <- newTVarIO bts
   -- Offline Pool + Anchor queues
   opwq <- newTBQueueIO 1000
@@ -363,6 +371,7 @@ mkSyncEnv metricSetters trce dbEnv syncOptions protoInfo nw maxLovelaceSupply nw
             maxLovelaceSupply
             systemStart
             syncOptions
+            (dncLedgerBackend syncNodeConfigFromFile)
       (Nothing, False) -> NoLedger <$> mkNoLedgerEnv trce protoInfo nw systemStart
       (Just _, False) -> do
         logWarning trce $
@@ -472,11 +481,11 @@ verifySnapshotPoint env snapPoints =
   catMaybes <$> mapM validLedgerFileToPoint snapPoints
   where
     validLedgerFileToPoint :: SnapshotPoint -> IO (Maybe (CardanoPoint, Bool))
-    validLedgerFileToPoint (OnDisk lsf) = do
-      hashes <- getSlotHash (envDbEnv env) (lsfSlotNo lsf)
-      let valid = find (\(_, h) -> lsfHash lsf == hashToAnnotation h) hashes
-      case valid of
-        Just (slot, hash) | slot == lsfSlotNo lsf -> pure $ convertToDiskPoint slot hash
+    validLedgerFileToPoint (OnDisk ds) = do
+      let slot = SlotNo (dsNumber ds)
+      hashes <- getSlotHash (envDbEnv env) slot
+      case hashes of
+        [(s, _h)] | s == slot -> pure $ convertToDiskPoint slot _h
         _ -> pure Nothing
     validLedgerFileToPoint (InMemory pnt) = do
       case pnt of

cardano-db-sync/src/Cardano/DbSync/Config.hs

@@ -87,6 +87,7 @@ coalesceConfig pcfg ncfg adjustGenesisPath = do
       , dncInsertOptions = extractInsertOptions pcfg
       , dncIpfsGateway = endsInSlash <$> pcIpfsGateway pcfg
       , dncSnapshotInterval = pcSnapshotInterval pcfg
+      , dncLedgerBackend = pcLedgerBackend pcfg
       }
 
 mkAdjustPath :: SyncPreConfig -> (FilePath -> FilePath)

cardano-db-sync/src/Cardano/DbSync/Config/Types.hs

@@ -30,6 +30,7 @@ module Cardano.DbSync.Config.Types (
   TxOutConfig (..),
   UseTxOutAddress (..),
   ForceTxIn (..),
+  LedgerBackend (..),
   LedgerInsertConfig (..),
   ShelleyInsertConfig (..),
   RewardsConfig (..),
@@ -150,6 +151,7 @@ data SyncNodeConfig = SyncNodeConfig
   , dncInsertOptions :: !SyncInsertOptions
   , dncIpfsGateway :: [Text]
   , dncSnapshotInterval :: !SnapshotIntervalConfig
+  , dncLedgerBackend :: !LedgerBackend
   }
 
 data SyncPreConfig = SyncPreConfig
@@ -163,6 +165,7 @@ data SyncPreConfig = SyncPreConfig
   , pcInsertConfig :: !SyncInsertConfig
   , pcIpfsGateway :: ![Text]
   , pcSnapshotInterval :: !SnapshotIntervalConfig
+  , pcLedgerBackend :: !LedgerBackend
   }
   deriving (Show)
 
@@ -224,6 +227,20 @@ newtype UseTxOutAddress = UseTxOutAddress {unUseTxOutAddress :: Bool}
   deriving (Eq, Show)
   deriving newtype (ToJSON, FromJSON)
 
+-- | Choose the backend for storing ledger tables (UTxO set).
+-- 'LedgerBackendInMemory' keeps everything in RAM (current default).
+-- 'LedgerBackendLSM' uses LSM trees on disk for lower memory usage.
+data LedgerBackend
+  = LedgerBackendInMemory
+  | LedgerBackendLSM (Maybe FilePath)
+  deriving (Eq, Show)
+
+instance FromJSON LedgerBackend where
+  parseJSON = Aeson.withText "LedgerBackend" $ \case
+    "inmemory" -> pure LedgerBackendInMemory
+    "lsm" -> pure (LedgerBackendLSM Nothing)
+    other -> fail $ "unexpected ledger_backend: " <> show other <> ". Expected \"inmemory\" or \"lsm\"."
+
 data LedgerInsertConfig
   = LedgerEnable
   | LedgerDisable
@@ -423,6 +440,7 @@ parseGenSyncNodeConfig o =
     <*> o .:? "insert_options" .!= def
     <*> o .:? "ipfs_gateway" .!= ["https://ipfs.io/ipfs"]
     <*> o .:? "snapshot_interval" .!= def
+    <*> o .:? "ledger_backend" .!= LedgerBackendInMemory
 
 instance FromJSON SyncProtocol where
   parseJSON o =