Skip to content

Commit b64316a

Browse files
kdermekderme
committed
Integrate lsm
1 parent bc7567d commit b64316a

19 files changed

Lines changed: 851 additions & 806 deletions

File tree

‎cardano-db-sync/app/cardano-db-sync.hs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,9 +67,10 @@ dbSyncMain = do
6767

6868
stateDirErrorMsg :: [Char]
6969
stateDirErrorMsg =
70-
"Error: If not using --state-dir then make sure to have ledger disabled. "
71-
<> "For more details view https://github.com/IntersectMBO/cardano-db-sync/blob"
72-
<> "/master/doc/syncing-and-rollbacks.md#ledger-state"
70+
"Error: --state-dir is required when ledger is enabled. "
71+
<> "Either provide --state-dir or set \"ledger\": \"disable\" in the config. "
72+
<> "See https://github.com/IntersectMBO/cardano-db-sync/blob"
73+
<> "/master/doc/configuration.md#ledger"
7374

7475
---------------------------------------------------------------------------------------------------
7576
-- Command Line Configurations

‎cardano-db-sync/cardano-db-sync.cabal‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,7 @@ library
108108
Cardano.DbSync.Era.Util
109109

110110
Cardano.DbSync.Ledger.Event
111+
Cardano.DbSync.Ledger.Snapshot
111112
Cardano.DbSync.Ledger.State
112113
Cardano.DbSync.Ledger.Types
113114

@@ -190,6 +191,8 @@ library
190191
, either
191192
, extra
192193
, filepath
194+
, fs-api
195+
, lsm-tree
193196
, groups
194197
, hasql
195198
, http-client
@@ -204,13 +207,16 @@ library
204207
, ouroboros-consensus
205208
, ouroboros-consensus:cardano
206209
, ouroboros-consensus:diffusion
210+
, ouroboros-consensus:lsm
211+
, resource-registry
207212
, ouroboros-consensus:protocol
208213
, ouroboros-network:api
209214
, ouroboros-network:framework
210215
, ouroboros-network:protocols
211216
, plutus-ledger-api
212217
, prometheus
213218
, psqueues
219+
, random
214220
, random-shuffle
215221
, scientific
216222
, serialise
@@ -250,7 +256,11 @@ executable cardano-db-sync
250256
-Wno-unsafe
251257
-threaded
252258
-rtsopts
253-
"-with-rtsopts=-A16m -N3 --disable-delayed-os-memory-return"
259+
260+
if arch(arm)
261+
ghc-options: "-with-rtsopts=-T -I0 -A16m -N1 --disable-delayed-os-memory-return"
262+
else
263+
ghc-options: "-with-rtsopts=-T -I0 -A16m -qg1 -qb1 -N2 --disable-delayed-os-memory-return"
254264

255265
autogen-modules: Paths_cardano_db_sync
256266
MigrationValidations
@@ -289,7 +299,11 @@ executable http-get-json-metadata
289299
-Wno-unsafe
290300
-threaded
291301
-rtsopts
292-
"-with-rtsopts=-A16m -N3 --disable-delayed-os-memory-return"
302+
303+
if arch(arm)
304+
ghc-options: "-with-rtsopts=-T -I0 -A16m -N1 --disable-delayed-os-memory-return"
305+
else
306+
ghc-options: "-with-rtsopts=-T -I0 -A16m -qg1 -qb1 -N2 --disable-delayed-os-memory-return"
293307

294308
build-depends: base
295309
, ansi-terminal
@@ -318,7 +332,11 @@ executable test-http-get-json-metadata
318332
-Wno-unsafe
319333
-threaded
320334
-rtsopts
321-
"-with-rtsopts=-A16m -N3 --disable-delayed-os-memory-return"
335+
336+
if arch(arm)
337+
ghc-options: "-with-rtsopts=-T -I0 -A16m -N1 --disable-delayed-os-memory-return"
338+
else
339+
ghc-options: "-with-rtsopts=-T -I0 -A16m -qg1 -qb1 -N2 --disable-delayed-os-memory-return"
322340

323341
build-depends: base
324342
, bytestring

‎cardano-db-sync/src/Cardano/DbSync/Api.hs‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -85,11 +85,11 @@ import Cardano.DbSync.Error
8585
import Cardano.DbSync.Ledger.Event (LedgerEvent (..))
8686
import Cardano.DbSync.Ledger.State (
8787
getHeaderHash,
88-
hashToAnnotation,
8988
listKnownSnapshots,
9089
mkHasLedgerEnv,
9190
)
92-
import Cardano.DbSync.Ledger.Types (HasLedgerEnv (..), LedgerStateFile (..), SnapshotPoint (..))
91+
import Cardano.DbSync.Ledger.Types (HasLedgerEnv (..), SnapshotPoint (..))
92+
import Ouroboros.Consensus.Storage.LedgerDB.Snapshots (DiskSnapshot (..))
9393
import Cardano.DbSync.LocalStateQuery
9494
import Cardano.DbSync.Types
9595
import Cardano.DbSync.Util
@@ -363,6 +363,7 @@ mkSyncEnv metricSetters trce dbEnv syncOptions protoInfo nw maxLovelaceSupply nw
363363
maxLovelaceSupply
364364
systemStart
365365
syncOptions
366+
(dncLedgerBackend syncNodeConfigFromFile)
366367
(Nothing, False) -> NoLedger <$> mkNoLedgerEnv trce protoInfo nw systemStart
367368
(Just _, False) -> do
368369
logWarning trce $
@@ -472,11 +473,11 @@ verifySnapshotPoint env snapPoints =
472473
catMaybes <$> mapM validLedgerFileToPoint snapPoints
473474
where
474475
validLedgerFileToPoint :: SnapshotPoint -> IO (Maybe (CardanoPoint, Bool))
475-
validLedgerFileToPoint (OnDisk lsf) = do
476-
hashes <- getSlotHash (envDbEnv env) (lsfSlotNo lsf)
477-
let valid = find (\(_, h) -> lsfHash lsf == hashToAnnotation h) hashes
478-
case valid of
479-
Just (slot, hash) | slot == lsfSlotNo lsf -> pure $ convertToDiskPoint slot hash
476+
validLedgerFileToPoint (OnDisk ds) = do
477+
let slot = SlotNo (dsNumber ds)
478+
hashes <- getSlotHash (envDbEnv env) slot
479+
case hashes of
480+
[(s, _h)] | s == slot -> pure $ convertToDiskPoint slot _h
480481
_ -> pure Nothing
481482
validLedgerFileToPoint (InMemory pnt) = do
482483
case pnt of

‎cardano-db-sync/src/Cardano/DbSync/Config.hs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ coalesceConfig pcfg ncfg adjustGenesisPath = do
8787
, dncInsertOptions = extractInsertOptions pcfg
8888
, dncIpfsGateway = endsInSlash <$> pcIpfsGateway pcfg
8989
, dncSnapshotInterval = pcSnapshotInterval pcfg
90+
, dncLedgerBackend = pcLedgerBackend pcfg
9091
}
9192

9293
mkAdjustPath :: SyncPreConfig -> (FilePath -> FilePath)

‎cardano-db-sync/src/Cardano/DbSync/Config/Types.hs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ module Cardano.DbSync.Config.Types (
3030
TxOutConfig (..),
3131
UseTxOutAddress (..),
3232
ForceTxIn (..),
33+
LedgerBackend (..),
3334
LedgerInsertConfig (..),
3435
ShelleyInsertConfig (..),
3536
RewardsConfig (..),
@@ -150,6 +151,7 @@ data SyncNodeConfig = SyncNodeConfig
150151
, dncInsertOptions :: !SyncInsertOptions
151152
, dncIpfsGateway :: [Text]
152153
, dncSnapshotInterval :: !SnapshotIntervalConfig
154+
, dncLedgerBackend :: !LedgerBackend
153155
}
154156

155157
data SyncPreConfig = SyncPreConfig
@@ -163,6 +165,7 @@ data SyncPreConfig = SyncPreConfig
163165
, pcInsertConfig :: !SyncInsertConfig
164166
, pcIpfsGateway :: ![Text]
165167
, pcSnapshotInterval :: !SnapshotIntervalConfig
168+
, pcLedgerBackend :: !LedgerBackend
166169
}
167170
deriving (Show)
168171

@@ -224,6 +227,20 @@ newtype UseTxOutAddress = UseTxOutAddress {unUseTxOutAddress :: Bool}
224227
deriving (Eq, Show)
225228
deriving newtype (ToJSON, FromJSON)
226229

230+
-- | Choose the backend for storing ledger tables (UTxO set).
231+
-- 'LedgerBackendInMemory' keeps everything in RAM (current default).
232+
-- 'LedgerBackendLSM' uses LSM trees on disk for lower memory usage.
233+
data LedgerBackend
234+
= LedgerBackendInMemory
235+
| LedgerBackendLSM (Maybe FilePath)
236+
deriving (Eq, Show)
237+
238+
instance FromJSON LedgerBackend where
239+
parseJSON = Aeson.withText "LedgerBackend" $ \case
240+
"inmemory" -> pure LedgerBackendInMemory
241+
"lsm" -> pure (LedgerBackendLSM Nothing)
242+
other -> fail $ "unexpected ledger_backend: " <> show other <> ". Expected \"inmemory\" or \"lsm\"."
243+
227244
data LedgerInsertConfig
228245
= LedgerEnable
229246
| LedgerDisable
@@ -423,6 +440,7 @@ parseGenSyncNodeConfig o =
423440
<*> o .:? "insert_options" .!= def
424441
<*> o .:? "ipfs_gateway" .!= ["https://ipfs.io/ipfs"]
425442
<*> o .:? "snapshot_interval" .!= def
443+
<*> o .:? "ledger_backend" .!= LedgerBackendInMemory
426444

427445
instance FromJSON SyncProtocol where
428446
parseJSON o =

0 commit comments

Comments
 (0)