Skip to content

Commit 507437c

Browse files
amesgengeo2a
amesgen
authored and
geo2a
committed
cardano-node: Integrate Predictable Ledger State Snapshots
1 parent dae73cc commit 507437c

10 files changed

Lines changed: 148 additions & 37 deletions

File tree

‎cardano-node/src/Cardano/Node/Configuration/LedgerDB.hs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,7 @@ noDeprecatedOptions = DeprecatedOptions []
7373

7474
data LedgerDbConfiguration =
7575
LedgerDbConfiguration
76-
NumOfDiskSnapshots
77-
SnapshotInterval
76+
SnapshotPolicyArgs
7877
QueryBatchSize
7978
LedgerDbSelectorFlag
8079
DeprecatedOptions

‎cardano-node/src/Cardano/Node/Configuration/POM.hs‎

Lines changed: 55 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ module Cardano.Node.Configuration.POM
2828
where
2929

3030
import Cardano.Crypto (RequiresNetworkMagic (..))
31+
import Cardano.Ledger.BaseTypes
3132
import Cardano.Logging.Types
3233
import Cardano.Network.ConsensusMode (ConsensusMode (..), defaultConsensusMode)
3334
import qualified Cardano.Network.Diffusion.Configuration as Cardano
@@ -46,7 +47,9 @@ import Ouroboros.Consensus.Node.Genesis (GenesisConfig, GenesisConfigF
4647
defaultGenesisConfigFlags, mkGenesisConfig)
4748
import Ouroboros.Consensus.Storage.LedgerDB.Args (QueryBatchSize (..))
4849
import Ouroboros.Consensus.Storage.LedgerDB.Snapshots (NumOfDiskSnapshots (..),
49-
SnapshotInterval (..))
50+
SnapshotDelayRange (..), SnapshotFrequency (..), SnapshotFrequencyArgs (..),
51+
SnapshotPolicyArgs (..), defaultSnapshotPolicyArgs)
52+
import Ouroboros.Consensus.Util.Args (OverrideOrDefault (..))
5053
import Ouroboros.Consensus.Storage.LedgerDB.V1.Args (FlushFrequency (..))
5154
import Ouroboros.Network.Diffusion.Configuration as Configuration
5255
import qualified Ouroboros.Network.Diffusion.Configuration as Ouroboros
@@ -484,8 +487,14 @@ instance FromJSON PartialNodeConfiguration where
484487
Nothing -> return Nothing
485488

486489
parseLedgerDbConfig v = do
487-
let snapInterval x = fmap (RequestedSnapshotInterval . secondsToDiffTime) <$> x .:? "SnapshotInterval"
488-
snapNum x = fmap RequestedNumOfDiskSnapshots <$> x .:? "NumOfDiskSnapshots"
490+
-- TODO maybe don't silently convert old format (which was in seconds)
491+
-- to new format (which is in slots), despite these being the same on
492+
-- mainnet?
493+
let snapInterval x = do
494+
si <- x .:? "SnapshotInterval"
495+
when (any (<= 0) si) $ fail $ "Non-positive SnapshotInterval: " <> show si
496+
pure $ Override . SlotNo <$> si
497+
snapNum x = fmap (Override . NumOfDiskSnapshots) <$> x .:? "NumOfDiskSnapshots"
489498

490499
mTopLevelSnapInterval <- snapInterval v
491500
mTopLevelSnapNum <- snapNum v
@@ -499,12 +508,48 @@ instance FromJSON PartialNodeConfiguration where
499508
mLedgerDB <- v .:? "LedgerDB"
500509
case mLedgerDB of
501510
Nothing -> do
502-
let si = fromMaybe DefaultSnapshotInterval mTopLevelSnapInterval
503-
sn = fromMaybe DefaultNumOfDiskSnapshots mTopLevelSnapNum
504-
return $ Just $ LedgerDbConfiguration sn si DefaultQueryBatchSize V2InMemory deprecatedOpts
511+
let si = fromMaybe UseDefault mTopLevelSnapInterval
512+
sn = fromMaybe UseDefault mTopLevelSnapNum
513+
sf = SnapshotFrequencyArgs {
514+
sfaInterval = unsafeNonZero . unSlotNo <$> si
515+
, sfaOffset = UseDefault
516+
, sfaRateLimit = UseDefault
517+
, sfaDelaySnapshotRange = UseDefault
518+
}
519+
spArgs = SnapshotPolicyArgs (SnapshotFrequency sf) sn
520+
return $ Just $ LedgerDbConfiguration spArgs DefaultQueryBatchSize V2InMemory deprecatedOpts
505521
Just ledgerDB -> flip (withObject "LedgerDB") ledgerDB $ \o -> do
506-
ldbSnapInterval <- (getLast . (Last mTopLevelSnapInterval <>) . Last <$> snapInterval o) .!= DefaultSnapshotInterval
507-
ldbSnapNum <- (getLast . (Last mTopLevelSnapNum <>) . Last <$> snapNum o) .!= DefaultNumOfDiskSnapshots
522+
-- Parse snapshot options from the "Snapshots" sub-object if present,
523+
-- otherwise fall back to the LedgerDB object for backward compatibility.
524+
let parseSnapshotOpts s = do
525+
sInterval <- (getLast . (Last mTopLevelSnapInterval <>) . Last <$> snapInterval s) .!= UseDefault
526+
sNum <- (getLast . (Last mTopLevelSnapNum <>) . Last <$> snapNum s) .!= UseDefault
527+
sOffset <- (fmap Override <$> s .:? "SlotOffset") .!= UseDefault
528+
sRateLimit <- (fmap (Override . secondsToDiffTime) <$> s .:? "RateLimit") .!= UseDefault
529+
sMinDelay <- s .:? "MinDelay"
530+
sMaxDelay <- s .:? "MaxDelay"
531+
sDelayRange <-
532+
case (sMinDelay, sMaxDelay) of
533+
(Just minDelay, Just maxDelay) ->
534+
if minDelay <= maxDelay then
535+
pure (Override (SnapshotDelayRange (secondsToDiffTime minDelay) (secondsToDiffTime maxDelay)))
536+
else fail $ "Invalid ledger snapshot delay range, MinDelay > MaxDelay: "
537+
<> show minDelay <> " > " <> show maxDelay
538+
-- use the default delay range if either min or max is unspecified
539+
_ -> pure UseDefault
540+
let sf = SnapshotFrequencyArgs {
541+
sfaInterval = unsafeNonZero . unSlotNo <$> sInterval
542+
, sfaOffset = sOffset
543+
, sfaRateLimit = sRateLimit
544+
, sfaDelaySnapshotRange = sDelayRange
545+
}
546+
pure $ SnapshotPolicyArgs (SnapshotFrequency sf) sNum
547+
548+
mSnapshotsVal <- o .:? "Snapshots"
549+
spArgs <- case mSnapshotsVal of
550+
Nothing -> parseSnapshotOpts o
551+
Just sv -> flip (withObject "Snapshots") sv parseSnapshotOpts
552+
508553
qsize <- (fmap RequestedQueryBatchSize <$> o .:? "QueryBatchSize") .!= DefaultQueryBatchSize
509554
backend <- o .:? "Backend" .!= "V2InMemory"
510555
selector <- case backend of
@@ -519,7 +564,7 @@ instance FromJSON PartialNodeConfiguration where
519564
lsmPath :: Maybe FilePath <- o .:? "LSMDatabasePath"
520565
pure $ V2LSM lsmPath
521566
_ -> fail $ "Malformed LedgerDB Backend: " <> backend
522-
pure $ Just $ LedgerDbConfiguration ldbSnapNum ldbSnapInterval qsize selector deprecatedOpts
567+
pure $ Just $ LedgerDbConfiguration spArgs qsize selector deprecatedOpts
523568

524569
parseByronProtocol v = do
525570
primary <- v .:? "ByronGenesisFile"
@@ -683,8 +728,7 @@ defaultPartialNodeConfiguration =
683728
, pncLedgerDbConfig =
684729
Last $ Just $
685730
LedgerDbConfiguration
686-
DefaultNumOfDiskSnapshots
687-
DefaultSnapshotInterval
731+
defaultSnapshotPolicyArgs
688732
DefaultQueryBatchSize
689733
V2InMemory
690734
noDeprecatedOptions

‎cardano-node/src/Cardano/Node/Run.hs‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -561,15 +561,11 @@ handleSimpleNode blockType runP tracers nc networkMagic onKernel = do
561561
Just version_ -> Map.takeWhileAntitone (<= version_)
562562

563563
LedgerDbConfiguration
564-
snapInterval
565-
numSnaps
564+
snapshotPolicyArgs
566565
queryBatchSize
567566
ldbBackend
568567
deprecatedOpts = ncLedgerDbConfig nc
569568

570-
snapshotPolicyArgs :: SnapshotPolicyArgs
571-
snapshotPolicyArgs = SnapshotPolicyArgs numSnaps snapInterval
572-
573569
--------------------------------------------------------------------------------
574570
-- SIGHUP Handlers
575571
--------------------------------------------------------------------------------

‎cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ import Ouroboros.Network.Block (MaxSlotNo (..))
6060
import Data.Aeson (Object, ToJSON, Value (Object, String), object, toJSON, (.=))
6161
import qualified Data.ByteString.Base16 as B16
6262
import Data.Int (Int64)
63+
import qualified Data.List.NonEmpty as NonEmpty
6364
import Data.SOP (All, K (..), hcmap, hcollapse)
6465
import Data.Text (Text)
6566
import qualified Data.Text as Text
@@ -1772,6 +1773,14 @@ instance ( StandardHash blk
17721773
LedgerDB.MetadataBackendMismatch ->
17731774
" Snapshot was created for a different backend. Convert it with `snapshot-converter`."
17741775
_ -> ""
1776+
forHuman (LedgerDB.SnapshotRequestDelayed _snapshotRequestTime delayBeforeSnapshotting slots) =
1777+
Text.unwords ["Scheduling to take ledger state snapshots at slots "
1778+
, showT (NonEmpty.toList slots)
1779+
, ", with a randomised delay of"
1780+
, showT delayBeforeSnapshotting
1781+
]
1782+
forHuman (LedgerDB.SnapshotRequestCompleted) = "Completed taking a ledger state snapshot"
1783+
17751784

17761785
forMachine dtals (LedgerDB.TookSnapshot snap pt enclosedTiming) =
17771786
mconcat [ "kind" .= String "TookSnapshot"
@@ -1786,11 +1795,23 @@ instance ( StandardHash blk
17861795
mconcat [ "kind" .= String "InvalidSnapshot"
17871796
, "snapshot" .= forMachine dtals snap
17881797
, "failure" .= show failure ]
1798+
forMachine _dtals (LedgerDB.SnapshotRequestDelayed snapshotRequestTime delayBeforeSnapshotting slots) =
1799+
mconcat [ "kind" .= String "TraceLedgerDBEvent.LedgerDBSnapshotEvent.SnapshotRequestDelayed"
1800+
, "requestTime" .= show snapshotRequestTime
1801+
, "delayBeforeSnapshotting " .= show delayBeforeSnapshotting
1802+
, "slots" .= show slots
1803+
]
1804+
forMachine _dtals (LedgerDB.SnapshotRequestCompleted) =
1805+
mconcat [ "kind" .= String "TraceLedgerDBEvent.LedgerDBSnapshotEvent.SnapshotRequestCompleted"
1806+
]
1807+
17891808

17901809
instance MetaTrace (LedgerDB.TraceSnapshotEvent blk) where
17911810
namespaceFor LedgerDB.TookSnapshot {} = Namespace [] ["TookSnapshot"]
17921811
namespaceFor LedgerDB.DeletedSnapshot {} = Namespace [] ["DeletedSnapshot"]
17931812
namespaceFor LedgerDB.InvalidSnapshot {} = Namespace [] ["InvalidSnapshot"]
1813+
namespaceFor LedgerDB.SnapshotRequestDelayed {} = Namespace [] ["SnapshotRequestDelayed"]
1814+
namespaceFor LedgerDB.SnapshotRequestCompleted {} = Namespace [] ["SnapshotRequestCompleted"]
17941815

17951816
severityFor (Namespace _ ["TookSnapshot"]) _ = Just Info
17961817
severityFor (Namespace _ ["DeletedSnapshot"]) _ = Just Debug
@@ -1809,6 +1830,10 @@ instance MetaTrace (LedgerDB.TraceSnapshotEvent blk) where
18091830
, " seems to be from an old node or different backend, it will"
18101831
, " be deleted"
18111832
]
1833+
documentFor (Namespace _ ["SnapshotRequestDelayed"]) = Just
1834+
"A delayed snapshot requested was issued. The snapshot will be initiated at the specified timestamp, with the specified delay and for the specified slots"
1835+
documentFor (Namespace _ ["SnapshotRequestCompleted"]) = Just
1836+
"The delayed snapshot request was completed"
18121837
documentFor _ = Nothing
18131838

18141839
allNamespaces =

‎cardano-node/test/Test/Cardano/Node/POM.hs‎

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
{-# LANGUAGE ScopedTypeVariables #-}
12
{-# LANGUAGE NamedFieldPuns #-}
23
{-# LANGUAGE OverloadedStrings #-}
34
{-# LANGUAGE PatternSynonyms #-}
@@ -22,17 +23,18 @@ import Cardano.Rpc.Server.Config (makeRpcConfig)
2223
import Ouroboros.Consensus.Node (NodeDatabasePaths (..))
2324
import Ouroboros.Consensus.Node.Genesis (disableGenesisConfig)
2425
import Ouroboros.Consensus.Storage.LedgerDB.Args
25-
import Ouroboros.Consensus.Storage.LedgerDB.Snapshots (NumOfDiskSnapshots (..),
26-
SnapshotInterval (..))
26+
import Ouroboros.Consensus.Storage.LedgerDB.Snapshots (defaultSnapshotPolicyArgs)
2727
import Ouroboros.Network.Block (SlotNo (..))
2828
import Ouroboros.Network.PeerSelection.PeerSharing (PeerSharing (..))
2929
import Ouroboros.Network.TxSubmission.Inbound.V2.Types
3030

3131
import Data.Bifunctor (first)
32+
import qualified Data.ByteString.Lazy as LBS
3233
import Data.Monoid (Last (..))
3334
import Data.String
3435
import Data.Text (Text)
3536

37+
import Data.Aeson (eitherDecode)
3638
import Hedgehog (Property, discover, withTests, (===))
3739
import qualified Hedgehog
3840
import Hedgehog.Internal.Property (evalEither, failWith)
@@ -284,12 +286,48 @@ eExpectedConfig = do
284286
, ncConsensusMode = PraosMode
285287
, ncGenesisConfig = disableGenesisConfig
286288
, ncResponderCoreAffinityPolicy = NoResponderCoreAffinity
287-
, ncLedgerDbConfig = LedgerDbConfiguration DefaultNumOfDiskSnapshots DefaultSnapshotInterval DefaultQueryBatchSize V2InMemory noDeprecatedOptions
289+
, ncLedgerDbConfig = LedgerDbConfiguration defaultSnapshotPolicyArgs DefaultQueryBatchSize V2InMemory noDeprecatedOptions
288290
, ncRpcConfig
289291
, ncTxSubmissionLogicVersion = TxSubmissionLogicV1
290292
, ncTxSubmissionInitDelay = defaultTxSubmissionInitDelay
291293
}
292294

295+
-- | Test that the legacy flat LedgerDB snapshot config format (options directly
296+
-- under LedgerDB) parses identically to the new nested Snapshots format.
297+
--
298+
-- TODO: this test could be removed once the old format is deprecated.
299+
prop_legacySnapshotFormat_POM :: Property
300+
prop_legacySnapshotFormat_POM =
301+
withTests 1 . Hedgehog.property $ do
302+
let legacyJson = "{ " <> dummyRequiredValues <> ", "
303+
<> "\"LedgerDB\": {"
304+
<> " \"Backend\": \"V2InMemory\","
305+
<> " \"SnapshotInterval\": 4320,"
306+
<> " \"NumOfDiskSnapshots\": 2"
307+
<> "} }"
308+
newJson = "{ " <> dummyRequiredValues <> ", "
309+
<> "\"LedgerDB\": {"
310+
<> " \"Backend\": \"V2InMemory\","
311+
<> " \"Snapshots\": {"
312+
<> " \"SnapshotInterval\": 4320,"
313+
<> " \"NumOfDiskSnapshots\": 2"
314+
<> " }"
315+
<> "} }"
316+
legacyConfig :: PartialNodeConfiguration <- evalEither $ eitherDecode legacyJson
317+
newConfig :: PartialNodeConfiguration <- evalEither $ eitherDecode newJson
318+
pncLedgerDbConfig legacyConfig === pncLedgerDbConfig newConfig
319+
where
320+
dummyRequiredValues :: LBS.ByteString
321+
dummyRequiredValues = mconcat
322+
[ "\"ByronGenesisFile\": \"x\""
323+
, ", \"ShelleyGenesisFile\": \"x\""
324+
, ", \"AlonzoGenesisFile\": \"x\""
325+
, ", \"ConwayGenesisFile\": \"x\""
326+
, ", \"LastKnownBlockVersion-Major\": 0"
327+
, ", \"LastKnownBlockVersion-Minor\": 0"
328+
, ", \"LastKnownBlockVersion-Alt\": 0"
329+
]
330+
293331
-- -----------------------------------------------------------------------------
294332

295333
tests :: IO Bool

‎configuration/cardano/mainnet-config-legacy.json‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,11 @@
1313
"LastKnownBlockVersion-Minor": 0,
1414
"LedgerDB": {
1515
"Backend": "V2InMemory",
16-
"NumOfDiskSnapshots": 2,
1716
"QueryBatchSize": 100000,
18-
"SnapshotInterval": 4320
17+
"Snapshots": {
18+
"NumOfDiskSnapshots": 2,
19+
"SnapshotInterval": 4320
20+
}
1921
},
2022
"MaxKnownMajorProtocolVersion": 2,
2123
"MinNodeVersion": "10.7.0",

‎configuration/cardano/mainnet-config.json‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,11 @@
1313
"LastKnownBlockVersion-Minor": 0,
1414
"LedgerDB": {
1515
"Backend": "V2InMemory",
16-
"NumOfDiskSnapshots": 2,
1716
"QueryBatchSize": 100000,
18-
"SnapshotInterval": 4320
17+
"Snapshots": {
18+
"NumOfDiskSnapshots": 2,
19+
"SnapshotInterval": 4320
20+
}
1921
},
2022
"MaxKnownMajorProtocolVersion": 2,
2123
"MinNodeVersion": "10.7.0",

‎configuration/cardano/mainnet-config.yaml‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -80,19 +80,20 @@ ConsensusMode: PraosMode
8080
# Additional configuration options can be found at:
8181
# https://ouroboros-consensus.cardano.intersectmbo.org/docs/for-developers/utxo-hd/migrating
8282
LedgerDB:
83-
# The time interval between snapshots, in seconds.
84-
SnapshotInterval: 4320
85-
86-
# The number of disk snapshots to keep.
87-
NumOfDiskSnapshots: 2
83+
# The backend can either be in memory with `V2InMemory` or on disk with
84+
# `V1LMDB`.
85+
Backend: V2InMemory
8886

8987
# When querying the store for a big range of UTxOs (such as with
9088
# QueryUTxOByAddress), the store will be read in batches of this size.
9189
QueryBatchSize: 100000
9290

93-
# The backend can either be in memory with `V2InMemory` or on disk with
94-
# `V1LMDB`.
95-
Backend: V2InMemory
91+
Snapshots:
92+
# The time interval between snapshots, in seconds.
93+
SnapshotInterval: 4320
94+
95+
# The number of disk snapshots to keep.
96+
NumOfDiskSnapshots: 2
9697

9798
##### Version Information #####
9899

‎configuration/cardano/testnet-template-config-legacy.json‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,11 @@
1212
"LastKnownBlockVersion-Minor": 1,
1313
"LedgerDB": {
1414
"Backend": "V2InMemory",
15-
"NumOfDiskSnapshots": 2,
1615
"QueryBatchSize": 100000,
17-
"SnapshotInterval": 216
16+
"Snapshots": {
17+
"NumOfDiskSnapshots": 2,
18+
"SnapshotInterval": 216
19+
}
1820
},
1921
"MaxConcurrencyDeadline": 4,
2022
"MaxKnownMajorProtocolVersion": 2,

‎configuration/cardano/testnet-template-config.json‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,11 @@
1313
"LastKnownBlockVersion-Minor": 1,
1414
"LedgerDB": {
1515
"Backend": "V2InMemory",
16-
"NumOfDiskSnapshots": 2,
1716
"QueryBatchSize": 100000,
18-
"SnapshotInterval": 216
17+
"Snapshots": {
18+
"NumOfDiskSnapshots": 2,
19+
"SnapshotInterval": 216
20+
}
1921
},
2022
"MaxConcurrencyDeadline": 4,
2123
"MaxKnownMajorProtocolVersion": 2,

0 commit comments

Comments
 (0)