Non-native snapshots hook

bench/plutus-scripts-bench/plutus-scripts-bench.cabal

@@ -82,7 +82,7 @@ library
   -- IOG dependencies
   --------------------------
   build-depends:
-    , cardano-api             ^>=10.19
+    , cardano-api             ^>=10.20
     , plutus-ledger-api       ^>=1.53
     , plutus-tx               ^>=1.53
     , plutus-tx-plugin        ^>=1.53

bench/tx-generator/tx-generator.cabal

@@ -111,7 +111,7 @@ library
                       , attoparsec-aeson
                       , base16-bytestring
                       , bytestring
-                      , cardano-api ^>= 10.19
+                      , cardano-api ^>= 10.20
                       , cardano-binary
                       , cardano-cli ^>= 10.13
                       , cardano-crypto-class

cabal.project

@@ -13,7 +13,7 @@ repository cardano-haskell-packages
 -- See CONTRIBUTING for information about these, including some Nix commands
 -- you need to run if you change them
 index-state:
-  , hackage.haskell.org 2025-10-17T00:26:22Z
+  , hackage.haskell.org 2025-10-23T13:39:53Z
   , cardano-haskell-packages 2025-11-07T15:42:47Z
 
 packages:
@@ -61,14 +61,79 @@ package plutus-scripts-bench
 allow-newer:
   , katip:Win32
 
-if impl (ghc >= 9.12)
-  allow-newer:
-    -- https://github.com/kapralVV/Unique/issues/11
-    , Unique:hashable
-
-    -- https://github.com/Gabriella439/Haskell-Pipes-Safe-Library/pull/70
-    , pipes-safe:base
-
 -- IMPORTANT
 -- Do NOT add more source-repository-package stanzas here unless they are strictly
 -- temporary! Please read the section in CONTRIBUTING about updating dependencies.
+source-repository-package
+  type: git
+  location: https://github.com/intersectmbo/cardano-cli.git
+  tag: 6691afc0a51e16b98236f85fda4e8d6b49174370
+  --sha256: sha256-TDHJeC+chGZJsIgRUpw2hZuLErktkxIfFbc+WHFHn0k=
+  subdir: cardano-cli
+
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/cardano-api
+  tag: 58941c7b1e71be38a3ba63a1b815ac386574287e
+  --sha256: sha256-D/+yB+jclri/K5XPrM+ygBrN5bUizamX+zNeE5AqWpI=
+  subdir:
+    cardano-api
+
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/ouroboros-consensus
+  tag: 163895614992007a09675fd15980578ecd6e1e91
+  --sha256: sha256-1rbaFlFzSiMNe6vzUMuIsO4COF16adUZYak/jfyw4IU=
+  subdir:
+    ouroboros-consensus
+    ouroboros-consensus-diffusion
+    ouroboros-consensus-cardano
+
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/cardano-ledger
+  tag: fb09078fa55015c881303a2ddb609c024cec258f
+  --sha256: sha256-9Y9CRiyMn0AWD+C4aNVMaJgrj3FDAYfCX4VrLvtoMaI=
+  subdir:
+    eras/allegra/impl
+    eras/alonzo/impl
+    eras/alonzo/test-suite
+    eras/babbage/impl
+    eras/conway/impl
+    eras/dijkstra/impl
+    eras/mary/impl
+    eras/shelley/impl
+    eras/shelley/test-suite
+    eras/shelley-ma/test-suite
+    libs/cardano-ledger-api
+    libs/cardano-ledger-core
+    libs/cardano-ledger-binary
+    libs/cardano-protocol-tpraos
+    libs/non-integral
+    libs/small-steps
+    libs/cardano-data
+    libs/set-algebra
+    libs/vector-map
+    eras/byron/chain/executable-spec
+    eras/byron/ledger/executable-spec
+    eras/byron/ledger/impl
+    eras/byron/crypto
+
+-- Backported version of https://github.com/IntersectMBO/ouroboros-network/pull/5161
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/ouroboros-network
+  tag: dfcb1f9c578ec8cd5114fea9696e7be6c9001323
+  --sha256: sha256-omXPdi/T/f3gq9rOH20zX+x3WvaxlB704g07RF/13Nk=
+  subdir:
+    ouroboros-network-api
+    ouroboros-network
+
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/lsm-tree
+  tag: 3c51ea5ac6400f32a7bdf17260325d4d0a360a98
+  --sha256: sha256-44SHqtAciPhoI6ysLt8RdNvlVvN91GWHMnr68vOU2gQ=
+  subdir:
+    lsm-tree
+    blockio

cardano-node-chairman/cardano-node-chairman.cabal

@@ -44,7 +44,7 @@ executable cardano-node-chairman
   build-depends:        cardano-api
                       , cardano-crypto-class
                       , cardano-git-rev ^>= 0.2.2
-                      , cardano-ledger-core ^>= 1.18
+                      , cardano-ledger-core ^>= 1.19
                       , cardano-node ^>= 10.6
                       , cardano-prelude
                       , containers

cardano-node/app/cardano-node.hs

@@ -8,8 +8,9 @@ import           Cardano.Git.Rev (gitRev)
 import           Cardano.Node.Configuration.POM (PartialNodeConfiguration(..))
 import           Cardano.Node.Handlers.TopLevel
 import           Cardano.Node.Parsers (nodeCLIParser, parserHelpHeader, parserHelpOptions,
-                   renderHelpDoc)
+                   renderHelpDoc, parseSnapshotsCmd)
 import           Cardano.Node.Run (runNode)
+import           Cardano.Snapshots.Run (canonicalizeSnapshots, NodeDatabasePaths)
 import           Cardano.Node.Tracing.Documentation (TraceDocumentationCmd (..),
                    parseTraceDocumentationCmd, runTraceDocumentationCmd)
 
@@ -37,6 +38,7 @@ main = do
         warnIfSet args pncMaybeMempoolCapacityOverride "mempool-capacity-override" "MempoolCapacityBytesOverride"
         runNode args
       TraceDocumentation tdc -> runTraceDocumentationCmd tdc
+      CanonicalizeSnapshotsCmd cfg db -> canonicalizeSnapshots cfg db
       VersionCmd  -> runVersionCommand
 
     where
@@ -54,29 +56,32 @@ main = do
 
       opts :: Opt.ParserInfo Command
       opts =
-        Opt.info (fmap RunCmd nodeCLIParser
+        let pp = fmap RunCmd nodeCLIParser
                   <|> fmap TraceDocumentation parseTraceDocumentationCmd
                   <|> parseVersionCmd
-                  <**> helperBrief "help" "Show this help text" nodeCliHelpMain)
+                  <|> fmap (uncurry CanonicalizeSnapshotsCmd) parseSnapshotsCmd
+        in Opt.info (pp
+                  <**> helperBrief "help" "Show this help text" (nodeCliHelpMain pp))
 
           ( Opt.fullDesc <>
-            Opt.progDesc "Start node of the Cardano blockchain."
+            Opt.progDesc "The Cardano blockchain node"
           )
 
       helperBrief :: String -> String -> String -> Parser (a -> a)
       helperBrief l d helpText = Opt.abortOption (Opt.InfoMsg helpText) $ mconcat
         [ Opt.long l
         , Opt.help d ]
 
-      nodeCliHelpMain :: String
-      nodeCliHelpMain = renderHelpDoc 80 $
-        parserHelpHeader "cardano-node" nodeCLIParser
+      nodeCliHelpMain :: Parser a -> String
+      nodeCliHelpMain pp = renderHelpDoc 80 $
+        parserHelpHeader "cardano-node" pp
         <$$> ""
         <$$> parserHelpOptions nodeCLIParser
 
 
 data Command = RunCmd PartialNodeConfiguration
              | TraceDocumentation TraceDocumentationCmd
+             | CanonicalizeSnapshotsCmd (Maybe FilePath) (Maybe NodeDatabasePaths)
              | VersionCmd
 
 -- Yes! A --version flag or version command. Either guess is right!

cardano-node/cardano-node.cabal

@@ -132,6 +132,7 @@ library
                         Cardano.Tracing.Shutdown
                         Cardano.Tracing.Startup
                         Cardano.Tracing.Tracers
+                        Cardano.Snapshots.Run
 
   other-modules:        Paths_cardano_node
   autogen-modules:      Paths_cardano_node
@@ -141,7 +142,7 @@ library
                       , async
                       , base16-bytestring
                       , bytestring
-                      , cardano-api ^>= 10.19
+                      , cardano-api ^>= 10.20
                       , cardano-crypto-class
                       , cardano-crypto-wrapper
                       , cardano-git-rev ^>=0.2.2
@@ -188,8 +189,8 @@ library
                       , network-mux >= 0.8
                       , nothunks
                       , optparse-applicative-fork >= 0.18.1
-                      , ouroboros-consensus ^>= 0.28
-                      , ouroboros-consensus-cardano ^>= 0.26
+                      , ouroboros-consensus:{ouroboros-consensus, ouroboros-consensus-lmdb, ouroboros-consensus-lsm} ^>= 0.28
+                      , ouroboros-consensus-cardano:{ouroboros-consensus-cardano, snapshot-conversion} ^>= 0.26
                       , ouroboros-consensus-diffusion ^>= 0.24
                       , ouroboros-consensus-protocol
                       , ouroboros-network-api ^>= 0.16
@@ -198,6 +199,7 @@ library
                       , ouroboros-network-protocols ^>= 0.15
                       , prettyprinter
                       , prettyprinter-ansi-terminal
+                      , process
                       , psqueues
                       , random
                       , resource-registry

cardano-node/src/Cardano/Node/Configuration/LedgerDB.hs

@@ -1,28 +1,35 @@
 {-# LANGUAGE DataKinds #-}
 {-# LANGUAGE DerivingStrategies #-}
+{-# LANGUAGE FlexibleContexts #-}
 {-# LANGUAGE GeneralisedNewtypeDeriving #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+{-# LANGUAGE TypeApplications #-}
 
 {-# OPTIONS_GHC -Wno-orphans #-}
 
 module Cardano.Node.Configuration.LedgerDB (
-    DeprecatedOptions (..)
-  , LedgerDbConfiguration (..)
-  , LedgerDbSelectorFlag(..)
-  , Gigabytes
-  , noDeprecatedOptions
-  , selectorToArgs
-  ) where
+    DeprecatedOptions (..),
+    LedgerDbConfiguration (..),
+    LedgerDbSelectorFlag (..),
+    Gigabytes,
+    noDeprecatedOptions,
+    selectorToArgs,
+) where
 
+import           Ouroboros.Consensus.Ledger.SupportsProtocol
+import           Ouroboros.Consensus.Storage.LedgerDB.API
 import           Ouroboros.Consensus.Storage.LedgerDB.Args
 import           Ouroboros.Consensus.Storage.LedgerDB.Snapshots
 import qualified Ouroboros.Consensus.Storage.LedgerDB.V1.Args as V1
-import           Ouroboros.Consensus.Storage.LedgerDB.V1.BackingStore.Impl.LMDB (LMDBLimits (..))
-import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.Args as V2
-import           Ouroboros.Consensus.Util.Args
+import qualified Ouroboros.Consensus.Storage.LedgerDB.V1.BackingStore.Impl.LMDB as LMDB
+import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.InMemory as InMemory
+import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.LSM as LSM
 
 import qualified Data.Aeson.Types as Aeson (FromJSON)
 import           Data.Maybe (fromMaybe)
-import           Data.SOP.Dict
+import           Data.Proxy
+import           System.FilePath
+import           System.Random (StdGen)
 
 -- | Choose the LedgerDB Backend
 --
@@ -34,21 +41,25 @@ import           Data.SOP.Dict
 --
 -- - 'V1LMDB': uses less memory but is somewhat slower.
 --
--- - 'V1InMemory': Not intended for production. It is an in-memory reproduction
---   of the LMDB implementation.
+-- - 'V2LSM': Uses the LSM backend.
 data LedgerDbSelectorFlag =
     V1LMDB
       V1.FlushFrequency
       -- ^ The frequency at which changes are flushed to the disk.
       (Maybe FilePath)
-      -- ^ Path for the live tables.
+      -- ^ Path for the live tables. If not provided the default will be used
+      -- (@<fast-storage>/lmdb@).
       (Maybe Gigabytes)
       -- ^ A map size can be specified, this is the maximum disk space the LMDB
       -- database can fill. If not provided, the default of 16GB will be used.
       (Maybe Int)
       -- ^ An override to the max number of readers.
-  | V1InMemory V1.FlushFrequency
   | V2InMemory
+  | V2LSM
+      (Maybe FilePath)
+      -- ^ Maybe a custom path to the LSM database. If not provided the default
+      -- will be used (@<fast-storage>/lsm@).
+
   deriving (Eq, Show)
 
 -- | Some options that existed in the TopLevel were now moved to a
@@ -118,24 +129,23 @@ toBytes (Gigabytes x) = x * 1024 * 1024 * 1024
 -- * The @lmdb-simple@ and @haskell-lmdb@ forked repositories.
 -- * The official LMDB API documentation at
 --    <http://www.lmdb.tech/doc/group__mdb.html>.
-defaultLMDBLimits :: LMDBLimits
-defaultLMDBLimits = LMDBLimits {
-    lmdbMapSize = 16 * 1024 * 1024 * 1024
-  , lmdbMaxDatabases = 10
-  , lmdbMaxReaders = 16
+defaultLMDBLimits :: LMDB.LMDBLimits
+defaultLMDBLimits = LMDB.LMDBLimits {
+    LMDB.lmdbMapSize = 16 * 1024 * 1024 * 1024
+  , LMDB.lmdbMaxDatabases = 10
+  , LMDB.lmdbMaxReaders = 16
   }
 
-defaultLMDBPath :: FilePath
-defaultLMDBPath = "mainnet/db/lmdb"
+defaultLMDBPath :: FilePath -> FilePath
+defaultLMDBPath = (</> "lmdb")
 
-selectorToArgs :: LedgerDbSelectorFlag -> Complete LedgerDbFlavorArgs IO
-selectorToArgs (V1InMemory ff)  = LedgerDbFlavorArgsV1 $ V1.V1Args ff V1.InMemoryBackingStoreArgs
-selectorToArgs V2InMemory      = LedgerDbFlavorArgsV2 $ V2.V2Args V2.InMemoryHandleArgs
-selectorToArgs (V1LMDB ff fp l mxReaders) =
-    LedgerDbFlavorArgsV1
-  $ V1.V1Args ff
-  $ V1.LMDBBackingStoreArgs
-      (fromMaybe defaultLMDBPath fp)
-      (maybe id (\overrideMaxReaders lim -> lim { lmdbMaxReaders = overrideMaxReaders }) mxReaders
-       $ maybe id (\ll lim -> lim { lmdbMapSize = toBytes ll }) l defaultLMDBLimits)
-      Dict
+selectorToArgs :: forall blk. (LedgerSupportsProtocol blk, LedgerSupportsLedgerDB blk) => LedgerDbSelectorFlag -> FilePath -> StdGen -> (LedgerDbBackendArgs IO blk, StdGen)
+selectorToArgs V2InMemory _ = InMemory.mkInMemoryArgs
+selectorToArgs (V1LMDB ff fp l mxReaders) fastStoragePath =
+    LMDB.mkLMDBArgs
+        ff
+        (fromMaybe (defaultLMDBPath fastStoragePath) fp)
+        ( maybe id (\overrideMaxReaders lim -> lim{LMDB.lmdbMaxReaders = overrideMaxReaders}) mxReaders $
+            maybe id (\ll lim -> lim{LMDB.lmdbMapSize = toBytes ll}) l defaultLMDBLimits
+        )
+selectorToArgs (V2LSM fp) fastStoragePath = LSM.mkLSMArgs (Proxy @blk) (fromMaybe "lsm" fp) fastStoragePath