From b4e531462e604fb9b9fbd5d8f2e312547f2c538e Mon Sep 17 00:00:00 2001 From: blindchaser Date: Sun, 9 Aug 2026 18:47:41 -0400 Subject: [PATCH 01/13] feat(seidb): add exact-version state store snapshots Co-authored-by: Cursor --- app/config_fuzz_test.go | 44 +- app/seidb.go | 7 + app/testdata/state-store.golden | 4 + app/testdata/state-store.keys.golden | 1 + sei-cosmos/server/config/config.go | 8 + .../config/testdata/server_config.golden | 4 + sei-cosmos/storev2/rootmulti/store.go | 8 + sei-db/common/utils/path.go | 7 + sei-db/config/sc_config.go | 29 + sei-db/config/ss_config.go | 39 + sei-db/config/ss_config_test.go | 106 +++ sei-db/config/toml.go | 23 + sei-db/db_engine/pebbledb/mvcc/db.go | 73 +- sei-db/db_engine/pebbledb/mvcc/db_test.go | 51 +- sei-db/db_engine/types/types.go | 58 ++ sei-db/state_db/sc/composite/store.go | 9 +- sei-db/state_db/ss/composite/snapshot.go | 672 +++++++++++++++ .../state_db/ss/composite/snapshot_metrics.go | 94 +++ sei-db/state_db/ss/composite/snapshot_test.go | 777 ++++++++++++++++++ sei-db/state_db/ss/composite/store.go | 44 +- sei-db/state_db/ss/cosmos/store.go | 21 + sei-db/state_db/ss/evm/store.go | 84 ++ 22 files changed, 2127 insertions(+), 36 deletions(-) create mode 100644 sei-db/config/ss_config_test.go create mode 100644 sei-db/state_db/ss/composite/snapshot.go create mode 100644 sei-db/state_db/ss/composite/snapshot_metrics.go create mode 100644 sei-db/state_db/ss/composite/snapshot_test.go diff --git a/app/config_fuzz_test.go b/app/config_fuzz_test.go index fd735504b9..5da665231f 100644 --- a/app/config_fuzz_test.go +++ b/app/config_fuzz_test.go @@ -23,11 +23,9 @@ import ( // // - parseSCConfigs guards almost every read with `if v := opts.Get(k); v != nil`, // so a key absent from an older app.toml keeps its non-zero in-code default. -// - parseSSConfigs guards nothing. Every read is a bare cast of a possibly-nil -// value, so an absent key resolves to the zero value and overwrites the -// default. ss-keep-recent becomes 0 (keep everything, unbounded disk growth), -// ss-async-write-buffer becomes 0 (synchronous writes), ss-backend becomes "" -// and ss-enable becomes false. +// - parseSSConfigs leaves most reads unguarded, so an absent key resolves to +// the zero value and overwrites the default. SnapshotEnable is deliberately +// guarded so older app.toml files inherit the new default. // // Neither reader returns an error, so nothing about the second case is visible at // boot. It is recorded here as behavior rather than reported as a defect: the @@ -84,8 +82,8 @@ var scKeys = []configtest.KeySpec{ }, } -// ssKeys is the [state-store] read-site manifest. Every row is unguarded and -// unchecked — the section has no presence checks at all. +// ssKeys is the [state-store] read-site manifest. Every row is unchecked; +// SnapshotEnable is guarded while the legacy rows remain unguarded. // // StateStoreConfig also carries KeepLastVersion and UseDefaultComparer, which are // absent here because parseSSConfigs reads neither: they hold their in-code @@ -112,6 +110,10 @@ var ssKeys = []configtest.KeySpec{ {Key: FlagSSImportNumWorkers, Path: "ImportNumWorkers", Cast: configtest.CastInt, Unguarded: true}, {Key: FlagSSDirectory, Path: "DBDirectory", Cast: configtest.CastString, Unguarded: true}, {Key: FlagSSReadWriteMetrics, Path: "EnableReadWriteMetrics", Cast: configtest.CastBool, Unguarded: true}, + { + Key: FlagSSSnapshotEnable, Path: "SnapshotEnable", Cast: configtest.CastBool, + Why: "guarded so app.toml files created before SS snapshots keep the default-off rollout", + }, {Key: FlagEVMSSDirectory, Path: "EVMDBDirectory", Cast: configtest.CastString, Unguarded: true}, {Key: FlagEVMSSSeparateDBs, Path: "SeparateEVMSubDBs", Cast: configtest.CastBool, Unguarded: true}, {Key: FlagEVMSSSplit, Path: "EVMSplit", Cast: configtest.CastBool, Unguarded: true}, @@ -253,9 +255,8 @@ func FuzzParseSCConfigs(f *testing.F) { } // FuzzParseSSConfigs drives every [state-store] key through arbitrary raw values. -// Because the whole section is unguarded, the property being pinned for a nil -// value is the clobber itself: the resolved field must equal the cast's zero, not -// the in-code default. +// For legacy unguarded rows, a nil value must clobber the field to the cast's +// zero. Guarded rows such as SnapshotEnable must retain their in-code default. func FuzzParseSSConfigs(f *testing.F) { seeds := configtest.NewSeeds(f, fuzzing.ConfigValue) @@ -277,7 +278,8 @@ func FuzzParseSSConfigs(f *testing.F) { seeds.AddRow(uint(3), fuzzing.KindInt64, "", int64(200000), false) seeds.AddRow(uint(3), fuzzing.KindNil, "", int64(0), false) // nil clobbers KeepRecent to 0 seeds.AddRow(uint(6), fuzzing.KindString, "/var/lib/sei/ss", int64(0), false) - seeds.AddRow(uint(10), fuzzing.KindBoolString, "", int64(0), true) + seeds.AddRow(uint(8), fuzzing.KindBool, "", int64(0), true) // explicit snapshot opt-in; the default is off + seeds.AddRow(uint(11), fuzzing.KindBoolString, "", int64(0), true) // The clobber cuts both ways for the four rows below. Because the section is unguarded, // an absent key resolves them to their cast's zero, and so does the malformed seed on an @@ -287,7 +289,7 @@ func FuzzParseSSConfigs(f *testing.F) { seeds.AddRow(uint(4), fuzzing.KindInt64, "", int64(1800), false) // prune every 30 min rather than the default 600s seeds.AddRow(uint(5), fuzzing.KindInt64, "", int64(4), false) // four import workers rather than the default 1 seeds.AddRow(uint(7), fuzzing.KindBool, "", int64(0), true) // pebbledb read/write metrics on; the default is off - seeds.AddRow(uint(9), fuzzing.KindBool, "", int64(0), true) // EVM state in its own sub-DBs; the default is shared + seeds.AddRow(uint(10), fuzzing.KindBool, "", int64(0), true) // EVM state in its own sub-DBs; the default is shared configtest.CheckEveryRowHasADiscriminatingSeed(f, "state-store", readSS, ssKeys, seeds) @@ -518,10 +520,9 @@ func TestParseSCConfigsAbsentBaseline(t *testing.T) { } } -// TestParseSSConfigsAbsentBaselineIsZeroClobbered records the clobber in full: an -// app.toml with no [state-store] section resolves to a config in which every -// operator-visible knob has been overwritten with a zero value, including the two -// that change the node's disk behavior without any log line. +// TestParseSSConfigsAbsentBaselineIsZeroClobbered records the legacy clobber: +// every unguarded operator-visible knob resolves to zero, while guarded fields +// such as SnapshotEnable retain their in-code default. func TestParseSSConfigsAbsentBaselineIsZeroClobbered(t *testing.T) { got := parseSSConfigs(configtest.AppOpts{}) @@ -665,14 +666,13 @@ func TestDefaultsMatchTheRecordedValues(t *testing.T) { func TestManifestNamesEveryField(t *testing.T) { t.Run("state-store", func(t *testing.T) { configtest.CheckManifestCoversEveryField(t, "state-store", config.DefaultStateStoreConfig(), ssKeys, - // Both are tagged mapstructure but no [state-store] key reaches either: parseSSConfigs - // reads neither, so both hold their in-code defaults on every node. pebbledb consumes - // them at construction (KeepLastVersion in mvcc pruning, UseDefaultComparer in the - // comparer selection), which is worth stating rather than omitting — a field a config - // struct carries that configuration cannot address is exactly what a replacement - // manager would otherwise try to map a key onto. + // These fields have no independent [state-store] key. The first two + // retain their in-code defaults; snapshot cadence is derived from SC. "KeepLastVersion", "UseDefaultComparer", + "SnapshotInterval", + "SnapshotKeepRecent", + "SnapshotMinTimeInterval", ) }) t.Run("light_invariance", func(t *testing.T) { diff --git a/app/seidb.go b/app/seidb.go index 9307af2bb0..716583de37 100644 --- a/app/seidb.go +++ b/app/seidb.go @@ -48,6 +48,7 @@ const ( FlagSSPruneInterval = "state-store.ss-prune-interval" FlagSSImportNumWorkers = "state-store.ss-import-num-workers" FlagSSReadWriteMetrics = "state-store.ss-enable-read-write-metrics" + FlagSSSnapshotEnable = "state-store.ss-snapshot-enable" // EVM SS optimization (embedded in SS config, controlled via write/read mode) FlagEVMSSDirectory = "state-store.evm-ss-db-directory" @@ -204,6 +205,12 @@ func parseSSConfigs(appOpts servertypes.AppOptions) config.StateStoreConfig { ssConfig.DBDirectory = cast.ToString(appOpts.Get(FlagSSDirectory)) ssConfig.EnableReadWriteMetrics = cast.ToBool(appOpts.Get(FlagSSReadWriteMetrics)) + // An absent key is an app.toml rendered before SS snapshots existed. Keep + // the in-code default (off) rather than relying on a nil cast. + if v := appOpts.Get(FlagSSSnapshotEnable); v != nil { + ssConfig.SnapshotEnable = cast.ToBool(v) + } + // EVM optimization fields (embedded in SS config) ssConfig.EVMDBDirectory = cast.ToString(appOpts.Get(FlagEVMSSDirectory)) ssConfig.SeparateEVMSubDBs = cast.ToBool(appOpts.Get(FlagEVMSSSeparateDBs)) diff --git a/app/testdata/state-store.golden b/app/testdata/state-store.golden index c60c8b49e2..57d01bd92f 100644 --- a/app/testdata/state-store.golden +++ b/app/testdata/state-store.golden @@ -8,6 +8,10 @@ ImportNumWorkers = int(1) EnableReadWriteMetrics = bool(false) KeepLastVersion = bool(true) UseDefaultComparer = bool(false) +SnapshotEnable = bool(false) +SnapshotInterval = int64(0) +SnapshotKeepRecent = int(0) +SnapshotMinTimeInterval = time.Duration(0s) EVMSplit = bool(false) EVMDBDirectory = string("") SeparateEVMSubDBs = bool(false) diff --git a/app/testdata/state-store.keys.golden b/app/testdata/state-store.keys.golden index fc808ad460..a96612730c 100644 --- a/app/testdata/state-store.keys.golden +++ b/app/testdata/state-store.keys.golden @@ -6,6 +6,7 @@ "state-store.ss-import-num-workers" "state-store.ss-db-directory" "state-store.ss-enable-read-write-metrics" +"state-store.ss-snapshot-enable" "state-store.evm-ss-db-directory" "state-store.evm-ss-separate-dbs" "state-store.evm-ss-split" diff --git a/sei-cosmos/server/config/config.go b/sei-cosmos/server/config/config.go index c46a296125..7bf8b4d70a 100644 --- a/sei-cosmos/server/config/config.go +++ b/sei-cosmos/server/config/config.go @@ -508,6 +508,13 @@ func GetConfig(v *viper.Viper) (Config, error) { memIAVLConfig.SnapshotPrefetchThreshold = v.GetFloat64("state-commit.sc-snapshot-prefetch-threshold") } + // Absent key means an app.toml rendered before SS snapshots existed, which + // should keep the in-code default (off) rather than rely on viper's zero. + ssSnapshotEnable := config.DefaultStateStoreConfig().SnapshotEnable + if v.IsSet("state-store.ss-snapshot-enable") { + ssSnapshotEnable = v.GetBool("state-store.ss-snapshot-enable") + } + // Apply the in-code default when the key is absent so that nodes upgrading // with an older app.toml (which lacks this key) are still bounded rather // than running with unlimited connections. @@ -636,6 +643,7 @@ func GetConfig(v *viper.Viper) (Config, error) { EnableReadWriteMetrics: v.GetBool( "state-store.ss-enable-read-write-metrics", ), + SnapshotEnable: ssSnapshotEnable, EVMSplit: v.GetBool("state-store.evm-ss-split"), EVMDBDirectory: v.GetString("state-store.evm-ss-db-directory"), SeparateEVMSubDBs: v.GetBool("state-store.evm-ss-separate-dbs"), diff --git a/sei-cosmos/server/config/testdata/server_config.golden b/sei-cosmos/server/config/testdata/server_config.golden index a2ce09015b..7327621a2f 100644 --- a/sei-cosmos/server/config/testdata/server_config.golden +++ b/sei-cosmos/server/config/testdata/server_config.golden @@ -139,6 +139,10 @@ StateStore.ImportNumWorkers = int(1) StateStore.EnableReadWriteMetrics = bool(false) StateStore.KeepLastVersion = bool(true) StateStore.UseDefaultComparer = bool(false) +StateStore.SnapshotEnable = bool(false) +StateStore.SnapshotInterval = int64(0) +StateStore.SnapshotKeepRecent = int(0) +StateStore.SnapshotMinTimeInterval = time.Duration(0s) StateStore.EVMSplit = bool(false) StateStore.EVMDBDirectory = string("") StateStore.SeparateEVMSubDBs = bool(false) diff --git a/sei-cosmos/storev2/rootmulti/store.go b/sei-cosmos/storev2/rootmulti/store.go index 43b8387347..23ddf5ab9c 100644 --- a/sei-cosmos/storev2/rootmulti/store.go +++ b/sei-cosmos/storev2/rootmulti/store.go @@ -48,6 +48,10 @@ var ( _ types.Queryable = (*Store)(nil) ) +type stateStoreSnapshotScheduler interface { + ScheduleSnapshot(version int64) +} + type Store struct { mtx sync.RWMutex scStore sctypes.Committer @@ -136,6 +140,7 @@ func NewStore( scDir: scDir, } if ssConfig.Enable { + config.AlignSSSnapshotWithSC(scConfig, &ssConfig) ssStore, err := ss.NewStateStore(homeDir, ssConfig) if err != nil { panic(err) @@ -250,6 +255,9 @@ func (rs *Store) flush() error { if err := rs.ssStore.SetLatestVersion(currentVersion); err != nil { panic(err) } + if scheduler, ok := rs.ssStore.(stateStoreSnapshotScheduler); ok { + scheduler.ScheduleSnapshot(currentVersion) + } storev2Metrics.ssVersion.Record(context.Background(), currentVersion) // TODO(PLT-353): remove once storev2_ss_version verified telemetry.SetGauge(float32(currentVersion), "storeV2", "ss", "version") diff --git a/sei-db/common/utils/path.go b/sei-db/common/utils/path.go index d8a03fe841..d073091bad 100644 --- a/sei-db/common/utils/path.go +++ b/sei-db/common/utils/path.go @@ -7,6 +7,8 @@ import ( "strings" ) +const StateStoreSnapshotsDirName = "snapshots" + // DirExists returns true if path exists and is a directory. func DirExists(path string) bool { info, err := os.Stat(path) @@ -63,6 +65,11 @@ func GetEVMStateStorePath(homePath string, backend string) string { return filepath.Join(homePath, "data", "state_store", "evm", backend) } +// GetStateStoreSnapshotsPath returns the path for online state-store snapshots. +func GetStateStoreSnapshotsPath(homePath string) string { + return filepath.Join(homePath, "data", "state_store", StateStoreSnapshotsDirName) +} + // GetReceiptStorePath returns the path for the receipt store. // New nodes use data/ledger/receipt/{backend}; existing nodes with // data/receipt.db continue using the legacy path for backward compatibility. diff --git a/sei-db/config/sc_config.go b/sei-db/config/sc_config.go index 48ec3635ba..6b3bda0a54 100644 --- a/sei-db/config/sc_config.go +++ b/sei-db/config/sc_config.go @@ -2,6 +2,7 @@ package config import ( "fmt" + "time" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" @@ -16,6 +17,34 @@ const ( legacySCWriteModeCosmosOnly = "cosmos_only" ) +// EffectiveMemIAVLSnapshotCadence resolves memIAVL's snapshot cadence the way +// Options.FillDefaults resolves it at OpenDB, so a caller mirroring the cadence +// onto another backend sees the values memIAVL will actually run with rather +// than the raw config. A zero means "unset" here, not "disabled": memIAVL heals +// it to the default, so mirroring the raw zero would silently disable snapshots +// on the mirroring backend. +func EffectiveMemIAVLSnapshotCadence(cfg memiavl.Config) (interval, keepRecent uint32) { + interval = cfg.SnapshotInterval + if interval == 0 { + interval = memiavl.DefaultSnapshotInterval + } + keepRecent = cfg.SnapshotKeepRecent + if keepRecent == 0 { + keepRecent = memiavl.DefaultSnapshotKeepRecent + } + return interval, keepRecent +} + +// EffectiveMemIAVLSnapshotMinTimeInterval resolves the minimum wall-clock +// interval the same way memIAVL Options.FillDefaults does. +func EffectiveMemIAVLSnapshotMinTimeInterval(cfg memiavl.Config) time.Duration { + seconds := cfg.SnapshotMinTimeInterval + if seconds == 0 { + seconds = memiavl.DefaultSnapshotMinTimeInterval + } + return time.Duration(seconds) * time.Second +} + // StateCommitConfig defines configuration for the state commit (SC) layer. type StateCommitConfig struct { // Enable defines if the state-commit (SeiDB) should be enabled. diff --git a/sei-db/config/ss_config.go b/sei-db/config/ss_config.go index a3a89b898e..03b22aed6a 100644 --- a/sei-db/config/ss_config.go +++ b/sei-db/config/ss_config.go @@ -1,5 +1,7 @@ package config +import "time" + // DBBackend defines the SS DB backend. type DBBackend string @@ -63,6 +65,25 @@ type StateStoreConfig struct { // defaults to false (use MVCCComparer for backwards compatibility) UseDefaultComparer bool `mapstructure:"use-default-comparer"` + // SnapshotEnable controls whether the state store takes periodic online + // snapshots. Snapshots are Pebble checkpoints (hardlink trees), so each + // retained snapshot pins the SSTs it references and prevents compaction + // from reclaiming them. Steady-state disk overhead is therefore the + // compaction churn accumulated over SnapshotInterval blocks, per retained + // snapshot — significant on a multi-TB state store. Managed snapshots have + // no lease in this release, so consumers must quiesce generation and pruning + // before using a snapshot directory. + // defaults to false + SnapshotEnable bool `mapstructure:"snapshot-enable"` + + // SnapshotInterval, SnapshotKeepRecent, and SnapshotMinTimeInterval are + // mirrored from the state-commit snapshot settings at runtime by + // AlignSSSnapshotWithSC. They are intentionally not exposed in app.toml; + // SnapshotEnable is the only SS-side knob. + SnapshotInterval int64 `mapstructure:"-"` + SnapshotKeepRecent int `mapstructure:"-"` + SnapshotMinTimeInterval time.Duration `mapstructure:"-"` + // --- EVM optimization fields --- // EVMSplit controls whether EVM data is routed to a dedicated SS backend. @@ -93,7 +114,25 @@ func DefaultStateStoreConfig() StateStoreConfig { ImportNumWorkers: DefaultSSImportWorkers, KeepLastVersion: true, UseDefaultComparer: false, + SnapshotEnable: false, EVMSplit: false, SeparateEVMSubDBs: false, } } + +// AlignSSSnapshotWithSC mirrors the state-commit interval, minimum time +// interval, and retention settings onto the state store. SC and SS apply their +// in-flight gates independently, so this does not promise identical retained +// heights. When SS snapshots are disabled the cadence is zeroed. +func AlignSSSnapshotWithSC(scConfig StateCommitConfig, ssConfig *StateStoreConfig) { + if !ssConfig.SnapshotEnable { + ssConfig.SnapshotInterval = 0 + ssConfig.SnapshotKeepRecent = 0 + ssConfig.SnapshotMinTimeInterval = 0 + return + } + interval, keepRecent := EffectiveMemIAVLSnapshotCadence(scConfig.MemIAVLConfig) + ssConfig.SnapshotInterval = int64(interval) + ssConfig.SnapshotKeepRecent = int(keepRecent) + ssConfig.SnapshotMinTimeInterval = EffectiveMemIAVLSnapshotMinTimeInterval(scConfig.MemIAVLConfig) +} diff --git a/sei-db/config/ss_config_test.go b/sei-db/config/ss_config_test.go new file mode 100644 index 0000000000..543f50ba0c --- /dev/null +++ b/sei-db/config/ss_config_test.go @@ -0,0 +1,106 @@ +package config + +import ( + "testing" + "time" + + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" + "github.com/stretchr/testify/require" +) + +func TestAlignSSSnapshotWithSC(t *testing.T) { + scConfig := DefaultStateCommitConfig() + ssConfig := DefaultStateStoreConfig() + ssConfig.SnapshotEnable = true + + scConfig.MemIAVLConfig.SnapshotInterval = 123 + scConfig.MemIAVLConfig.SnapshotKeepRecent = 4 + scConfig.MemIAVLConfig.SnapshotMinTimeInterval = 17 + + AlignSSSnapshotWithSC(scConfig, &ssConfig) + + require.Equal(t, int64(123), ssConfig.SnapshotInterval) + require.Equal(t, 4, ssConfig.SnapshotKeepRecent) + require.Equal(t, 17*time.Second, ssConfig.SnapshotMinTimeInterval) +} + +func TestAlignSSSnapshotWithSCHealsZeroToSCDefaults(t *testing.T) { + scConfig := DefaultStateCommitConfig() + ssConfig := DefaultStateStoreConfig() + ssConfig.SnapshotEnable = true + + scConfig.MemIAVLConfig.SnapshotInterval = 0 + scConfig.MemIAVLConfig.SnapshotKeepRecent = 0 + + AlignSSSnapshotWithSC(scConfig, &ssConfig) + + require.Equal(t, int64(memiavl.DefaultSnapshotInterval), ssConfig.SnapshotInterval) + require.Equal(t, memiavl.DefaultSnapshotKeepRecent, ssConfig.SnapshotKeepRecent) + require.Equal( + t, + time.Duration(memiavl.DefaultSnapshotMinTimeInterval)*time.Second, + ssConfig.SnapshotMinTimeInterval, + ) +} + +func TestDefaultStateStoreConfigDisablesSnapshots(t *testing.T) { + require.False(t, DefaultStateStoreConfig().SnapshotEnable, + "snapshots require an explicit ss-snapshot-enable opt-in") +} + +// A zero cadence is what the snapshot manager reads as "do not run", so the +// off switch has to zero it rather than mirror SC's. +func TestAlignSSSnapshotWithSCZeroesCadenceWhenDisabled(t *testing.T) { + scConfig := DefaultStateCommitConfig() + scConfig.MemIAVLConfig.SnapshotInterval = 123 + scConfig.MemIAVLConfig.SnapshotKeepRecent = 4 + scConfig.MemIAVLConfig.SnapshotMinTimeInterval = 17 + + ssConfig := DefaultStateStoreConfig() + ssConfig.SnapshotEnable = false + + AlignSSSnapshotWithSC(scConfig, &ssConfig) + + require.Zero(t, ssConfig.SnapshotInterval) + require.Zero(t, ssConfig.SnapshotKeepRecent) + require.Zero(t, ssConfig.SnapshotMinTimeInterval) +} + +// FlatKV and SS both mirror memIAVL's cadence, and they must resolve it +// identically or the two backends drift onto different snapshot heights. +func TestAlignSSSnapshotMatchesEffectiveMemIAVLCadence(t *testing.T) { + for _, tc := range []struct { + name string + interval, keepRecent uint32 + minTime uint32 + wantInterval int64 + wantMinTime time.Duration + }{ + { + name: "explicit", interval: 500, keepRecent: 3, minTime: 45, + wantInterval: 500, wantMinTime: 45 * time.Second, + }, + { + name: "zero heals to default", interval: 0, keepRecent: 0, + wantInterval: memiavl.DefaultSnapshotInterval, + wantMinTime: time.Duration(memiavl.DefaultSnapshotMinTimeInterval) * time.Second, + }, + } { + t.Run(tc.name, func(t *testing.T) { + scConfig := DefaultStateCommitConfig() + scConfig.MemIAVLConfig.SnapshotInterval = tc.interval + scConfig.MemIAVLConfig.SnapshotKeepRecent = tc.keepRecent + scConfig.MemIAVLConfig.SnapshotMinTimeInterval = tc.minTime + + ssConfig := DefaultStateStoreConfig() + ssConfig.SnapshotEnable = true + AlignSSSnapshotWithSC(scConfig, &ssConfig) + + wantInterval, wantKeepRecent := EffectiveMemIAVLSnapshotCadence(scConfig.MemIAVLConfig) + require.Equal(t, int64(wantInterval), ssConfig.SnapshotInterval) + require.Equal(t, int(wantKeepRecent), ssConfig.SnapshotKeepRecent) + require.Equal(t, tc.wantInterval, ssConfig.SnapshotInterval) + require.Equal(t, tc.wantMinTime, ssConfig.SnapshotMinTimeInterval) + }) + } +} diff --git a/sei-db/config/toml.go b/sei-db/config/toml.go index d7e782bb89..0fc2c459e1 100644 --- a/sei-db/config/toml.go +++ b/sei-db/config/toml.go @@ -140,6 +140,29 @@ ss-import-num-workers = {{ .StateStore.ImportNumWorkers }} # Applies when ss-backend = "pebbledb". Default: false. ss-enable-read-write-metrics = {{ .StateStore.EnableReadWriteMetrics }} +# SnapshotEnable controls whether the state store takes periodic online +# snapshots. The cadence is not configurable here: it mirrors the state-commit +# snapshot interval, minimum time interval, and retention settings. SC and SS +# apply their in-flight gates independently, so a skipped boundary can differ. +# Snapshots are PebbleDB checkpoints, i.e. hardlink trees. Creating one blocks +# each backend's SS apply worker for the full WAL flush, filesystem sync, and +# checkpoint operation; a full async queue then applies write backpressure. It +# does not copy data up front. Startup rejects snapshot configurations where a +# live SS database and the snapshot root cannot hardlink to each other. Each +# enabled Cosmos and EVM SS database must therefore use the same filesystem. A +# custom Cosmos SS directory moves the snapshot root beside that directory. +# Retained snapshots pin referenced SSTs, so compaction cannot reclaim them. +# Expect steady-state disk overhead on the order of the compaction churn over +# one snapshot interval per retained snapshot, which is substantial on a +# multi-TB state store. +# Managed snapshot directories have no lease in this release. Do not pack an +# archive or serve state sync directly from them while the node is running: +# retention can remove a directory during use. Stop the node, or use external +# coordination that prevents pruning, before consuming a snapshot. Snapshot +# attempts, skips, outcomes, duration, in-flight state, height, count, and +# apparent bytes are exported through ss_snapshot_* metrics. Default: false. +ss-snapshot-enable = {{ .StateStore.SnapshotEnable }} + # EVMDBDirectory defines the directory for the optional EVM state-store DB(s). # If unset, defaults to /data/evm_ss when EVM SS is enabled. evm-ss-db-directory = "{{ .StateStore.EVMDBDirectory }}" diff --git a/sei-db/db_engine/pebbledb/mvcc/db.go b/sei-db/db_engine/pebbledb/mvcc/db.go index 114659be57..936bcdd6a1 100644 --- a/sei-db/db_engine/pebbledb/mvcc/db.go +++ b/sei-db/db_engine/pebbledb/mvcc/db.go @@ -101,12 +101,12 @@ type VersionedChangesets struct { Version int64 Changesets []*proto.NamedChangeSet Done chan struct{} // non-nil for barrier: closed when this entry is processed + // AtDrain, when non-nil, is run by the apply goroutine in queue order + // instead of applying a changeset. See ScheduleAtDrain. + AtDrain func() } -func OpenDB(dataDir string, config config.StateStoreConfig) (types.StateStore, error) { - cache := pebble.NewCache(1024 * 1024 * 32) - defer cache.Unref() - +func newPebbleOptions(config config.StateStoreConfig, cache *pebble.Cache) *pebble.Options { // Select comparer based on config. Note: UseDefaultComparer is NOT backwards compatible // with existing databases created with MVCCComparer - Pebble will refuse to open due to // comparer name mismatch. Only use UseDefaultComparer for NEW databases. @@ -162,6 +162,14 @@ func OpenDB(dataDir string, config config.StateStoreConfig) (types.StateStore, e //TODO: add a new config and check if readonly = true to support readonly mode + return opts +} + +func OpenDB(dataDir string, config config.StateStoreConfig) (types.StateStore, error) { + cache := pebble.NewCache(1024 * 1024 * 32) + defer cache.Unref() + + opts := newPebbleOptions(config, cache) db, err := pebble.Open(dataDir, opts) if err != nil { return nil, fmt.Errorf("failed to open PebbleDB: %w", err) @@ -278,6 +286,46 @@ func (db *Database) PebbleMetrics() *pebble.Metrics { return db.storage.Metrics() } +// Checkpoint writes a point-in-time snapshot of the database into destDir +// (which must not exist yet). Pebble implements this with hardlinks to +// already-fsynced SSTs plus a flushed WAL. SS schedules it on the apply +// goroutine at an ordered queue boundary, so that backend cannot apply more +// changes until the WAL flush, filesystem sync, and checkpoint creation finish. +// Satisfies types.Checkpointable. +func (db *Database) Checkpoint(destDir string) error { + if err := db.storage.Checkpoint(destDir, pebble.WithFlushedWAL()); err != nil { + return fmt.Errorf("pebble checkpoint to %q: %w", destDir, err) + } + return nil +} + +// SetCheckpointVersion writes version into a completed checkpoint without +// changing the live database marker. +func (db *Database) SetCheckpointVersion(destDir string, version int64) error { + if version < 0 { + return fmt.Errorf("version must be non-negative") + } + + opts := newPebbleOptions(db.config, nil) + opts.DisableAutomaticCompactions = true + checkpoint, err := pebble.Open(destDir, opts) + if err != nil { + return fmt.Errorf("open checkpoint %q to set version: %w", destDir, err) + } + + var marker [VersionSize]byte + binary.LittleEndian.PutUint64(marker[:], uint64(version)) + setErr := checkpoint.Set([]byte(latestVersionKey), marker[:], pebble.Sync) + closeErr := checkpoint.Close() + if setErr != nil { + setErr = fmt.Errorf("set checkpoint version %d: %w", version, setErr) + } + if closeErr != nil { + closeErr = fmt.Errorf("close checkpoint after setting version %d: %w", version, closeErr) + } + return errors.Join(setErr, closeErr) +} + func (db *Database) SetLatestVersion(version int64) error { if version < 0 { return fmt.Errorf("version must be non-negative") @@ -490,6 +538,10 @@ func (db *Database) ApplyChangesetAsync(version int64, changesets []*proto.Named func (db *Database) writeAsyncInBackground() { defer db.asyncWriteWG.Done() for nextChange := range db.pendingChanges { + if nextChange.AtDrain != nil { + nextChange.AtDrain() + continue + } if nextChange.Done != nil { close(nextChange.Done) continue @@ -508,6 +560,19 @@ func (db *Database) WaitForPendingWrites() { <-done } +// ScheduleAtDrain runs fn on the apply goroutine at the point in the queue +// where every changeset enqueued before this call has been applied and none +// enqueued after it has. Unlike WaitForPendingWrites it does not block the +// caller, which is what lets a caller capture the DB at an exact version +// without stalling the block it is committing: the version is pinned by fn's +// position in the queue rather than by when it runs. +// +// fn runs on the writer, so it must not enqueue more work on this DB (that +// deadlocks once the buffer fills) and must not panic. +func (db *Database) ScheduleAtDrain(fn func()) { + db.pendingChanges <- VersionedChangesets{AtDrain: fn} +} + // Prune dispatches between descending- and ascending-mode implementations // depending on the on-disk encoding detected at open time. func (db *Database) Prune(version int64) error { diff --git a/sei-db/db_engine/pebbledb/mvcc/db_test.go b/sei-db/db_engine/pebbledb/mvcc/db_test.go index b923188718..18ba53dad7 100644 --- a/sei-db/db_engine/pebbledb/mvcc/db_test.go +++ b/sei-db/db_engine/pebbledb/mvcc/db_test.go @@ -1,12 +1,15 @@ package mvcc import ( + "encoding/binary" + "path/filepath" "testing" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/sei-protocol/sei-chain/sei-db/config" - "github.com/sei-protocol/sei-chain/sei-db/db_engine/test" + sstest "github.com/sei-protocol/sei-chain/sei-db/db_engine/test" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" ) @@ -46,3 +49,49 @@ func TestStorageTestSuiteDefaultComparer(t *testing.T) { suite.Run(t, s) } + +func TestVersionedCheckpointPreservesFutureLiveMarker(t *testing.T) { + cfg := config.DefaultStateStoreConfig() + cfg.Backend = config.PebbleDBBackend + + store, err := OpenDB(filepath.Join(t.TempDir(), "live"), cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + require.NoError(t, store.SetLatestVersion(10)) + + dest := filepath.Join(t.TempDir(), "snapshot") + done := make(chan error, 1) + types.ScheduleCheckpoint(store, dest, nil, func(err error) { + done <- err + }) + require.NoError(t, <-done) + require.NoError(t, types.SetCheckpointVersion(store, dest, 5)) + require.Equal(t, int64(10), store.GetLatestVersion()) + marker, closer, err := store.(*Database).storage.Get([]byte(latestVersionKey)) + require.NoError(t, err) + require.Equal(t, uint64(10), binary.LittleEndian.Uint64(marker)) + require.NoError(t, closer.Close()) + + checkpoint, err := OpenDB(dest, cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, checkpoint.Close()) }) + require.Equal(t, int64(5), checkpoint.GetLatestVersion()) +} + +func TestScheduledCheckpointCanBeCanceledAtBarrier(t *testing.T) { + cfg := config.DefaultStateStoreConfig() + cfg.Backend = config.PebbleDBBackend + + store, err := OpenDB(filepath.Join(t.TempDir(), "live"), cfg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + + dest := filepath.Join(t.TempDir(), "snapshot") + done := make(chan error, 1) + types.ScheduleCheckpoint(store, dest, func() bool { return false }, func(err error) { + done <- err + }) + + require.ErrorIs(t, <-done, types.ErrCheckpointCanceled) + require.NoDirExists(t, dest) +} diff --git a/sei-db/db_engine/types/types.go b/sei-db/db_engine/types/types.go index 00096bf691..d490b03b90 100644 --- a/sei-db/db_engine/types/types.go +++ b/sei-db/db_engine/types/types.go @@ -1,6 +1,8 @@ package types import ( + "errors" + "fmt" "io" "github.com/sei-protocol/sei-chain/sei-db/proto" @@ -120,6 +122,62 @@ type Checkpointable interface { Checkpoint(destDir string) error } +// CheckpointVersionSetter writes the logical height into a completed +// checkpoint without changing the live database. +type CheckpointVersionSetter interface { + SetCheckpointVersion(destDir string, version int64) error +} + +// DrainBarrier is an optional capability for engines that apply changesets from +// an async queue. It lets a caller place work at an exact point in the write +// order without waiting for the queue to drain. +type DrainBarrier interface { + ScheduleAtDrain(fn func()) +} + +// CheckpointScheduler coordinates checkpoints for stores with in-flight writes. +type CheckpointScheduler interface { + SupportsCheckpoint() bool + ScheduleCheckpoint(destDir string, shouldRun func() bool, done func(error)) + SetCheckpointVersion(destDir string, version int64) error +} + +// ErrCheckpointCanceled reports that a queued checkpoint was canceled before +// it started. +var ErrCheckpointCanceled = errors.New("state store checkpoint canceled") + +// ScheduleCheckpoint checkpoints an engine after all writes already enqueued +// on it have been applied. +func ScheduleCheckpoint(db StateStore, destDir string, shouldRun func() bool, done func(error)) { + cp, ok := db.(Checkpointable) + if !ok { + done(fmt.Errorf("state store backend %T does not support checkpoints", db)) + return + } + barrier, ok := db.(DrainBarrier) + if !ok { + done(fmt.Errorf("state store backend %T does not support ordered checkpoint barriers", db)) + return + } + barrier.ScheduleAtDrain(func() { + if shouldRun != nil && !shouldRun() { + done(ErrCheckpointCanceled) + return + } + done(cp.Checkpoint(destDir)) + }) +} + +// SetCheckpointVersion makes a completed checkpoint self-describing without +// changing the live database. +func SetCheckpointVersion(db StateStore, destDir string, version int64) error { + setter, ok := db.(CheckpointVersionSetter) + if !ok { + return fmt.Errorf("state store backend %T cannot set checkpoint versions", db) + } + return setter.SetCheckpointVersion(destDir, version) +} + // --------------------------------------------------------------------------- // SS DB layer // --------------------------------------------------------------------------- diff --git a/sei-db/state_db/sc/composite/store.go b/sei-db/state_db/sc/composite/store.go index 0bca128e2d..c82d7a0ad6 100644 --- a/sei-db/state_db/sc/composite/store.go +++ b/sei-db/state_db/sc/composite/store.go @@ -247,14 +247,7 @@ func NewCompositeCommitStore( // different default. Note that mirroring a raw 0 is never correct here (0 means // "disable auto-snapshots" for FlatKV), which is why the zero is resolved first. func alignFlatKVSnapshotWithMemIAVL(cfg *config.StateCommitConfig) { - interval := cfg.MemIAVLConfig.SnapshotInterval - if interval == 0 { - interval = memiavl.DefaultSnapshotInterval - } - keepRecent := cfg.MemIAVLConfig.SnapshotKeepRecent - if keepRecent == 0 { - keepRecent = memiavl.DefaultSnapshotKeepRecent - } + interval, keepRecent := config.EffectiveMemIAVLSnapshotCadence(cfg.MemIAVLConfig) cfg.FlatKVConfig.SnapshotInterval = interval cfg.FlatKVConfig.SnapshotKeepRecent = keepRecent } diff --git a/sei-db/state_db/ss/composite/snapshot.go b/sei-db/state_db/ss/composite/snapshot.go new file mode 100644 index 0000000000..8a5487087e --- /dev/null +++ b/sei-db/state_db/ss/composite/snapshot.go @@ -0,0 +1,672 @@ +package composite + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + "time" + + "github.com/sei-protocol/sei-chain/sei-db/common/utils" + "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" +) + +// Online state-store snapshots. Every SnapshotInterval blocks the store takes a +// Pebble checkpoint of each backend while the node keeps producing blocks. +// Checkpoints are hardlink trees, so they do not copy database contents, but +// each one occupies its backend's apply goroutine for the full checkpoint +// operation. Writes continue to enter the bounded queue, but a full queue +// applies backpressure until the checkpoint finishes. The result is an +// immutable, crash-consistent image of the query store. +// +// On-disk layout under the snapshot root. By default the root is +// /data/state_store/snapshots. A custom Cosmos SS directory moves it +// to the sibling -snapshots directory so Pebble can use hardlinks. +// +// snapshots/ +// current -> snapshot-NNNNN (symlink to newest snapshot) +// snapshot-NNNNN/ (immutable; NNNNN = label version) +// cosmos// (Pebble checkpoint of Cosmos SS) +// evm// (Pebble checkpoint of EVM SS, if split) +// / (when EVM sub-DBs are separate) +// +// Snapshots are eligible at the same interval boundaries and minimum time +// cadence as state commit. Each layer applies its in-flight gate independently, +// so a skipped boundary can differ. For every accepted SS snapshot, the label +// is exact: it is the version the write path had just handed to the backends +// when the snapshot was requested. Placing a barrier in each backend's apply +// queue — rather than sampling what the backends had applied — makes that label +// exact without the request having to wait. See requestSnapshot. +// +// The barrier orders only the async block-commit queues. Import, recovery, +// pruning, and direct version-marker writes bypass those queues and must not +// call ScheduleSnapshot. The rootmulti commit path owns the trigger, including +// the explicit trigger for an empty block. +// +// Managed snapshot directories have no lease. A live consumer must not rely on +// a path remaining present across a retention pass. Until a lease API exists, +// consumers must stop the node or use external coordination that prevents +// pruning before they open or copy a snapshot. +const ( + // SnapshotsDirName is the directory under data/state_store that holds + // online snapshots. + SnapshotsDirName = utils.StateStoreSnapshotsDirName + + snapshotPrefix = "snapshot-" + // snapshotDirLen is "snapshot-" + 20-digit zero-padded version. + snapshotDirLen = len(snapshotPrefix) + 20 + + snapshotCurrentLink = "current" + snapshotCurrentTmpLink = "current-tmp" + snapshotTmpPrefix = "tmp-" + snapshotSizeFile = ".apparent-size" +) + +// SnapshotDirName returns the directory name for a snapshot labeled with the +// given version. +func SnapshotDirName(version int64) string { + return fmt.Sprintf("%s%020d", snapshotPrefix, version) +} + +// ParseSnapshotVersion parses a snapshot directory name; ok is false for +// anything that is not a snapshot-<20 digits> name. +func ParseSnapshotVersion(name string) (version int64, ok bool) { + if !strings.HasPrefix(name, snapshotPrefix) || len(name) != snapshotDirLen { + return 0, false + } + v, err := strconv.ParseInt(name[len(snapshotPrefix):], 10, 64) + if err != nil || v < 0 { + return 0, false + } + return v, true +} + +// ListSnapshotVersions returns the labels of all snapshots under root in +// ascending order. A missing root is not an error (no snapshots yet). +func ListSnapshotVersions(root string) ([]int64, error) { + entries, err := os.ReadDir(root) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("read snapshots dir %q: %w", root, err) + } + var versions []int64 + for _, entry := range entries { + if !entry.IsDir() { + continue + } + if v, ok := ParseSnapshotVersion(entry.Name()); ok { + versions = append(versions, v) + } + } + slices.Sort(versions) + return versions, nil +} + +// snapshotManager owns the snapshots directory and the one-at-a-time discipline +// for filling it. It has no goroutine of its own: snapshots are requested from +// the write path and completed on the backends' apply goroutines. +type snapshotManager struct { + root string + backend string + interval int64 + keepRecent int + minTime time.Duration + + cosmosScheduler types.CheckpointScheduler + evmScheduler types.CheckpointScheduler + snapshotSizes map[int64]int64 + + mu sync.Mutex + // lastRequested is the newest label already requested or on disk, so a + // boundary is not snapshotted twice across a restart or a re-sent version. + lastRequested int64 + lastRequestAt time.Time + inFlight bool + stopped bool + // scheduling closes the gap between accepting a request and enqueueing its + // barriers. Close waits for it before closing backend queues. + scheduling sync.WaitGroup + // publishing tracks the goroutine finishing the accepted snapshot off. + publishing sync.WaitGroup + + // publishMu serializes the publish step, which reads and rewrites the + // shared directory (the current link, and pruning). + publishMu sync.Mutex + lastPublished int64 +} + +type checkpointTarget struct { + store types.CheckpointScheduler + dest string +} + +// startSnapshotManager wires the manager into the composite store. Snapshot +// enablement is fail-closed: every backend must support checkpoints, and every +// live DB must be able to hardlink into root. Pebble otherwise silently falls +// back to copying SSTs across filesystems while its apply worker is blocked. +func (s *CompositeStateStore) startSnapshotManager(root string, sourceDirs []string) error { + if s.config.SnapshotInterval <= 0 { + return nil + } + cosmosScheduler, ok := s.cosmosStore.(types.CheckpointScheduler) + if !ok || !cosmosScheduler.SupportsCheckpoint() { + return fmt.Errorf("cosmos backend %q does not support checkpoints", s.config.Backend) + } + var evmScheduler types.CheckpointScheduler + if s.evmStore != nil { + evmScheduler, ok = s.evmStore.(types.CheckpointScheduler) + if !ok || !evmScheduler.SupportsCheckpoint() { + return fmt.Errorf("EVM backend %q does not support checkpoints", s.config.Backend) + } + } + if err := verifySnapshotHardlinks(root, sourceDirs); err != nil { + return err + } + m := &snapshotManager{ + root: root, + backend: s.config.Backend, + interval: s.config.SnapshotInterval, + keepRecent: s.config.SnapshotKeepRecent, + minTime: s.config.SnapshotMinTimeInterval, + cosmosScheduler: cosmosScheduler, + evmScheduler: evmScheduler, + snapshotSizes: map[int64]int64{}, + } + m.lastRequested = m.newestSnapshotVersion() + m.lastPublished = m.lastRequested + m.lastRequestAt = m.snapshotModTime(m.lastRequested) + m.removeStaleTmpDirs() + m.prune() + if m.lastPublished > 0 { + if err := m.updateCurrentLink(SnapshotDirName(m.lastPublished)); err != nil { + logger.Error("failed to restore state store snapshot current link", + "version", m.lastPublished, "error", err) + } + snapshotMetrics.CurrentHeight.Record(context.Background(), m.lastPublished) + } + s.snapshotMgr = m + logger.Info("state store snapshotting enabled", + "root", root, + "interval", m.interval, + "minTimeInterval", m.minTime, + "keepRecent", m.keepRecent, + ) + return nil +} + +func verifySnapshotHardlinks(root string, sourceDirs []string) error { + if err := os.MkdirAll(root, 0o750); err != nil { + return fmt.Errorf("create snapshot root %q: %w", root, err) + } + for _, sourceDir := range sourceDirs { + probe, err := os.CreateTemp(sourceDir, ".ss-snapshot-link-probe-*") + if err != nil { + return fmt.Errorf("create hardlink probe in state store %q: %w", sourceDir, err) + } + source := probe.Name() + if err := probe.Close(); err != nil { + _ = os.Remove(source) + return fmt.Errorf("close hardlink probe in state store %q: %w", sourceDir, err) + } + target := filepath.Join(root, filepath.Base(source)) + if err := os.Link(source, target); err != nil { + _ = os.Remove(source) + return fmt.Errorf( + "state store %q cannot hardlink snapshots into %q; place all SS databases and the snapshot root on one filesystem: %w", + sourceDir, + root, + err, + ) + } + if err := os.Remove(source); err != nil { + _ = os.Remove(target) + return fmt.Errorf("remove hardlink probe %q: %w", source, err) + } + if err := os.Remove(target); err != nil { + return fmt.Errorf("remove hardlink probe %q: %w", target, err) + } + } + return nil +} + +// stop prevents further snapshots, waits for accepted requests to enqueue their +// barriers, and then waits for active publication. Queued barriers are canceled +// before they start when backend close drains their queues. +func (m *snapshotManager) stop() { + m.mu.Lock() + m.stopped = true + m.mu.Unlock() + m.scheduling.Wait() + m.publishing.Wait() +} + +func (m *snapshotManager) isRunning() bool { + m.mu.Lock() + defer m.mu.Unlock() + return !m.stopped +} + +// maybeSnapshot takes a snapshot when version lands on an interval boundary. +// It is called from the write path for every version, so the common case is the +// modulo test and nothing else. +func (m *snapshotManager) maybeSnapshot(version int64) { + if m == nil || version <= 0 || m.interval <= 0 || version%m.interval != 0 { + return + } + now := time.Now() + m.mu.Lock() + previous := m.lastRequested + previousRequestAt := m.lastRequestAt + var skipReason string + accepted := false + switch { + case m.stopped || version <= m.lastRequested: + // A repeated commit-path call is expected and is not a skipped attempt. + case m.inFlight: + skipReason = "in_flight" + case !m.lastRequestAt.IsZero() && now.Sub(m.lastRequestAt) < m.minTime: + skipReason = "minimum_time_interval" + default: + m.lastRequested = version + m.lastRequestAt = now + m.inFlight = true + m.scheduling.Add(1) + recordSnapshotInFlight(1) + accepted = true + } + m.mu.Unlock() + if !accepted { + if skipReason != "" { + recordSnapshotSkipped(skipReason) + } + return + } + defer m.scheduling.Done() + start := time.Now() + recordSnapshotAttempt() + if err := m.requestSnapshot(version, start); err != nil { + recordSnapshotCompletion(start, "failure") + m.mu.Lock() + if m.lastRequested == version { + m.lastRequested = previous + m.lastRequestAt = previousRequestAt + m.inFlight = false + recordSnapshotInFlight(0) + } + m.mu.Unlock() + logger.Error("state store snapshot failed", "version", version, "error", err) + } +} + +func (m *snapshotManager) finishSnapshot() { + m.mu.Lock() + m.inFlight = false + recordSnapshotInFlight(0) + m.mu.Unlock() +} + +// requestSnapshot asks every backend to checkpoint itself into a staging +// directory and publishes the result once they all have. +// +// The label is exact because of when this runs: the caller has just enqueued +// version on the backends and has not enqueued anything above it, so a barrier +// placed in each apply queue now captures that backend with everything up to +// version applied and nothing after it. The backends reach their barriers +// independently and at different wall-clock times, and the caller waits for +// none of it — enqueueing a barrier costs what enqueueing a changeset costs. +func (m *snapshotManager) requestSnapshot(version int64, start time.Time) error { + name := SnapshotDirName(version) + finalDir := filepath.Join(m.root, name) + if _, err := os.Stat(finalDir); err == nil { + return fmt.Errorf("snapshot dir %q already exists", finalDir) + } else if !os.IsNotExist(err) { + return fmt.Errorf("inspect snapshot dir %q: %w", finalDir, err) + } + tmpDir := filepath.Join(m.root, snapshotTmpPrefix+name) + if err := os.RemoveAll(tmpDir); err != nil { + return fmt.Errorf("clear stale snapshot tmp dir: %w", err) + } + + targets := []checkpointTarget{ + {m.cosmosScheduler, filepath.Join(tmpDir, "cosmos", m.backend)}, + } + if m.evmScheduler != nil { + targets = append(targets, checkpointTarget{ + m.evmScheduler, + filepath.Join(tmpDir, "evm", m.backend), + }) + } + for _, target := range targets { + if err := os.MkdirAll(filepath.Dir(target.dest), 0o750); err != nil { + _ = os.RemoveAll(tmpDir) + return fmt.Errorf("create snapshot dir: %w", err) + } + } + + var ( + mu sync.Mutex + remaining = len(targets) + firstErr error + ) + // Set up before scheduling because callbacks can complete while the loop is + // still scheduling the remaining targets. + for _, target := range targets { + target.store.ScheduleCheckpoint(target.dest, m.isRunning, func(err error) { + mu.Lock() + if err != nil && firstErr == nil { + firstErr = err + } + remaining-- + last, outcome := remaining == 0, firstErr + mu.Unlock() + if !last { + return + } + m.startPublish(version, tmpDir, finalDir, targets, outcome, start) + }) + } + return nil +} + +// startPublish hands a finished set of checkpoints off to a goroutine. It runs +// on whichever backend's apply goroutine finished last, so it must not do the +// work itself: publishing renames directories and prunes old snapshots, and a +// writer stalled on that is a writer not applying blocks. +func (m *snapshotManager) startPublish( + version int64, + tmpDir, finalDir string, + targets []checkpointTarget, + checkpointErr error, + start time.Time, +) { + // Taken under the same lock stop uses, so no goroutine is registered after + // stop has started waiting. + m.mu.Lock() + if m.stopped { + m.mu.Unlock() + _ = os.RemoveAll(tmpDir) + recordSnapshotCompletion(start, "canceled") + m.finishSnapshot() + return + } + m.publishing.Add(1) + m.mu.Unlock() + + go func() { + defer m.publishing.Done() + defer m.finishSnapshot() + if checkpointErr != nil { + if errors.Is(checkpointErr, types.ErrCheckpointCanceled) { + recordSnapshotCompletion(start, "canceled") + } else { + recordSnapshotCompletion(start, "failure") + logger.Error("state store snapshot failed", "version", version, "error", checkpointErr) + } + _ = os.RemoveAll(tmpDir) + return + } + for _, target := range targets { + if err := target.store.SetCheckpointVersion(target.dest, version); err != nil { + recordSnapshotCompletion(start, "failure") + logger.Error("failed to set state store snapshot version", + "version", version, "dir", target.dest, "error", err) + _ = os.RemoveAll(tmpDir) + return + } + } + if m.publish(version, tmpDir, finalDir, start) { + recordSnapshotCompletion(start, "success") + } else { + recordSnapshotCompletion(start, "failure") + } + }() +} + +func (m *snapshotManager) publish(version int64, tmpDir, finalDir string, start time.Time) bool { + apparentBytes, sizeErr := snapshotDirApparentBytes(tmpDir) + if sizeErr != nil { + logger.Error("failed to measure state store snapshot", "dir", tmpDir, "error", sizeErr) + } else if err := writeSnapshotSize(tmpDir, apparentBytes); err != nil { + logger.Error("failed to persist state store snapshot size", "dir", tmpDir, "error", err) + sizeErr = err + } + + m.publishMu.Lock() + defer m.publishMu.Unlock() + defer m.prune() + + if err := os.Rename(tmpDir, finalDir); err != nil { + logger.Error("failed to finalize state store snapshot", "version", version, "error", err) + _ = os.RemoveAll(tmpDir) + return false + } + if err := syncDir(m.root); err != nil { + logger.Error("failed to persist state store snapshot publication", + "version", version, "dir", finalDir, "error", err) + return false + } + if sizeErr == nil { + if m.snapshotSizes == nil { + m.snapshotSizes = map[int64]int64{} + } + m.snapshotSizes[version] = apparentBytes + } + logger.Info("state store snapshot created", + "version", version, "dir", finalDir, "took", time.Since(start).String()) + + // Snapshots can finish out of order, so only move the link forward. + if version > m.lastPublished { + if err := m.updateCurrentLink(SnapshotDirName(version)); err != nil { + // The snapshot itself is intact and discoverable by name; only the + // convenience symlink is stale. The link is part of the publication + // contract, so record this attempt as a failure. + logger.Error("failed to update state store snapshot current link", + "version", version, "error", err) + return false + } + m.lastPublished = version + } + snapshotMetrics.CurrentHeight.Record(context.Background(), m.lastPublished) + return true +} + +func (m *snapshotManager) newestSnapshotVersion() int64 { + versions, err := ListSnapshotVersions(m.root) + if err != nil { + logger.Error("failed to list state store snapshots", "error", err) + return 0 + } + if len(versions) == 0 { + return 0 + } + return versions[len(versions)-1] +} + +func (m *snapshotManager) snapshotModTime(version int64) time.Time { + if version <= 0 { + return time.Time{} + } + info, err := os.Stat(filepath.Join(m.root, SnapshotDirName(version))) + if err != nil { + logger.Error("failed to read state store snapshot modification time", + "version", version, "error", err) + return time.Time{} + } + return info.ModTime() +} + +// removeStaleTmpDirs clears staging directories left behind by a crash or a +// shutdown that landed mid-snapshot. They are named after the snapshot they +// were staging, so they would otherwise sit there until that exact boundary +// came round again. +func (m *snapshotManager) removeStaleTmpDirs() { + tmpLink := filepath.Join(m.root, snapshotCurrentTmpLink) + if err := os.Remove(tmpLink); err != nil && !os.IsNotExist(err) { + logger.Error("failed to remove stale state store snapshot link", "path", tmpLink, "error", err) + } + + entries, err := os.ReadDir(m.root) + if err != nil { + if !os.IsNotExist(err) { + logger.Error("failed to scan state store snapshots dir", "error", err) + } + return + } + for _, entry := range entries { + if !entry.IsDir() || !strings.HasPrefix(entry.Name(), snapshotTmpPrefix) { + continue + } + dir := filepath.Join(m.root, entry.Name()) + if err := os.RemoveAll(dir); err != nil { + logger.Error("failed to remove stale snapshot tmp dir", "dir", dir, "error", err) + continue + } + logger.Info("removed stale state store snapshot tmp dir", "dir", dir) + } +} + +// updateCurrentLink atomically points the current symlink at name. +func (m *snapshotManager) updateCurrentLink(name string) error { + tmpLink := filepath.Join(m.root, snapshotCurrentTmpLink) + _ = os.Remove(tmpLink) + if err := os.Symlink(name, tmpLink); err != nil { + return fmt.Errorf("create snapshot current symlink: %w", err) + } + if err := os.Rename(tmpLink, filepath.Join(m.root, snapshotCurrentLink)); err != nil { + return fmt.Errorf("swap snapshot current symlink: %w", err) + } + return syncDir(m.root) +} + +func syncDir(path string) error { + dir, err := os.Open(path) + if err != nil { + return fmt.Errorf("open directory %q for sync: %w", path, err) + } + syncErr := dir.Sync() + closeErr := dir.Close() + if syncErr != nil { + syncErr = fmt.Errorf("sync directory %q: %w", path, syncErr) + } + if closeErr != nil { + closeErr = fmt.Errorf("close directory %q after sync: %w", path, closeErr) + } + return errors.Join(syncErr, closeErr) +} + +// prune removes all but the newest 1+keepRecent snapshots. +func (m *snapshotManager) prune() { + versions, err := ListSnapshotVersions(m.root) + if err != nil { + logger.Error("failed to list state store snapshots for pruning", "error", err) + return + } + defer m.recordRetentionMetrics() + keep := 1 + m.keepRecent + if len(versions) <= keep { + return + } + for _, v := range versions[:len(versions)-keep] { + dir := filepath.Join(m.root, SnapshotDirName(v)) + if err := os.RemoveAll(dir); err != nil { + logger.Error("failed to prune state store snapshot", "dir", dir, "error", err) + continue + } + logger.Info("pruned state store snapshot", "dir", dir) + } +} + +func (m *snapshotManager) recordRetentionMetrics() { + versions, err := ListSnapshotVersions(m.root) + if err != nil { + logger.Error("failed to list state store snapshots for metrics", "error", err) + return + } + snapshotMetrics.RetainedCount.Record(context.Background(), int64(len(versions))) + + if m.snapshotSizes == nil { + m.snapshotSizes = map[int64]int64{} + } + retained := make(map[int64]struct{}, len(versions)) + var apparentBytes int64 + for _, version := range versions { + retained[version] = struct{}{} + if size, ok := m.snapshotSizes[version]; ok { + apparentBytes += size + continue + } + dir := filepath.Join(m.root, SnapshotDirName(version)) + size, err := readSnapshotSize(dir) + if err != nil { + if !os.IsNotExist(err) { + logger.Error("failed to read state store snapshot size", "dir", dir, "error", err) + } + continue + } + m.snapshotSizes[version] = size + apparentBytes += size + } + for version := range m.snapshotSizes { + if _, ok := retained[version]; !ok { + delete(m.snapshotSizes, version) + } + } + snapshotMetrics.ApparentBytes.Record(context.Background(), apparentBytes) +} + +func snapshotDirApparentBytes(dir string) (int64, error) { + var apparentBytes int64 + err := filepath.WalkDir(dir, func(_ string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if !entry.Type().IsRegular() { + return nil + } + info, err := entry.Info() + if err != nil { + return err + } + apparentBytes += info.Size() + return nil + }) + return apparentBytes, err +} + +func writeSnapshotSize(dir string, size int64) error { + path := filepath.Join(dir, snapshotSizeFile) + file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) + if err != nil { + return err + } + _, writeErr := fmt.Fprintf(file, "%d\n", size) + syncErr := file.Sync() + closeErr := file.Close() + if err := errors.Join(writeErr, syncErr, closeErr); err != nil { + return err + } + return syncDir(dir) +} + +func readSnapshotSize(dir string) (int64, error) { + data, err := os.ReadFile(filepath.Join(dir, snapshotSizeFile)) + if err != nil { + return 0, err + } + size, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64) + if err != nil { + return 0, fmt.Errorf("parse snapshot size in %q: %w", dir, err) + } + if size < 0 { + return 0, fmt.Errorf("snapshot size in %q must be non-negative", dir) + } + return size, nil +} diff --git a/sei-db/state_db/ss/composite/snapshot_metrics.go b/sei-db/state_db/ss/composite/snapshot_metrics.go new file mode 100644 index 0000000000..d908055f01 --- /dev/null +++ b/sei-db/state_db/ss/composite/snapshot_metrics.go @@ -0,0 +1,94 @@ +package composite + +import ( + "context" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + + commonmetrics "github.com/sei-protocol/sei-chain/sei-db/common/metrics" +) + +var snapshotMeter = otel.Meter("seidb_ss_snapshot") + +var snapshotMetrics = struct { + Attempts metric.Int64Counter + Skipped metric.Int64Counter + Completions metric.Int64Counter + Duration metric.Float64Histogram + InFlight metric.Int64Gauge + CurrentHeight metric.Int64Gauge + RetainedCount metric.Int64Gauge + ApparentBytes metric.Int64Gauge +}{ + Attempts: must(snapshotMeter.Int64Counter( + "ss_snapshot_attempts", + metric.WithDescription("Number of state-store snapshot attempts"), + metric.WithUnit("{count}"), + )), + Skipped: must(snapshotMeter.Int64Counter( + "ss_snapshot_skipped", + metric.WithDescription("Number of state-store snapshot boundaries skipped by a scheduling gate"), + metric.WithUnit("{count}"), + )), + Completions: must(snapshotMeter.Int64Counter( + "ss_snapshot_completions", + metric.WithDescription("Number of completed state-store snapshot attempts"), + metric.WithUnit("{count}"), + )), + Duration: must(snapshotMeter.Float64Histogram( + "ss_snapshot_duration", + metric.WithDescription("Time from a state-store snapshot request to completion"), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(commonmetrics.LongLatencyBuckets...), + )), + InFlight: must(snapshotMeter.Int64Gauge( + "ss_snapshot_in_flight", + metric.WithDescription("Whether one state-store snapshot is currently in flight"), + )), + CurrentHeight: must(snapshotMeter.Int64Gauge( + "ss_snapshot_current_height", + metric.WithDescription("Height of the newest published state-store snapshot"), + )), + RetainedCount: must(snapshotMeter.Int64Gauge( + "ss_snapshot_retained_count", + metric.WithDescription("Number of retained state-store snapshots"), + metric.WithUnit("{count}"), + )), + ApparentBytes: must(snapshotMeter.Int64Gauge( + "ss_snapshot_retained_apparent_bytes", + metric.WithDescription("Apparent bytes referenced by retained state-store snapshots; hardlinks can share physical blocks"), + metric.WithUnit("By"), + )), +} + +func must[V any](instrument V, err error) V { + if err != nil { + panic(err) + } + return instrument +} + +func recordSnapshotAttempt() { + snapshotMetrics.Attempts.Add(context.Background(), 1) +} + +func recordSnapshotSkipped(reason string) { + snapshotMetrics.Skipped.Add( + context.Background(), + 1, + metric.WithAttributes(attribute.String("reason", reason)), + ) +} + +func recordSnapshotInFlight(value int64) { + snapshotMetrics.InFlight.Record(context.Background(), value) +} + +func recordSnapshotCompletion(start time.Time, outcome string) { + attrs := metric.WithAttributes(attribute.String("outcome", outcome)) + snapshotMetrics.Completions.Add(context.Background(), 1, attrs) + snapshotMetrics.Duration.Record(context.Background(), time.Since(start).Seconds(), attrs) +} diff --git a/sei-db/state_db/ss/composite/snapshot_test.go b/sei-db/state_db/ss/composite/snapshot_test.go new file mode 100644 index 0000000000..abddd96878 --- /dev/null +++ b/sei-db/state_db/ss/composite/snapshot_test.go @@ -0,0 +1,777 @@ +package composite + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/sei-protocol/sei-chain/sei-db/config" + "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" + "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/cosmos" + "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/evm" + "github.com/stretchr/testify/require" +) + +type noCheckpointStateStore struct { + types.StateStore +} + +type noBarrierStateStore struct { + types.StateStore +} + +func (*noBarrierStateStore) Checkpoint(string) error { + return nil +} + +func (*noBarrierStateStore) SetCheckpointVersion(string, int64) error { + return nil +} + +type controlledSnapshotScheduler struct { + pending chan func() + entered chan struct{} + checkpointCalls int +} + +func (*controlledSnapshotScheduler) SupportsCheckpoint() bool { + return true +} + +func (s *controlledSnapshotScheduler) ScheduleCheckpoint( + destDir string, + shouldRun func() bool, + done func(error), +) { + if s.entered != nil { + close(s.entered) + } + s.pending <- func() { + if !shouldRun() { + done(types.ErrCheckpointCanceled) + return + } + s.checkpointCalls++ + _ = os.MkdirAll(destDir, 0o750) + done(nil) + } +} + +func (*controlledSnapshotScheduler) SetCheckpointVersion(string, int64) error { + return nil +} + +func bankChangeset(key, value string) []*proto.NamedChangeSet { + return []*proto.NamedChangeSet{ + { + Name: "bank", + Changeset: proto.ChangeSet{ + Pairs: []*proto.KVPair{{Key: []byte(key), Value: []byte(value)}}, + }, + }, + } +} + +// evmStorageKey builds a key in the EVM storage family (0x03 prefix), which +// routes to the storage sub-DB when sub-DBs are separate. +func evmStorageKey() []byte { + return append([]byte{0x03}, make([]byte, 20+32)...) +} + +// setupSnapshotStore opens a store with snapshotting on at a small interval so +// tests can cross boundaries cheaply. It returns the store and its snapshots +// root. +func setupSnapshotStore(t *testing.T, interval int64, keepRecent int, separateEVMSubDBs bool) (*CompositeStateStore, string) { + t.Helper() + dir := t.TempDir() + store, err := NewCompositeStateStore(config.StateStoreConfig{ + Backend: "pebbledb", + AsyncWriteBuffer: 100, + KeepRecent: 100000, + EVMSplit: true, + SeparateEVMSubDBs: separateEVMSubDBs, + EVMDBDirectory: filepath.Join(dir, "evm_ss"), + SnapshotEnable: true, + SnapshotInterval: interval, + SnapshotKeepRecent: keepRecent, + }, dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + require.NotNil(t, store.snapshotMgr) + return store, filepath.Join(dir, "data", "state_store", SnapshotsDirName) +} + +type pendingWaiter interface { + WaitForPendingWrites() +} + +// settle waits until every snapshot requested so far has been published and its +// pruning finished. Snapshot barriers sit in the backends' apply queues, so +// draining those queues is what guarantees the barriers ran. +func settle(t *testing.T, store *CompositeStateStore) { + t.Helper() + if w, ok := store.cosmosStore.(pendingWaiter); ok { + w.WaitForPendingWrites() + } + if w, ok := store.evmStore.(pendingWaiter); ok { + w.WaitForPendingWrites() + } + store.snapshotMgr.publishing.Wait() +} + +func writeBlock(t *testing.T, store *CompositeStateStore, version int64) { + t.Helper() + require.NoError(t, store.ApplyChangesetAsync(version, []*proto.NamedChangeSet{ + { + Name: "bank", + Changeset: proto.ChangeSet{ + Pairs: []*proto.KVPair{{Key: []byte("balance"), Value: []byte{byte(version)}}}, + }, + }, + { + Name: evm.EVMStoreKey, + Changeset: proto.ChangeSet{ + Pairs: []*proto.KVPair{{Key: evmStorageKey(), Value: []byte{byte(version)}}}, + }, + }, + })) +} + +// The snapshot manager keys off the mirrored cadence, so the ss-snapshot-enable +// switch has to reach it as a zero interval and leave no manager running. +func TestSnapshotManagerRespectsSnapshotEnable(t *testing.T) { + for _, tc := range []struct { + name string + enable bool + wantRunning bool + }{ + {name: "enabled", enable: true, wantRunning: true}, + {name: "disabled", enable: false, wantRunning: false}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + ssConfig := config.DefaultStateStoreConfig() + ssConfig.SnapshotEnable = tc.enable + config.AlignSSSnapshotWithSC(config.DefaultStateCommitConfig(), &ssConfig) + + store, err := NewCompositeStateStore(ssConfig, dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + + if tc.wantRunning { + require.NotNil(t, store.snapshotMgr, "explicit opt-in starts snapshotting") + require.Positive(t, ssConfig.SnapshotInterval) + } else { + require.Nil(t, store.snapshotMgr) + require.Zero(t, ssConfig.SnapshotInterval) + } + }) + } +} + +func TestCustomStateStoreDirectoryMovesSnapshotRootBesideDatabase(t *testing.T) { + home := t.TempDir() + customDB := filepath.Join(t.TempDir(), "cosmos-state") + cfg := config.DefaultStateStoreConfig() + cfg.Backend = config.PebbleDBBackend + cfg.DBDirectory = customDB + cfg.SnapshotEnable = true + cfg.SnapshotInterval = 5 + cfg.SnapshotKeepRecent = 1 + + store, err := NewCompositeStateStore(cfg, home) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + + require.Equal(t, customDB+"-"+SnapshotsDirName, store.snapshotMgr.root) +} + +func TestSnapshotHardlinkPreflightCleansProbeFiles(t *testing.T) { + source := t.TempDir() + root := t.TempDir() + require.NoError(t, verifySnapshotHardlinks(root, []string{source})) + + sourceEntries, err := os.ReadDir(source) + require.NoError(t, err) + require.Empty(t, sourceEntries) + rootEntries, err := os.ReadDir(root) + require.NoError(t, err) + require.Empty(t, rootEntries) +} + +func TestSnapshotHardlinkPreflightRejectsCrossFilesystem(t *testing.T) { + root, err := os.MkdirTemp("/dev/shm", "ss-snapshot-test-*") + if err != nil { + t.Skipf("no separate /dev/shm filesystem: %v", err) + } + t.Cleanup(func() { require.NoError(t, os.RemoveAll(root)) }) + + err = verifySnapshotHardlinks(root, []string{t.TempDir()}) + if err == nil { + t.Skip("temporary directory and /dev/shm use the same filesystem") + } + require.ErrorContains(t, err, "cannot hardlink snapshots") +} + +func TestSnapshotManagerRejectsUnsupportedBackend(t *testing.T) { + store := &CompositeStateStore{ + cosmosStore: cosmos.NewCosmosStateStore(&noCheckpointStateStore{}), + config: config.StateStoreConfig{ + Backend: config.RocksDBBackend, + SnapshotInterval: 10, + }, + } + + err := store.startSnapshotManager(t.TempDir(), nil) + require.ErrorContains(t, err, "does not support checkpoints") + require.Nil(t, store.snapshotMgr) +} + +func TestSnapshotManagerRejectsBackendWithoutBarrier(t *testing.T) { + store := &CompositeStateStore{ + cosmosStore: cosmos.NewCosmosStateStore(&noBarrierStateStore{}), + config: config.StateStoreConfig{ + Backend: config.PebbleDBBackend, + SnapshotInterval: 10, + }, + } + + err := store.startSnapshotManager(t.TempDir(), nil) + require.ErrorContains(t, err, "does not support checkpoints") + require.Nil(t, store.snapshotMgr) +} + +func TestSnapshotStopCancelsQueuedCheckpoint(t *testing.T) { + scheduler := &controlledSnapshotScheduler{pending: make(chan func(), 1)} + manager := &snapshotManager{ + root: t.TempDir(), + backend: config.PebbleDBBackend, + interval: 5, + keepRecent: 1, + cosmosScheduler: scheduler, + } + + manager.maybeSnapshot(5) + manager.stop() + (<-scheduler.pending)() + + require.Zero(t, scheduler.checkpointCalls) + versions, err := ListSnapshotVersions(manager.root) + require.NoError(t, err) + require.Empty(t, versions) +} + +func TestSnapshotManagerAllowsOnlyOneInFlightSnapshot(t *testing.T) { + scheduler := &controlledSnapshotScheduler{pending: make(chan func(), 2)} + manager := &snapshotManager{ + root: t.TempDir(), + backend: config.PebbleDBBackend, + interval: 5, + keepRecent: 1, + cosmosScheduler: scheduler, + } + + manager.maybeSnapshot(5) + manager.maybeSnapshot(10) + require.Len(t, scheduler.pending, 1, "a second boundary must not enqueue while one snapshot is active") + + (<-scheduler.pending)() + manager.publishing.Wait() + require.False(t, manager.inFlight) + require.Equal(t, int64(5), manager.lastRequested) +} + +func TestSnapshotManagerAppliesMinimumTimeInterval(t *testing.T) { + scheduler := &controlledSnapshotScheduler{pending: make(chan func(), 2)} + manager := &snapshotManager{ + root: t.TempDir(), + backend: config.PebbleDBBackend, + interval: 5, + minTime: time.Hour, + keepRecent: 1, + cosmosScheduler: scheduler, + } + + manager.maybeSnapshot(5) + (<-scheduler.pending)() + manager.publishing.Wait() + + manager.maybeSnapshot(10) + require.Empty(t, scheduler.pending, "a rapid boundary must be skipped") + + manager.mu.Lock() + manager.lastRequestAt = time.Now().Add(-2 * time.Hour) + manager.mu.Unlock() + manager.maybeSnapshot(10) + require.Len(t, scheduler.pending, 1) + (<-scheduler.pending)() + manager.publishing.Wait() +} + +func TestSnapshotStopWaitsForBarrierScheduling(t *testing.T) { + scheduler := &controlledSnapshotScheduler{ + pending: make(chan func()), + entered: make(chan struct{}), + } + manager := &snapshotManager{ + root: t.TempDir(), + backend: config.PebbleDBBackend, + interval: 5, + keepRecent: 1, + cosmosScheduler: scheduler, + } + + requestDone := make(chan struct{}) + go func() { + manager.maybeSnapshot(5) + close(requestDone) + }() + <-scheduler.entered + + stopDone := make(chan struct{}) + go func() { + manager.stop() + close(stopDone) + }() + require.Never(t, func() bool { + select { + case <-stopDone: + return true + default: + return false + } + }, 50*time.Millisecond, 5*time.Millisecond) + + callback := <-scheduler.pending + <-requestDone + <-stopDone + callback() + require.False(t, manager.inFlight) +} + +// Snapshot labels are the interval boundaries themselves, not whatever version +// the store happened to be at when some background pass noticed. That is the +// property the in-queue barrier buys. It keeps each accepted SS snapshot's +// contents aligned with its own label even when SC independently skips that +// boundary. +func TestSnapshotTakenAtExactIntervalBoundaries(t *testing.T) { + store, root := setupSnapshotStore(t, 5, 5, false) + + for v := int64(1); v <= 12; v++ { + writeBlock(t, store, v) + if v%5 == 0 { + settle(t, store) + } + } + settle(t, store) + + versions, err := ListSnapshotVersions(root) + require.NoError(t, err) + require.Equal(t, []int64{5, 10}, versions, + "snapshots must land on interval boundaries and nowhere else") + + target, err := os.Readlink(filepath.Join(root, snapshotCurrentLink)) + require.NoError(t, err) + require.Equal(t, SnapshotDirName(10), target) + + snapDir := filepath.Join(root, SnapshotDirName(10)) + apparentBytes, err := readSnapshotSize(snapDir) + require.NoError(t, err) + require.Positive(t, apparentBytes) + reopened, err := NewCompositeStateStore(config.StateStoreConfig{ + Backend: config.PebbleDBBackend, + AsyncWriteBuffer: 0, + KeepRecent: 100000, + EVMSplit: true, + DBDirectory: filepath.Join(snapDir, "cosmos", config.PebbleDBBackend), + EVMDBDirectory: filepath.Join(snapDir, "evm", config.PebbleDBBackend), + }, t.TempDir()) + require.NoError(t, err) + defer reopened.Close() + + require.Equal(t, int64(10), reopened.GetLatestVersion()) + cosmosValue, err := reopened.Get("bank", 12, []byte("balance")) + require.NoError(t, err) + require.Equal(t, []byte{10}, cosmosValue, "snapshot 10 must exclude Cosmos writes 11 and 12") + evmValue, err := reopened.Get(evm.EVMStoreKey, 12, evmStorageKey()) + require.NoError(t, err) + require.Equal(t, []byte{10}, evmValue, "snapshot 10 must exclude EVM writes 11 and 12") +} + +func TestSnapshotTakenAtExactIntervalBoundaryWithoutEVMSplit(t *testing.T) { + dir := t.TempDir() + ssConfig := config.DefaultStateStoreConfig() + ssConfig.Backend = config.PebbleDBBackend + ssConfig.AsyncWriteBuffer = 100 + ssConfig.KeepRecent = 100000 + ssConfig.EVMSplit = false + ssConfig.SnapshotEnable = true + ssConfig.SnapshotInterval = 5 + ssConfig.SnapshotKeepRecent = 1 + + store, err := NewCompositeStateStore(ssConfig, dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + require.NotNil(t, store.snapshotMgr) + + for version := int64(1); version <= 5; version++ { + require.NoError(t, store.ApplyChangesetAsync(version, bankChangeset("balance", "value"))) + } + settle(t, store) + + root := filepath.Join(dir, "data", "state_store", SnapshotsDirName) + versions, err := ListSnapshotVersions(root) + require.NoError(t, err) + require.Equal(t, []int64{5}, versions) +} + +// A snapshot must be a complete image of every version at or below its label, +// reopenable as a store in its own right. +func TestSnapshotReopensWithEveryVersionBelowLabel(t *testing.T) { + store, root := setupSnapshotStore(t, 10, 5, false) + + for v := int64(1); v <= 10; v++ { + writeBlock(t, store, v) + } + settle(t, store) + + const label = int64(10) + snapDir := filepath.Join(root, SnapshotDirName(label)) + require.DirExists(t, snapDir) + + reopened, err := NewCompositeStateStore(config.StateStoreConfig{ + Backend: "pebbledb", + AsyncWriteBuffer: 0, + KeepRecent: 100000, + EVMSplit: true, + DBDirectory: filepath.Join(snapDir, "cosmos", "pebbledb"), + EVMDBDirectory: filepath.Join(snapDir, "evm", "pebbledb"), + }, t.TempDir()) + require.NoError(t, err) + defer reopened.Close() + + require.Equal(t, label, reopened.GetLatestVersion(), + "the label is the version the snapshot was requested at") + for v := int64(1); v <= label; v++ { + val, err := reopened.Get("bank", v, []byte("balance")) + require.NoError(t, err) + require.Equal(t, []byte{byte(v)}, val, "cosmos version %d missing from snapshot", v) + val, err = reopened.Get(evm.EVMStoreKey, v, evmStorageKey()) + require.NoError(t, err) + require.Equal(t, []byte{byte(v)}, val, "evm version %d missing from snapshot", v) + } +} + +// The reason the barrier has to be a message in every queue rather than a wait: +// a block that only touches storage keys is enqueued only on the storage sub-DB, +// so the idle sub-DBs never observe that version and no amount of waiting would +// tell them it passed. Every sub-DB must still be captured. +func TestSnapshotCapturesIdleEVMSubDBs(t *testing.T) { + store, root := setupSnapshotStore(t, 5, 5, true) + + // Storage keys only: codehash, code and misc sub-DBs stay idle throughout. + for v := int64(1); v <= 5; v++ { + require.NoError(t, store.ApplyChangesetAsync(v, []*proto.NamedChangeSet{ + { + Name: evm.EVMStoreKey, + Changeset: proto.ChangeSet{ + Pairs: []*proto.KVPair{{Key: evmStorageKey(), Value: []byte{byte(v)}}}, + }, + }, + })) + } + settle(t, store) + + evmRoot := filepath.Join(root, SnapshotDirName(5), "evm", "pebbledb") + for _, storeType := range evm.AllEVMStoreTypes() { + name := evm.StoreTypeName(storeType) + subDir := filepath.Join(evmRoot, name) + require.DirExists(t, subDir, "sub-DB %s missing from snapshot", name) + // A checkpoint always carries a manifest. An empty directory would mean + // the barrier never reached that sub-DB. + manifests, err := filepath.Glob(filepath.Join(subDir, "MANIFEST-*")) + require.NoError(t, err) + require.NotEmpty(t, manifests, "sub-DB %s was not checkpointed", name) + } + + // The storage sub-DB is the one that actually took writes, and it must be + // readable at every version up to the label. + storage, err := NewCompositeStateStore(config.StateStoreConfig{ + Backend: "pebbledb", + AsyncWriteBuffer: 0, + KeepRecent: 100000, + // EVM sub-DBs are opened with the plain byte comparer. + UseDefaultComparer: true, + DBDirectory: filepath.Join(evmRoot, evm.StoreTypeName(evm.StoreStorage)), + }, t.TempDir()) + require.NoError(t, err) + defer storage.Close() + + for v := int64(1); v <= 5; v++ { + val, err := storage.Get(evm.EVMStoreKey, v, evmStorageKey()) + require.NoError(t, err) + require.Equal(t, []byte{byte(v)}, val, "evm storage version %d missing from snapshot", v) + } +} + +// An interval boundary that happens to be an empty block arrives through +// SetLatestVersion rather than the changeset path, and must still snapshot — +// otherwise a quiet chain skips whole intervals. +func TestSnapshotTakenOnEmptyBoundaryBlock(t *testing.T) { + store, root := setupSnapshotStore(t, 5, 5, false) + + for v := int64(1); v <= 4; v++ { + writeBlock(t, store, v) + } + // Block 5 is empty: marker only, nothing enqueued. + require.NoError(t, store.SetLatestVersion(5)) + store.ScheduleSnapshot(5) + settle(t, store) + + versions, err := ListSnapshotVersions(root) + require.NoError(t, err) + require.Equal(t, []int64{5}, versions) + + // Every data version below the label is inside the snapshot, and the + // checkpoint marker advances to the empty block's version. + snapDir := filepath.Join(root, SnapshotDirName(5)) + reopened, err := NewCompositeStateStore(config.StateStoreConfig{ + Backend: "pebbledb", + AsyncWriteBuffer: 0, + KeepRecent: 100000, + DBDirectory: filepath.Join(snapDir, "cosmos", "pebbledb"), + }, t.TempDir()) + require.NoError(t, err) + defer reopened.Close() + + require.Equal(t, int64(5), reopened.GetLatestVersion()) + val, err := reopened.Get("bank", 4, []byte("balance")) + require.NoError(t, err) + require.Equal(t, []byte{4}, val) +} + +func TestSetLatestVersionDoesNotSnapshotDuringImport(t *testing.T) { + store, root := setupSnapshotStore(t, 5, 5, false) + nodes := make(chan types.SnapshotNode) + importDone := make(chan error, 1) + go func() { + importDone <- store.Import(5, nodes) + }() + closed := false + t.Cleanup(func() { + if !closed { + close(nodes) + <-importDone + } + }) + + nodes <- types.SnapshotNode{StoreKey: "bank", Key: []byte("balance"), Value: []byte{5}} + require.NoError(t, store.SetLatestVersion(5)) + settle(t, store) + + versions, err := ListSnapshotVersions(root) + require.NoError(t, err) + require.Empty(t, versions, "direct restore metadata writes must not trigger a snapshot") + + close(nodes) + closed = true + require.NoError(t, <-importDone) +} + +// TestSnapshotPrune verifies retention: with keepRecent=1, only the newest two +// snapshots survive and current tracks the newest. +func TestSnapshotPrune(t *testing.T) { + store, root := setupSnapshotStore(t, 5, 1, false) + + for v := int64(1); v <= 15; v++ { + writeBlock(t, store, v) + if v%5 == 0 { + settle(t, store) + } + } + settle(t, store) + + versions, err := ListSnapshotVersions(root) + require.NoError(t, err) + require.Equal(t, []int64{10, 15}, versions) + + target, err := os.Readlink(filepath.Join(root, snapshotCurrentLink)) + require.NoError(t, err) + require.Equal(t, SnapshotDirName(15), target) +} + +func TestSnapshotManagerResumesFromNewestSnapshot(t *testing.T) { + dir := t.TempDir() + cfg := config.DefaultStateStoreConfig() + cfg.Backend = config.PebbleDBBackend + cfg.AsyncWriteBuffer = 100 + cfg.KeepRecent = 100000 + cfg.SnapshotEnable = true + cfg.SnapshotInterval = 5 + cfg.SnapshotKeepRecent = 1 + cfg.SnapshotMinTimeInterval = time.Hour + + store, err := NewCompositeStateStore(cfg, dir) + require.NoError(t, err) + for version := int64(1); version <= 5; version++ { + require.NoError(t, store.ApplyChangesetAsync(version, bankChangeset("balance", "value"))) + } + settle(t, store) + + root := filepath.Join(dir, "data", "state_store", SnapshotsDirName) + snapshotDir := filepath.Join(root, SnapshotDirName(5)) + before, err := os.Stat(snapshotDir) + require.NoError(t, err) + require.NoError(t, store.Close()) + require.NoError(t, os.Remove(filepath.Join(root, snapshotCurrentLink))) + + reopened, err := NewCompositeStateStore(cfg, dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, reopened.Close()) }) + require.Equal(t, int64(5), reopened.snapshotMgr.lastRequested) + require.WithinDuration(t, before.ModTime(), reopened.snapshotMgr.lastRequestAt, time.Second) + target, err := os.Readlink(filepath.Join(root, snapshotCurrentLink)) + require.NoError(t, err) + require.Equal(t, SnapshotDirName(5), target) + + reopened.ScheduleSnapshot(5) + settle(t, reopened) + after, err := os.Stat(snapshotDir) + require.NoError(t, err) + require.True(t, os.SameFile(before, after), "restart must not replace an existing boundary snapshot") + + for version := int64(6); version <= 10; version++ { + require.NoError(t, reopened.ApplyChangesetAsync(version, bankChangeset("balance", "value"))) + } + settle(t, reopened) + versions, err := ListSnapshotVersions(root) + require.NoError(t, err) + require.Equal(t, []int64{5}, versions, "restart must preserve the minimum-time gate") +} + +func TestOutOfOrderPublishDoesNotMoveCurrentBackward(t *testing.T) { + root := t.TempDir() + manager := &snapshotManager{root: root, keepRecent: 5} + + publish := func(version int64) { + tmpDir := filepath.Join(root, snapshotTmpPrefix+SnapshotDirName(version)) + require.NoError(t, os.MkdirAll(tmpDir, 0o750)) + manager.publish(version, tmpDir, filepath.Join(root, SnapshotDirName(version)), time.Now()) + } + publish(10) + publish(5) + + target, err := os.Readlink(filepath.Join(root, snapshotCurrentLink)) + require.NoError(t, err) + require.Equal(t, SnapshotDirName(10), target) +} + +func TestSnapshotManagerPrunesExistingSnapshotsAtStartup(t *testing.T) { + dir := t.TempDir() + root := filepath.Join(dir, "data", "state_store", SnapshotsDirName) + for _, version := range []int64{5, 10, 15} { + require.NoError(t, os.MkdirAll(filepath.Join(root, SnapshotDirName(version)), 0o750)) + } + + cfg := config.DefaultStateStoreConfig() + cfg.Backend = config.PebbleDBBackend + cfg.SnapshotEnable = true + cfg.SnapshotInterval = 5 + cfg.SnapshotKeepRecent = 1 + store, err := NewCompositeStateStore(cfg, dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + + versions, err := ListSnapshotVersions(root) + require.NoError(t, err) + require.Equal(t, []int64{10, 15}, versions) +} + +func TestFailedPublishStillEnforcesRetention(t *testing.T) { + root := t.TempDir() + for _, version := range []int64{5, 10, 15} { + require.NoError(t, os.MkdirAll(filepath.Join(root, SnapshotDirName(version)), 0o750)) + } + manager := &snapshotManager{root: root, keepRecent: 1} + + published := manager.publish( + 20, + filepath.Join(root, "missing-staging-dir"), + filepath.Join(root, SnapshotDirName(20)), + time.Now(), + ) + require.False(t, published) + + versions, err := ListSnapshotVersions(root) + require.NoError(t, err) + require.Equal(t, []int64{10, 15}, versions) +} + +func TestRetentionMetricsCacheSnapshotSizes(t *testing.T) { + root := t.TempDir() + snapshotDir := filepath.Join(root, SnapshotDirName(5)) + require.NoError(t, os.MkdirAll(snapshotDir, 0o750)) + dataFile := filepath.Join(snapshotDir, "data.sst") + require.NoError(t, os.WriteFile(dataFile, []byte("one"), 0o600)) + require.NoError(t, writeSnapshotSize(snapshotDir, 3)) + + manager := &snapshotManager{root: root} + manager.recordRetentionMetrics() + require.Equal(t, int64(3), manager.snapshotSizes[5]) + + // Published snapshots are immutable, so later metric records reuse the + // cached total rather than walking every retained hardlink tree again. + require.NoError(t, os.WriteFile(dataFile, []byte("a longer value"), 0o600)) + manager.recordRetentionMetrics() + require.Equal(t, int64(3), manager.snapshotSizes[5]) + + require.NoError(t, os.RemoveAll(snapshotDir)) + manager.recordRetentionMetrics() + require.NotContains(t, manager.snapshotSizes, int64(5)) +} + +func TestSnapshotRequestReturnsUnexpectedStatError(t *testing.T) { + root := t.TempDir() + name := SnapshotDirName(5) + require.NoError(t, os.Symlink(name, filepath.Join(root, name))) + + manager := &snapshotManager{root: root, backend: config.PebbleDBBackend} + err := manager.requestSnapshot(5, time.Now()) + require.ErrorContains(t, err, "inspect snapshot dir") +} + +// A crash mid-snapshot leaves a staging directory named after the boundary it +// was staging, which would otherwise sit there until that exact boundary came +// round again. +func TestStaleSnapshotTmpDirRemovedAtStartup(t *testing.T) { + dir := t.TempDir() + root := filepath.Join(dir, "data", "state_store", SnapshotsDirName) + stale := filepath.Join(root, snapshotTmpPrefix+SnapshotDirName(40)) + require.NoError(t, os.MkdirAll(filepath.Join(stale, "cosmos"), 0o750)) + tmpLink := filepath.Join(root, snapshotCurrentTmpLink) + require.NoError(t, os.Symlink(filepath.Base(stale), tmpLink)) + + ssConfig := config.DefaultStateStoreConfig() + ssConfig.SnapshotEnable = true + config.AlignSSSnapshotWithSC(config.DefaultStateCommitConfig(), &ssConfig) + store, err := NewCompositeStateStore(ssConfig, dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + + require.NoDirExists(t, stale) + _, err = os.Lstat(tmpLink) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func TestParseSnapshotVersion(t *testing.T) { + v, ok := ParseSnapshotVersion(SnapshotDirName(219140000)) + require.True(t, ok) + require.Equal(t, int64(219140000), v) + + for _, bad := range []string{"snapshot-", "snapshot-123", "current", "tmp-snapshot-00000000000000000010", "snapshot-0000000000000000001x"} { + _, ok := ParseSnapshotVersion(bad) + require.False(t, ok, "expected %q to be rejected", bad) + } +} diff --git a/sei-db/state_db/ss/composite/store.go b/sei-db/state_db/ss/composite/store.go index d88c77647e..1f15cefb59 100644 --- a/sei-db/state_db/ss/composite/store.go +++ b/sei-db/state_db/ss/composite/store.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "fmt" "os" + "path/filepath" "sync" dbm "github.com/tendermint/tm-db" @@ -34,6 +35,7 @@ type CompositeStateStore struct { cosmosStore types.StateStore // CosmosStateStore wrapping MVCC DB evmStore types.StateStore // EVMStateStore wrapping sub MVCC DBs (nil if disabled) pruningManager *pruning.Manager + snapshotMgr *snapshotManager config config.StateStoreConfig closeOnce sync.Once closeErr error @@ -60,6 +62,7 @@ func NewCompositeStateStore( cosmosStore: cosmosStore, config: ssConfig, } + snapshotSourceDirs := []string{dbHome} if ssConfig.EVMSplit { evmDir := ssConfig.EVMDBDirectory @@ -79,6 +82,16 @@ func NewCompositeStateStore( return nil, fmt.Errorf("failed to create EVM store: %w", err) } cs.evmStore = evmStore + if ssConfig.SeparateEVMSubDBs { + for _, storeType := range evm.AllEVMStoreTypes() { + snapshotSourceDirs = append( + snapshotSourceDirs, + filepath.Join(evmDir, evm.StoreTypeName(storeType)), + ) + } + } else { + snapshotSourceDirs = append(snapshotSourceDirs, evmDir) + } logger.Info("EVM state store enabled", "dir", evmDir, "separateDBs", ssConfig.SeparateEVMSubDBs, @@ -103,6 +116,20 @@ func NewCompositeStateStore( return nil, err } + if ssConfig.SnapshotInterval > 0 { + snapshotRoot := utils.GetStateStoreSnapshotsPath(homeDir) + if ssConfig.DBDirectory != "" { + cleanDBHome := filepath.Clean(dbHome) + snapshotRoot = filepath.Join( + filepath.Dir(cleanDBHome), + filepath.Base(cleanDBHome)+"-"+utils.StateStoreSnapshotsDirName, + ) + } + if err := cs.startSnapshotManager(snapshotRoot, snapshotSourceDirs); err != nil { + _ = cs.Close() + return nil, fmt.Errorf("start state store snapshot manager: %w", err) + } + } cs.StartPruning() return cs, nil @@ -221,6 +248,9 @@ func (s *CompositeStateStore) GetEarliestVersion() int64 { func (s *CompositeStateStore) Close() error { s.closeOnce.Do(func() { + if s.snapshotMgr != nil { + s.snapshotMgr.stop() + } if s.pruningManager != nil { s.pruningManager.Stop() } @@ -289,7 +319,11 @@ func (s *CompositeStateStore) ApplyChangesetSync(version int64, changesets []*pr func (s *CompositeStateStore) ApplyChangesetAsync(version int64, changesets []*proto.NamedChangeSet) error { if s.evmStore == nil { - return s.cosmosStore.ApplyChangesetAsync(version, changesets) + if err := s.cosmosStore.ApplyChangesetAsync(version, changesets); err != nil { + return err + } + s.ScheduleSnapshot(version) + return nil } evmChangesets := filterEVMChangesets(changesets) @@ -303,9 +337,17 @@ func (s *CompositeStateStore) ApplyChangesetAsync(version int64, changesets []*p return fmt.Errorf("evm store async enqueue failed: %w", err) } } + s.ScheduleSnapshot(version) return nil } +// ScheduleSnapshot asks the snapshot manager to capture version after the +// block-commit path has enqueued all state changes for that version. Callers +// must not use this hook for direct writes such as import, recovery, or prune. +func (s *CompositeStateStore) ScheduleSnapshot(version int64) { + s.snapshotMgr.maybeSnapshot(version) +} + func filterEVMChangesets(changesets []*proto.NamedChangeSet) []*proto.NamedChangeSet { var evmCS []*proto.NamedChangeSet for _, cs := range changesets { diff --git a/sei-db/state_db/ss/cosmos/store.go b/sei-db/state_db/ss/cosmos/store.go index 5b02d8ed15..02aeaf915c 100644 --- a/sei-db/state_db/ss/cosmos/store.go +++ b/sei-db/state_db/ss/cosmos/store.go @@ -76,3 +76,24 @@ func (s *CosmosStateStore) Import(version int64, ch <-chan types.SnapshotNode) e func (s *CosmosStateStore) Close() error { return s.db.Close() } + +func (s *CosmosStateStore) SupportsCheckpoint() bool { + _, checkpointable := s.db.(types.Checkpointable) + _, barrier := s.db.(types.DrainBarrier) + _, versionSetter := s.db.(types.CheckpointVersionSetter) + return checkpointable && barrier && versionSetter +} + +func (s *CosmosStateStore) ScheduleCheckpoint(destDir string, shouldRun func() bool, done func(error)) { + types.ScheduleCheckpoint(s.db, destDir, shouldRun, done) +} + +func (s *CosmosStateStore) SetCheckpointVersion(destDir string, version int64) error { + return types.SetCheckpointVersion(s.db, destDir, version) +} + +func (s *CosmosStateStore) WaitForPendingWrites() { + if w, ok := s.db.(interface{ WaitForPendingWrites() }); ok { + w.WaitForPendingWrites() + } +} diff --git a/sei-db/state_db/ss/evm/store.go b/sei-db/state_db/ss/evm/store.go index 01337940f3..a1d94146ff 100644 --- a/sei-db/state_db/ss/evm/store.go +++ b/sei-db/state_db/ss/evm/store.go @@ -2,6 +2,7 @@ package evm import ( "fmt" + "os" "path/filepath" "sync" @@ -370,6 +371,89 @@ func (s *EVMStateStore) Close() error { return lastErr } +func (s *EVMStateStore) SupportsCheckpoint() bool { + for _, db := range s.managedDBs { + if _, checkpointable := db.(types.Checkpointable); !checkpointable { + return false + } + if _, barrier := db.(types.DrainBarrier); !barrier { + return false + } + if _, versionSetter := db.(types.CheckpointVersionSetter); !versionSetter { + return false + } + } + return len(s.managedDBs) > 0 +} + +// ScheduleCheckpoint places one barrier on each managed apply queue. A sub-DB +// that did not receive a change at the target block stays at its last version, +// which is the correct state for that block. +func (s *EVMStateStore) ScheduleCheckpoint(destDir string, shouldRun func() bool, done func(error)) { + if !s.separateDBs { + db := s.primaryDB() + if db == nil { + done(nil) + return + } + types.ScheduleCheckpoint(db, destDir, shouldRun, done) + return + } + + if err := os.MkdirAll(destDir, 0o750); err != nil { + done(fmt.Errorf("create EVM checkpoint dir %q: %w", destDir, err)) + return + } + + storeTypes := AllEVMStoreTypes() + var ( + mu sync.Mutex + remaining = len(storeTypes) + firstErr error + ) + for _, storeType := range storeTypes { + name := StoreTypeName(storeType) + dest := filepath.Join(destDir, name) + types.ScheduleCheckpoint(s.subDBs[storeType], dest, shouldRun, func(err error) { + mu.Lock() + if err != nil && firstErr == nil { + firstErr = fmt.Errorf("checkpoint EVM sub-DB %s: %w", name, err) + } + remaining-- + last, outcome := remaining == 0, firstErr + mu.Unlock() + if last { + done(outcome) + } + }) + } +} + +func (s *EVMStateStore) SetCheckpointVersion(destDir string, version int64) error { + if !s.separateDBs { + db := s.primaryDB() + if db == nil { + return nil + } + return types.SetCheckpointVersion(db, destDir, version) + } + for _, storeType := range AllEVMStoreTypes() { + dest := filepath.Join(destDir, StoreTypeName(storeType)) + if err := types.SetCheckpointVersion(s.subDBs[storeType], dest, version); err != nil { + return fmt.Errorf("set EVM sub-DB %s checkpoint version: %w", StoreTypeName(storeType), err) + } + } + return nil +} + +func (s *EVMStateStore) WaitForPendingWrites() { + for _, db := range s.managedDBs { + if w, ok := db.(interface{ WaitForPendingWrites() }); ok { + w.WaitForPendingWrites() + } + } +} + func filterEVMChangesets(changesets []*proto.NamedChangeSet) []*proto.NamedChangeSet { filtered := make([]*proto.NamedChangeSet, 0, len(changesets)) for _, cs := range changesets { From 23e9796ff452103f2ca4c257b2b45892cac4c15c Mon Sep 17 00:00:00 2001 From: blindchaser Date: Mon, 10 Aug 2026 16:21:02 -0400 Subject: [PATCH 02/13] fix(seidb): mark managed snapshot paths as trusted Co-authored-by: Cursor --- sei-db/state_db/ss/composite/snapshot.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sei-db/state_db/ss/composite/snapshot.go b/sei-db/state_db/ss/composite/snapshot.go index 8a5487087e..fdc63dcd66 100644 --- a/sei-db/state_db/ss/composite/snapshot.go +++ b/sei-db/state_db/ss/composite/snapshot.go @@ -547,6 +547,7 @@ func (m *snapshotManager) updateCurrentLink(name string) error { } func syncDir(path string) error { + // #nosec G304 -- path is an internal database or snapshot directory, not request input. dir, err := os.Open(path) if err != nil { return fmt.Errorf("open directory %q for sync: %w", path, err) @@ -643,6 +644,7 @@ func snapshotDirApparentBytes(dir string) (int64, error) { func writeSnapshotSize(dir string, size int64) error { path := filepath.Join(dir, snapshotSizeFile) + // #nosec G304 -- dir is a managed snapshot directory and the file name is fixed. file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) if err != nil { return err @@ -657,6 +659,7 @@ func writeSnapshotSize(dir string, size int64) error { } func readSnapshotSize(dir string) (int64, error) { + // #nosec G304 -- dir is a managed snapshot directory and the file name is fixed. data, err := os.ReadFile(filepath.Join(dir, snapshotSizeFile)) if err != nil { return 0, err From 48f7ea825e5b9dc44a98afd48a282f139db5d150 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Mon, 10 Aug 2026 16:49:25 -0400 Subject: [PATCH 03/13] docs(config): restore the state-store read-site invariants The snapshot rows widened the exemption list and dropped the record of why KeepLastVersion and UseDefaultComparer are unreachable. Relocate that prose and state the derived cadence fields as a separate, differently-caused case. Co-authored-by: Cursor --- app/config_fuzz_test.go | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/app/config_fuzz_test.go b/app/config_fuzz_test.go index 5da665231f..1b8c7617ce 100644 --- a/app/config_fuzz_test.go +++ b/app/config_fuzz_test.go @@ -23,9 +23,12 @@ import ( // // - parseSCConfigs guards almost every read with `if v := opts.Get(k); v != nil`, // so a key absent from an older app.toml keeps its non-zero in-code default. -// - parseSSConfigs leaves most reads unguarded, so an absent key resolves to -// the zero value and overwrites the default. SnapshotEnable is deliberately -// guarded so older app.toml files inherit the new default. +// - parseSSConfigs leaves every legacy read unguarded. Each is a bare cast of a +// possibly-nil value, so an absent key resolves to the zero value and overwrites +// the default. ss-keep-recent becomes 0 (keep everything, unbounded disk growth), +// ss-async-write-buffer becomes 0 (synchronous writes), ss-backend becomes "" +// and ss-enable becomes false. ss-snapshot-enable is the one guarded read, so an +// app.toml written before SS snapshots existed keeps the in-code default. // // Neither reader returns an error, so nothing about the second case is visible at // boot. It is recorded here as behavior rather than reported as a defect: the @@ -520,9 +523,11 @@ func TestParseSCConfigsAbsentBaseline(t *testing.T) { } } -// TestParseSSConfigsAbsentBaselineIsZeroClobbered records the legacy clobber: -// every unguarded operator-visible knob resolves to zero, while guarded fields -// such as SnapshotEnable retain their in-code default. +// TestParseSSConfigsAbsentBaselineIsZeroClobbered records the clobber in full: an +// app.toml with no [state-store] section resolves to a config in which every +// unguarded operator-visible knob has been overwritten with a zero value, including +// the two that change the node's disk behavior without any log line. SnapshotEnable +// is the one field that survives, because its read is guarded. func TestParseSSConfigsAbsentBaselineIsZeroClobbered(t *testing.T) { got := parseSSConfigs(configtest.AppOpts{}) @@ -666,10 +671,18 @@ func TestDefaultsMatchTheRecordedValues(t *testing.T) { func TestManifestNamesEveryField(t *testing.T) { t.Run("state-store", func(t *testing.T) { configtest.CheckManifestCoversEveryField(t, "state-store", config.DefaultStateStoreConfig(), ssKeys, - // These fields have no independent [state-store] key. The first two - // retain their in-code defaults; snapshot cadence is derived from SC. + // Both are tagged mapstructure but no [state-store] key reaches either: parseSSConfigs + // reads neither, so both hold their in-code defaults on every node. pebbledb consumes + // them at construction (KeepLastVersion in mvcc pruning, UseDefaultComparer in the + // comparer selection), which is worth stating rather than omitting — a field a config + // struct carries that configuration cannot address is exactly what a replacement + // manager would otherwise try to map a key onto. "KeepLastVersion", "UseDefaultComparer", + // The three below are unreachable for a different reason, and the distinction is the + // point: they are tagged mapstructure:"-" so no key can bind them even in principle, + // and AlignSSSnapshotWithSC derives all three at runtime from the state-commit cadence. + // ss-snapshot-enable is the only SS-side knob, and it has a row of its own above. "SnapshotInterval", "SnapshotKeepRecent", "SnapshotMinTimeInterval", From b3b7ed4e8183822fe02f4cf9d1f7037397b0fab3 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Mon, 10 Aug 2026 22:28:49 -0400 Subject: [PATCH 04/13] fix(seidb): give the SS snapshot trigger one owner ApplyChangesetAsync is a general StateStore method with callers outside the commit path, so triggering from there handed a snapshot trigger to anyone who wrote a changeset. rootmulti.flush already sees both the populated and the empty block, so it takes the trigger for both and the store method documents a contract it enforces. The capability is now resolved once at startup, pinned by a compile-time assertion and logged if a future wrapper loses the method, rather than re-asserted per block where a failure silently skipped the boundary. Also make the EVM no-managed-DB branches errors instead of reporting a checkpoint that did not happen, and record three contracts that were only discoverable by reading the code: a boundary lost past the barrier is not retried, snapshots do not survive an SS rollback, and enabling them on a non-pebbledb backend fails startup. Co-authored-by: Cursor --- sei-cosmos/storev2/rootmulti/store.go | 32 +++++++- sei-cosmos/storev2/rootmulti/store_test.go | 51 +++++++++++++ sei-db/config/toml.go | 16 ++-- sei-db/config/toml_test.go | 1 + sei-db/state_db/ss/composite/snapshot.go | 24 +++++- sei-db/state_db/ss/composite/snapshot_test.go | 76 +++++++++++++++++-- sei-db/state_db/ss/composite/store.go | 19 ++--- sei-db/state_db/ss/evm/store.go | 8 +- 8 files changed, 197 insertions(+), 30 deletions(-) diff --git a/sei-cosmos/storev2/rootmulti/store.go b/sei-cosmos/storev2/rootmulti/store.go index 23ddf5ab9c..2f04ff75f6 100644 --- a/sei-cosmos/storev2/rootmulti/store.go +++ b/sei-cosmos/storev2/rootmulti/store.go @@ -37,6 +37,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashlog" sctypes "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/types" "github.com/sei-protocol/sei-chain/sei-db/state_db/ss" + sscomposite "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/composite" abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" dbm "github.com/tendermint/tm-db" ) @@ -48,14 +49,24 @@ var ( _ types.Queryable = (*Store)(nil) ) +// stateStoreSnapshotScheduler is the commit path's half of the SS snapshot +// contract: flush tells the state store which version it has just finished +// enqueueing, and the store decides whether that version is a boundary. type stateStoreSnapshotScheduler interface { ScheduleSnapshot(version int64) } +// ss.NewStateStore returns the interface, so the capability is resolved by type +// assertion at startup. This pins the only implementation, so wrapping the state +// store without carrying the method through fails the build here rather than +// silently ending SS snapshots at runtime. +var _ stateStoreSnapshotScheduler = (*sscomposite.CompositeStateStore)(nil) + type Store struct { mtx sync.RWMutex scStore sctypes.Committer ssStore seidbtypes.StateStore + ssSnapshots stateStoreSnapshotScheduler lastCommitInfo *types.CommitInfo storesParams map[types.StoreKey]storeParams storeKeys map[string]types.StoreKey @@ -155,6 +166,16 @@ func NewStore( panic("Enabling SS store without state sync could cause data corruption") } store.ssStore = ssStore + scheduler, ok := ssStore.(stateStoreSnapshotScheduler) + if !ok { + // Unreachable while CompositeStateStore is the only implementation, + // which the assertion above pins. Log rather than drop silently, so + // a wrapper that loses the method is visible as a boot line instead + // of as snapshots that never appear. + logger.Error("state store does not schedule snapshots; SS snapshots are disabled", + "type", fmt.Sprintf("%T", ssStore)) + } + store.ssSnapshots = scheduler } return store @@ -255,14 +276,19 @@ func (rs *Store) flush() error { if err := rs.ssStore.SetLatestVersion(currentVersion); err != nil { panic(err) } - if scheduler, ok := rs.ssStore.(stateStoreSnapshotScheduler); ok { - scheduler.ScheduleSnapshot(currentVersion) - } storev2Metrics.ssVersion.Record(context.Background(), currentVersion) // TODO(PLT-353): remove once storev2_ss_version verified telemetry.SetGauge(float32(currentVersion), "storeV2", "ss", "version") } } + // Both branches above have finished handing currentVersion to SS and have + // enqueued nothing above it, which is what makes an SS snapshot label exact. + // Triggering here rather than inside either branch keeps populated and empty + // blocks on one path. A repeat within the same block (flush runs twice, and + // the second pass sees an empty changeset) is ignored by the state store. + if rs.ssSnapshots != nil { + rs.ssSnapshots.ScheduleSnapshot(currentVersion) + } return rs.scStore.ApplyChangeSets(changeSets) } diff --git a/sei-cosmos/storev2/rootmulti/store_test.go b/sei-cosmos/storev2/rootmulti/store_test.go index 90b7aa98c9..90c298eb97 100644 --- a/sei-cosmos/storev2/rootmulti/store_test.go +++ b/sei-cosmos/storev2/rootmulti/store_test.go @@ -3,6 +3,7 @@ package rootmulti import ( "context" "fmt" + "path/filepath" "sync" "testing" "time" @@ -11,6 +12,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-cosmos/store/types" "github.com/sei-protocol/sei-chain/sei-cosmos/storev2/state" "github.com/sei-protocol/sei-chain/sei-db/config" + sscomposite "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/composite" abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types" "github.com/stretchr/testify/require" "golang.org/x/time/rate" @@ -135,6 +137,55 @@ func TestSCSS_WriteAndHistoricalRead(t *testing.T) { require.Equal(t, valV1, resp.Value) } +// flush owns the SS snapshot trigger for every block, so a boundary must be +// scheduled whether or not the block carried changesets. The composite package +// cannot pin this: its tests call ScheduleSnapshot themselves, so a regression +// in either branch of flush is invisible there. +func TestFlushSchedulesSSSnapshotAtABoundary(t *testing.T) { + for _, tc := range []struct { + name string + writeAtBlock int64 + }{ + {name: "boundary block is populated", writeAtBlock: 2}, + {name: "boundary block is empty", writeAtBlock: 1}, + } { + t.Run(tc.name, func(t *testing.T) { + home := t.TempDir() + scCfg := config.DefaultStateCommitConfig() + scCfg.Enable = true + scCfg.MemIAVLConfig.AsyncCommitBuffer = 0 + // SS mirrors the SC cadence, so this is what puts the SS boundary at 2. + scCfg.MemIAVLConfig.SnapshotInterval = 2 + scCfg.MemIAVLConfig.SnapshotKeepRecent = 1 + + ssCfg := config.DefaultStateStoreConfig() + ssCfg.Enable = true + ssCfg.SnapshotEnable = true + + store := NewStore(home, scCfg, ssCfg, []string{}) + defer func() { _ = store.Close() }() + require.NotNil(t, store.ssSnapshots, "SS snapshot capability was not resolved") + + key := types.NewKVStoreKey("bank") + store.MountStoreWithDB(key, types.StoreTypeIAVL, nil) + require.NoError(t, store.LoadLatestVersion()) + + for block := int64(1); block <= 2; block++ { + if block == tc.writeAtBlock { + store.GetStoreByName("bank").(types.KVStore).Set([]byte("k"), []byte("v")) + } + require.Equal(t, block, store.Commit(true).Version) + } + + root := filepath.Join(home, "data", "state_store", sscomposite.SnapshotsDirName) + require.Eventually(t, func() bool { + versions, err := sscomposite.ListSnapshotVersions(root) + return err == nil && len(versions) == 1 && versions[0] == 2 + }, 10*time.Second, 20*time.Millisecond, "boundary did not produce an SS snapshot") + }) + } +} + // TestCacheMultiStoreWithVersion_OnlyUsesSSStores verifies that CacheMultiStoreWithVersion // serves SS stores when enabled, and falls back to SC when SS is disabled, for // height=0 (latest) and explicit latest height. diff --git a/sei-db/config/toml.go b/sei-db/config/toml.go index 0fc2c459e1..9966cfc120 100644 --- a/sei-db/config/toml.go +++ b/sei-db/config/toml.go @@ -144,13 +144,15 @@ ss-enable-read-write-metrics = {{ .StateStore.EnableReadWriteMetrics }} # snapshots. The cadence is not configurable here: it mirrors the state-commit # snapshot interval, minimum time interval, and retention settings. SC and SS # apply their in-flight gates independently, so a skipped boundary can differ. -# Snapshots are PebbleDB checkpoints, i.e. hardlink trees. Creating one blocks -# each backend's SS apply worker for the full WAL flush, filesystem sync, and -# checkpoint operation; a full async queue then applies write backpressure. It -# does not copy data up front. Startup rejects snapshot configurations where a -# live SS database and the snapshot root cannot hardlink to each other. Each -# enabled Cosmos and EVM SS database must therefore use the same filesystem. A -# custom Cosmos SS directory moves the snapshot root beside that directory. +# Snapshots are PebbleDB checkpoints, i.e. hardlink trees, so ss-backend must be +# pebbledb. Enabling this on any other backend fails startup instead of running +# without snapshots. Creating one blocks each backend's SS apply worker for the +# full WAL flush, filesystem sync, and checkpoint operation; a full async queue +# then applies write backpressure. It does not copy data up front. Startup also +# rejects snapshot configurations where a live SS database and the snapshot root +# cannot hardlink to each other. Each enabled Cosmos and EVM SS database must +# therefore use the same filesystem. A custom Cosmos SS directory moves the +# snapshot root beside that directory. # Retained snapshots pin referenced SSTs, so compaction cannot reclaim them. # Expect steady-state disk overhead on the order of the compaction churn over # one snapshot interval per retained snapshot, which is substantial on a diff --git a/sei-db/config/toml_test.go b/sei-db/config/toml_test.go index deee57a2ea..8d264fa44c 100644 --- a/sei-db/config/toml_test.go +++ b/sei-db/config/toml_test.go @@ -96,6 +96,7 @@ func TestStateStoreConfigTemplate(t *testing.T) { require.Contains(t, output, "ss-prune-interval =", "Missing ss-prune-interval") require.Contains(t, output, "ss-import-num-workers =", "Missing ss-import-num-workers") require.Contains(t, output, "ss-enable-read-write-metrics = false", "Missing state-store read/write metrics flag") + require.Contains(t, output, "ss-snapshot-enable = false", "Missing or incorrect ss-snapshot-enable") require.Contains(t, output, `evm-ss-db-directory = ""`, "Missing evm-ss-db-directory") require.Contains(t, output, `evm-ss-split = false`, "Missing or incorrect evm-ss-split") require.Contains(t, output, "evm-ss-separate-dbs = false", "Missing or incorrect evm-ss-separate-dbs") diff --git a/sei-db/state_db/ss/composite/snapshot.go b/sei-db/state_db/ss/composite/snapshot.go index fdc63dcd66..ed43eb9606 100644 --- a/sei-db/state_db/ss/composite/snapshot.go +++ b/sei-db/state_db/ss/composite/snapshot.go @@ -46,8 +46,15 @@ import ( // // The barrier orders only the async block-commit queues. Import, recovery, // pruning, and direct version-marker writes bypass those queues and must not -// call ScheduleSnapshot. The rootmulti commit path owns the trigger, including -// the explicit trigger for an empty block. +// call ScheduleSnapshot. The rootmulti commit path owns the trigger for every +// block, populated or empty, and is the only caller of ScheduleSnapshot. +// +// SS rollback is not part of this feature, and the two do not compose yet. A +// rollback leaves lastRequested at the pre-rollback high-water mark, so the +// re-executed boundaries are read as repeats and skipped, and the already +// published snapshot-NNNNN directories keep labels that belong to the abandoned +// chain. Nothing in the layout tells a consumer of current that this happened, +// so the snapshot root must be cleared by hand after a rollback. // // Managed snapshot directories have no lease. A live consumer must not rely on // a path remaining present across a retention pass. Until a lease API exists, @@ -430,6 +437,19 @@ func (m *snapshotManager) startPublish( }() } +// publish moves a finished checkpoint into place and reports whether the whole +// publication succeeded. Retention runs either way. +// +// A boundary that fails anywhere past the barrier is given up on, and this is +// deliberate. maybeSnapshot restores lastRequested when requestSnapshot fails, +// because that failure happens before any barrier is enqueued and the boundary +// was never claimed. Once the barriers are out, the version they captured is +// the only image of that boundary there will ever be: the write path has moved +// on, so re-running the attempt would checkpoint a later state under the older +// label, which is the one thing the label is supposed to rule out. Recovery is +// therefore the next boundary rather than a retry of this one, at the cost of +// one snapshot interval of coverage. The error log and the outcome="failure" +// counter are the signal. func (m *snapshotManager) publish(version int64, tmpDir, finalDir string, start time.Time) bool { apparentBytes, sizeErr := snapshotDirApparentBytes(tmpDir) if sizeErr != nil { diff --git a/sei-db/state_db/ss/composite/snapshot_test.go b/sei-db/state_db/ss/composite/snapshot_test.go index abddd96878..ac825e91ef 100644 --- a/sei-db/state_db/ss/composite/snapshot_test.go +++ b/sei-db/state_db/ss/composite/snapshot_test.go @@ -121,9 +121,19 @@ func settle(t *testing.T, store *CompositeStateStore) { store.snapshotMgr.publishing.Wait() } +// commitBlock is what rootmulti.flush does for a populated block: enqueue the +// changesets, then hand the version to the snapshot manager. ApplyChangesetAsync +// alone schedules nothing, so tests that expect a snapshot must come through +// here. +func commitBlock(t *testing.T, store *CompositeStateStore, version int64, changesets []*proto.NamedChangeSet) { + t.Helper() + require.NoError(t, store.ApplyChangesetAsync(version, changesets)) + store.ScheduleSnapshot(version) +} + func writeBlock(t *testing.T, store *CompositeStateStore, version int64) { t.Helper() - require.NoError(t, store.ApplyChangesetAsync(version, []*proto.NamedChangeSet{ + commitBlock(t, store, version, []*proto.NamedChangeSet{ { Name: "bank", Changeset: proto.ChangeSet{ @@ -136,7 +146,7 @@ func writeBlock(t *testing.T, store *CompositeStateStore, version int64) { Pairs: []*proto.KVPair{{Key: evmStorageKey(), Value: []byte{byte(version)}}}, }, }, - })) + }) } // The snapshot manager keys off the mirrored cadence, so the ss-snapshot-enable @@ -417,7 +427,7 @@ func TestSnapshotTakenAtExactIntervalBoundaryWithoutEVMSplit(t *testing.T) { require.NotNil(t, store.snapshotMgr) for version := int64(1); version <= 5; version++ { - require.NoError(t, store.ApplyChangesetAsync(version, bankChangeset("balance", "value"))) + commitBlock(t, store, version, bankChangeset("balance", "value")) } settle(t, store) @@ -464,6 +474,58 @@ func TestSnapshotReopensWithEveryVersionBelowLabel(t *testing.T) { } } +// The property the barrier exists for: the label stays exact while the write +// path keeps going. Nothing is drained between block 10 and blocks 11 and 12, so +// the checkpoint runs with later versions already queued behind the barrier — the +// case a post-hoc "snapshot what has been applied" scheme would get wrong. +func TestSnapshotExcludesVersionsWrittenAfterTheBoundary(t *testing.T) { + store, root := setupSnapshotStore(t, 10, 5, false) + + const label = int64(10) + for v := int64(1); v <= label; v++ { + writeBlock(t, store, v) + } + for v := label + 1; v <= label+2; v++ { + writeBlock(t, store, v) + } + settle(t, store) + + snapDir := filepath.Join(root, SnapshotDirName(label)) + require.DirExists(t, snapDir) + + reopened, err := NewCompositeStateStore(config.StateStoreConfig{ + Backend: "pebbledb", + AsyncWriteBuffer: 0, + KeepRecent: 100000, + EVMSplit: true, + DBDirectory: filepath.Join(snapDir, "cosmos", "pebbledb"), + EVMDBDirectory: filepath.Join(snapDir, "evm", "pebbledb"), + }, t.TempDir()) + require.NoError(t, err) + defer reopened.Close() + + require.Equal(t, label, reopened.GetLatestVersion()) + // Reading above the label returns the label's value rather than 11 or 12, + // which is what "excluded" means for an MVCC store: the later writes are not + // in this image at any version. + for _, above := range []int64{label + 1, label + 2} { + val, err := reopened.Get("bank", above, []byte("balance")) + require.NoError(t, err) + require.Equal(t, []byte{byte(label)}, val, + "cosmos read at %d saw a write from after the boundary", above) + val, err = reopened.Get(evm.EVMStoreKey, above, evmStorageKey()) + require.NoError(t, err) + require.Equal(t, []byte{byte(label)}, val, + "evm read at %d saw a write from after the boundary", above) + } + + // The live store keeps them, so the snapshot dropped them rather than the + // writes never landing. + val, err := store.Get("bank", label+2, []byte("balance")) + require.NoError(t, err) + require.Equal(t, []byte{byte(label + 2)}, val) +} + // The reason the barrier has to be a message in every queue rather than a wait: // a block that only touches storage keys is enqueued only on the storage sub-DB, // so the idle sub-DBs never observe that version and no amount of waiting would @@ -473,14 +535,14 @@ func TestSnapshotCapturesIdleEVMSubDBs(t *testing.T) { // Storage keys only: codehash, code and misc sub-DBs stay idle throughout. for v := int64(1); v <= 5; v++ { - require.NoError(t, store.ApplyChangesetAsync(v, []*proto.NamedChangeSet{ + commitBlock(t, store, v, []*proto.NamedChangeSet{ { Name: evm.EVMStoreKey, Changeset: proto.ChangeSet{ Pairs: []*proto.KVPair{{Key: evmStorageKey(), Value: []byte{byte(v)}}}, }, }, - })) + }) } settle(t, store) @@ -616,7 +678,7 @@ func TestSnapshotManagerResumesFromNewestSnapshot(t *testing.T) { store, err := NewCompositeStateStore(cfg, dir) require.NoError(t, err) for version := int64(1); version <= 5; version++ { - require.NoError(t, store.ApplyChangesetAsync(version, bankChangeset("balance", "value"))) + commitBlock(t, store, version, bankChangeset("balance", "value")) } settle(t, store) @@ -643,7 +705,7 @@ func TestSnapshotManagerResumesFromNewestSnapshot(t *testing.T) { require.True(t, os.SameFile(before, after), "restart must not replace an existing boundary snapshot") for version := int64(6); version <= 10; version++ { - require.NoError(t, reopened.ApplyChangesetAsync(version, bankChangeset("balance", "value"))) + commitBlock(t, reopened, version, bankChangeset("balance", "value")) } settle(t, reopened) versions, err := ListSnapshotVersions(root) diff --git a/sei-db/state_db/ss/composite/store.go b/sei-db/state_db/ss/composite/store.go index 1f15cefb59..dc73e5d118 100644 --- a/sei-db/state_db/ss/composite/store.go +++ b/sei-db/state_db/ss/composite/store.go @@ -319,11 +319,7 @@ func (s *CompositeStateStore) ApplyChangesetSync(version int64, changesets []*pr func (s *CompositeStateStore) ApplyChangesetAsync(version int64, changesets []*proto.NamedChangeSet) error { if s.evmStore == nil { - if err := s.cosmosStore.ApplyChangesetAsync(version, changesets); err != nil { - return err - } - s.ScheduleSnapshot(version) - return nil + return s.cosmosStore.ApplyChangesetAsync(version, changesets) } evmChangesets := filterEVMChangesets(changesets) @@ -337,13 +333,18 @@ func (s *CompositeStateStore) ApplyChangesetAsync(version int64, changesets []*p return fmt.Errorf("evm store async enqueue failed: %w", err) } } - s.ScheduleSnapshot(version) return nil } -// ScheduleSnapshot asks the snapshot manager to capture version after the -// block-commit path has enqueued all state changes for that version. Callers -// must not use this hook for direct writes such as import, recovery, or prune. +// ScheduleSnapshot asks the snapshot manager to capture version once the caller +// has enqueued every state change for that version and nothing above it. +// +// This is deliberately not called from ApplyChangesetAsync. That method is part +// of the general StateStore interface and has callers outside the commit path, +// such as the benchmark wrappers, which would inherit a snapshot trigger they +// never asked for. The rootmulti commit path is the single choke point that +// sees both the populated and the empty block, so it owns the trigger. Direct +// writes such as import, recovery, and prune must not use this hook. func (s *CompositeStateStore) ScheduleSnapshot(version int64) { s.snapshotMgr.maybeSnapshot(version) } diff --git a/sei-db/state_db/ss/evm/store.go b/sei-db/state_db/ss/evm/store.go index a1d94146ff..daba3ad631 100644 --- a/sei-db/state_db/ss/evm/store.go +++ b/sei-db/state_db/ss/evm/store.go @@ -1,6 +1,7 @@ package evm import ( + "errors" "fmt" "os" "path/filepath" @@ -393,7 +394,10 @@ func (s *EVMStateStore) ScheduleCheckpoint(destDir string, shouldRun func() bool if !s.separateDBs { db := s.primaryDB() if db == nil { - done(nil) + // Unreachable: NewEVMStateStore either opens a managed DB or fails. + // Reporting success would publish a snapshot with no evm tree in it, + // which is only discovered by whoever tries to restore from it. + done(errors.New("EVM state store has no managed DB to checkpoint")) return } types.ScheduleCheckpoint(db, destDir, shouldRun, done) @@ -433,7 +437,7 @@ func (s *EVMStateStore) SetCheckpointVersion(destDir string, version int64) erro if !s.separateDBs { db := s.primaryDB() if db == nil { - return nil + return errors.New("EVM state store has no managed DB to stamp") } return types.SetCheckpointVersion(db, destDir, version) } From 83885c576d4cfcf7eb2c6ded7912202cb77cef69 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Tue, 11 Aug 2026 15:48:15 -0400 Subject: [PATCH 05/13] docs(seidb): note prune concurrency and reused-home caveats for SS snapshots Co-authored-by: Cursor --- sei-db/state_db/ss/composite/snapshot.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sei-db/state_db/ss/composite/snapshot.go b/sei-db/state_db/ss/composite/snapshot.go index ed43eb9606..dcf8f21607 100644 --- a/sei-db/state_db/ss/composite/snapshot.go +++ b/sei-db/state_db/ss/composite/snapshot.go @@ -48,13 +48,19 @@ import ( // pruning, and direct version-marker writes bypass those queues and must not // call ScheduleSnapshot. The rootmulti commit path owns the trigger for every // block, populated or empty, and is the only caller of ScheduleSnapshot. +// Because pruning bypasses the barrier, a checkpoint can capture a partially +// applied prune — the same state a crash mid-prune leaves on the live DB. +// Reads at the label version are unaffected; only historical reads near the +// pruning horizon can see it. // // SS rollback is not part of this feature, and the two do not compose yet. A // rollback leaves lastRequested at the pre-rollback high-water mark, so the // re-executed boundaries are read as repeats and skipped, and the already // published snapshot-NNNNN directories keep labels that belong to the abandoned // chain. Nothing in the layout tells a consumer of current that this happened, -// so the snapshot root must be cleared by hand after a rollback. +// so the snapshot root must be cleared by hand after a rollback. State-syncing +// to a height below existing snapshots in a reused home directory has the same +// shape and needs the same manual clearing. // // Managed snapshot directories have no lease. A live consumer must not rely on // a path remaining present across a retention pass. Until a lease API exists, From 14a7962b69683e2933ca49945184e8256a574c98 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Tue, 11 Aug 2026 23:48:55 -0400 Subject: [PATCH 06/13] fix(seidb): stamp snapshot version markers instead of inheriting them A snapshot taken while pruning ran was scrap. Pruning bypasses the apply queues, so no barrier orders a snapshot against it, and it advances the databases one at a time: composite.Prune finishes EVM, compaction included, before it starts the Cosmos scan, which is minutes apart on a large store. A snapshot inside that gap captured a pruned EVM tree beside an unpruned Cosmos one, and reopening it failed the earliest-version agreement check in NewCompositeStateStore. Publication now writes both markers into every checkpoint rather than letting each tree keep whatever it captured. The stamped floor is the highest earliest version any tree has reached, including across the EVM sub-DBs, which prune in parallel and finish at different times, so a snapshot never promises a version one of its trees has already dropped. SetCheckpointVersion becomes SetCheckpointMarkers and writes both markers under one open, so a checkpoint is never left describing half a range. Co-authored-by: Cursor --- sei-db/db_engine/pebbledb/mvcc/db.go | 33 +++++--- sei-db/db_engine/pebbledb/mvcc/db_test.go | 22 +++-- sei-db/db_engine/types/types.go | 29 +++++-- sei-db/state_db/ss/composite/snapshot.go | 48 ++++++++++- sei-db/state_db/ss/composite/snapshot_test.go | 82 ++++++++++++++++++- sei-db/state_db/ss/cosmos/store.go | 14 +++- sei-db/state_db/ss/evm/db_test.go | 28 +++++++ sei-db/state_db/ss/evm/store.go | 24 ++++-- 8 files changed, 241 insertions(+), 39 deletions(-) diff --git a/sei-db/db_engine/pebbledb/mvcc/db.go b/sei-db/db_engine/pebbledb/mvcc/db.go index 936bcdd6a1..faedc39c60 100644 --- a/sei-db/db_engine/pebbledb/mvcc/db.go +++ b/sei-db/db_engine/pebbledb/mvcc/db.go @@ -299,33 +299,44 @@ func (db *Database) Checkpoint(destDir string) error { return nil } -// SetCheckpointVersion writes version into a completed checkpoint without -// changing the live database marker. -func (db *Database) SetCheckpointVersion(destDir string, version int64) error { - if version < 0 { - return fmt.Errorf("version must be non-negative") +// SetCheckpointMarkers writes the version range into a completed checkpoint +// without changing the live database markers. Both are written under one open +// so a checkpoint is never left describing half a range. +func (db *Database) SetCheckpointMarkers(destDir string, latest, earliest int64) error { + if latest < 0 || earliest < 0 { + return fmt.Errorf("versions must be non-negative") + } + if earliest > latest { + return fmt.Errorf("earliest version %d is above latest version %d", earliest, latest) } opts := newPebbleOptions(db.config, nil) opts.DisableAutomaticCompactions = true checkpoint, err := pebble.Open(destDir, opts) if err != nil { - return fmt.Errorf("open checkpoint %q to set version: %w", destDir, err) + return fmt.Errorf("open checkpoint %q to set markers: %w", destDir, err) } - var marker [VersionSize]byte - binary.LittleEndian.PutUint64(marker[:], uint64(version)) - setErr := checkpoint.Set([]byte(latestVersionKey), marker[:], pebble.Sync) + setErr := setCheckpointMarker(checkpoint, latestVersionKey, latest) + if setErr == nil { + setErr = setCheckpointMarker(checkpoint, earliestVersionKey, earliest) + } closeErr := checkpoint.Close() if setErr != nil { - setErr = fmt.Errorf("set checkpoint version %d: %w", version, setErr) + setErr = fmt.Errorf("set checkpoint markers latest=%d earliest=%d: %w", latest, earliest, setErr) } if closeErr != nil { - closeErr = fmt.Errorf("close checkpoint after setting version %d: %w", version, closeErr) + closeErr = fmt.Errorf("close checkpoint after setting markers: %w", closeErr) } return errors.Join(setErr, closeErr) } +func setCheckpointMarker(checkpoint *pebble.DB, key string, version int64) error { + var marker [VersionSize]byte + binary.LittleEndian.PutUint64(marker[:], uint64(version)) + return checkpoint.Set([]byte(key), marker[:], pebble.Sync) +} + func (db *Database) SetLatestVersion(version int64) error { if version < 0 { return fmt.Errorf("version must be non-negative") diff --git a/sei-db/db_engine/pebbledb/mvcc/db_test.go b/sei-db/db_engine/pebbledb/mvcc/db_test.go index 18ba53dad7..709d2096e8 100644 --- a/sei-db/db_engine/pebbledb/mvcc/db_test.go +++ b/sei-db/db_engine/pebbledb/mvcc/db_test.go @@ -58,6 +58,7 @@ func TestVersionedCheckpointPreservesFutureLiveMarker(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) }) require.NoError(t, store.SetLatestVersion(10)) + require.NoError(t, store.SetEarliestVersion(4, false)) dest := filepath.Join(t.TempDir(), "snapshot") done := make(chan error, 1) @@ -65,17 +66,28 @@ func TestVersionedCheckpointPreservesFutureLiveMarker(t *testing.T) { done <- err }) require.NoError(t, <-done) - require.NoError(t, types.SetCheckpointVersion(store, dest, 5)) + // The caller decides both markers. Neither has to match what the copy + // inherited: the label comes from the barrier, and the earliest version is + // reconciled across every tree in the snapshot. + require.NoError(t, types.SetCheckpointMarkers(store, dest, 5, 5)) + require.Equal(t, int64(10), store.GetLatestVersion()) - marker, closer, err := store.(*Database).storage.Get([]byte(latestVersionKey)) - require.NoError(t, err) - require.Equal(t, uint64(10), binary.LittleEndian.Uint64(marker)) - require.NoError(t, closer.Close()) + require.Equal(t, int64(4), store.GetEarliestVersion()) + for key, want := range map[string]uint64{latestVersionKey: 10, earliestVersionKey: 4} { + marker, closer, err := store.(*Database).storage.Get([]byte(key)) + require.NoError(t, err) + require.Equal(t, want, binary.LittleEndian.Uint64(marker), "live %s changed", key) + require.NoError(t, closer.Close()) + } checkpoint, err := OpenDB(dest, cfg) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, checkpoint.Close()) }) require.Equal(t, int64(5), checkpoint.GetLatestVersion()) + require.Equal(t, int64(5), checkpoint.GetEarliestVersion()) + + require.Error(t, types.SetCheckpointMarkers(store, dest, 5, 6), + "an earliest version above the label describes an empty range") } func TestScheduledCheckpointCanBeCanceledAtBarrier(t *testing.T) { diff --git a/sei-db/db_engine/types/types.go b/sei-db/db_engine/types/types.go index d490b03b90..58bc64e178 100644 --- a/sei-db/db_engine/types/types.go +++ b/sei-db/db_engine/types/types.go @@ -122,10 +122,16 @@ type Checkpointable interface { Checkpoint(destDir string) error } -// CheckpointVersionSetter writes the logical height into a completed +// CheckpointMarkerSetter writes the logical version range into a completed // checkpoint without changing the live database. -type CheckpointVersionSetter interface { - SetCheckpointVersion(destDir string, version int64) error +// +// Both markers are stamped rather than inherited from the checkpoint. The +// latest marker has to be stamped because a checkpoint is a copy of a database +// whose marker may already have moved on. The earliest marker has to be stamped +// because pruning advances it outside the write queue and one database at a +// time, so two checkpoints of the same instant can disagree about it. +type CheckpointMarkerSetter interface { + SetCheckpointMarkers(destDir string, latest, earliest int64) error } // DrainBarrier is an optional capability for engines that apply changesets from @@ -139,7 +145,12 @@ type DrainBarrier interface { type CheckpointScheduler interface { SupportsCheckpoint() bool ScheduleCheckpoint(destDir string, shouldRun func() bool, done func(error)) - SetCheckpointVersion(destDir string, version int64) error + SetCheckpointMarkers(destDir string, latest, earliest int64) error + // HighestEarliestVersion reports the largest earliest version among the + // databases this store owns. GetEarliestVersion reports the smallest, which + // is the right answer for serving reads but the wrong one for stamping a + // snapshot: it would claim a range a partly pruned database cannot serve. + HighestEarliestVersion() int64 } // ErrCheckpointCanceled reports that a queued checkpoint was canceled before @@ -168,14 +179,14 @@ func ScheduleCheckpoint(db StateStore, destDir string, shouldRun func() bool, do }) } -// SetCheckpointVersion makes a completed checkpoint self-describing without +// SetCheckpointMarkers makes a completed checkpoint self-describing without // changing the live database. -func SetCheckpointVersion(db StateStore, destDir string, version int64) error { - setter, ok := db.(CheckpointVersionSetter) +func SetCheckpointMarkers(db StateStore, destDir string, latest, earliest int64) error { + setter, ok := db.(CheckpointMarkerSetter) if !ok { - return fmt.Errorf("state store backend %T cannot set checkpoint versions", db) + return fmt.Errorf("state store backend %T cannot set checkpoint markers", db) } - return setter.SetCheckpointVersion(destDir, version) + return setter.SetCheckpointMarkers(destDir, latest, earliest) } // --------------------------------------------------------------------------- diff --git a/sei-db/state_db/ss/composite/snapshot.go b/sei-db/state_db/ss/composite/snapshot.go index dcf8f21607..93b7ceb761 100644 --- a/sei-db/state_db/ss/composite/snapshot.go +++ b/sei-db/state_db/ss/composite/snapshot.go @@ -48,6 +48,12 @@ import ( // pruning, and direct version-marker writes bypass those queues and must not // call ScheduleSnapshot. The rootmulti commit path owns the trigger for every // block, populated or empty, and is the only caller of ScheduleSnapshot. +// +// Because pruning is outside the barrier and advances one database at a time, +// the version markers a snapshot publishes are stamped at publication rather +// than inherited from each checkpoint. Otherwise a snapshot taken during a +// prune pass would pair a pruned tree with an unpruned one and would be +// rejected on reopen. See stampedEarliest. // Because pruning bypasses the barrier, a checkpoint can capture a partially // applied prune — the same state a crash mid-prune leaves on the live DB. // Reads at the label version are unaffected; only historical reads near the @@ -426,11 +432,12 @@ func (m *snapshotManager) startPublish( _ = os.RemoveAll(tmpDir) return } + earliest := m.stampedEarliest(version) for _, target := range targets { - if err := target.store.SetCheckpointVersion(target.dest, version); err != nil { + if err := target.store.SetCheckpointMarkers(target.dest, version, earliest); err != nil { recordSnapshotCompletion(start, "failure") - logger.Error("failed to set state store snapshot version", - "version", version, "dir", target.dest, "error", err) + logger.Error("failed to set state store snapshot markers", + "version", version, "earliest", earliest, "dir", target.dest, "error", err) _ = os.RemoveAll(tmpDir) return } @@ -443,6 +450,41 @@ func (m *snapshotManager) startPublish( }() } +// stampedEarliest is the earliest version the published snapshot claims to +// serve. Every tree in one snapshot gets this same value. +// +// A prune pass is the reason it cannot be left to each checkpoint. Pruning +// bypasses the apply queues, so no barrier orders a snapshot against it, and it +// advances the databases one at a time: composite.Prune finishes EVM, including +// its compaction, before it starts the Cosmos scan. On a large store that gap +// is minutes long, and a snapshot taken inside it would capture a Cosmos tree +// that has not been pruned next to an EVM tree that has. Reopening that +// snapshot fails the earliest-version agreement check in NewCompositeStateStore +// and the snapshot is scrap. +// +// The value is the highest earliest version reached anywhere, because a +// snapshot must not promise a version one of its trees has already dropped. +// Reading it here rather than at checkpoint time can only overshoot what a tree +// holds, since an earliest marker never moves backward outside a restore, and +// overshooting is the safe direction: the snapshot serves less than it has +// rather than claiming more. It is clamped to the label so the pair stays +// ordered when retention is short enough for pruning to overtake the boundary. +func (m *snapshotManager) stampedEarliest(label int64) int64 { + earliest := m.cosmosScheduler.HighestEarliestVersion() + if m.evmScheduler != nil { + if evmEarliest := m.evmScheduler.HighestEarliestVersion(); evmEarliest > earliest { + earliest = evmEarliest + } + } + if earliest > label { + return label + } + if earliest < 0 { + return 0 + } + return earliest +} + // publish moves a finished checkpoint into place and reports whether the whole // publication succeeded. Retention runs either way. // diff --git a/sei-db/state_db/ss/composite/snapshot_test.go b/sei-db/state_db/ss/composite/snapshot_test.go index ac825e91ef..3852f53306 100644 --- a/sei-db/state_db/ss/composite/snapshot_test.go +++ b/sei-db/state_db/ss/composite/snapshot_test.go @@ -26,7 +26,7 @@ func (*noBarrierStateStore) Checkpoint(string) error { return nil } -func (*noBarrierStateStore) SetCheckpointVersion(string, int64) error { +func (*noBarrierStateStore) SetCheckpointMarkers(string, int64, int64) error { return nil } @@ -59,10 +59,14 @@ func (s *controlledSnapshotScheduler) ScheduleCheckpoint( } } -func (*controlledSnapshotScheduler) SetCheckpointVersion(string, int64) error { +func (*controlledSnapshotScheduler) SetCheckpointMarkers(string, int64, int64) error { return nil } +func (*controlledSnapshotScheduler) HighestEarliestVersion() int64 { + return 0 +} + func bankChangeset(key, value string) []*proto.NamedChangeSet { return []*proto.NamedChangeSet{ { @@ -526,6 +530,80 @@ func TestSnapshotExcludesVersionsWrittenAfterTheBoundary(t *testing.T) { require.Equal(t, []byte{byte(label + 2)}, val) } +// A prune pass advances each database's earliest marker on its own schedule and +// bypasses the barrier entirely, so the two SS trees disagree for as long as the +// pass runs — minutes on a large store. A snapshot taken in that window must +// still reopen, which is why the published markers are stamped rather than +// inherited from whatever each checkpoint happened to capture. +func TestSnapshotStampsOneEarliestVersionAcrossTrees(t *testing.T) { + store, root := setupSnapshotStore(t, 10, 5, false) + + for v := int64(1); v <= 9; v++ { + writeBlock(t, store, v) + } + // Mid-prune: EVM has advanced its earliest marker, Cosmos has not been + // reached yet. composite.Prune visits EVM first, so this is the real order. + require.NoError(t, store.evmStore.SetEarliestVersion(5, false)) + require.Equal(t, int64(0), store.cosmosStore.GetEarliestVersion()) + + writeBlock(t, store, 10) + settle(t, store) + + snapDir := filepath.Join(root, SnapshotDirName(10)) + require.DirExists(t, snapDir) + + reopened, err := NewCompositeStateStore(config.StateStoreConfig{ + Backend: "pebbledb", + AsyncWriteBuffer: 0, + KeepRecent: 100000, + EVMSplit: true, + DBDirectory: filepath.Join(snapDir, "cosmos", "pebbledb"), + EVMDBDirectory: filepath.Join(snapDir, "evm", "pebbledb"), + }, t.TempDir()) + require.NoError(t, err, "a snapshot taken during a prune pass must reopen") + defer reopened.Close() + + // The stamp is the highest earliest version any tree had reached, so the + // snapshot never claims to serve a version one of its trees has dropped. + require.Equal(t, int64(5), reopened.GetEarliestVersion()) + require.Equal(t, int64(5), reopened.cosmosStore.GetEarliestVersion()) + require.Equal(t, int64(5), reopened.evmStore.GetEarliestVersion()) +} + +// With separate sub-DBs the stamp has to reach every one of them, not just the +// tree root, or the reopened store reports the lowest sub-DB and the check that +// guards a mixed pair fails again. +func TestSnapshotStampsEveryEVMSubDB(t *testing.T) { + store, root := setupSnapshotStore(t, 10, 5, true) + + for v := int64(1); v <= 9; v++ { + writeBlock(t, store, v) + } + require.NoError(t, store.evmStore.SetEarliestVersion(6, false)) + require.Equal(t, int64(0), store.cosmosStore.GetEarliestVersion()) + + writeBlock(t, store, 10) + settle(t, store) + + snapDir := filepath.Join(root, SnapshotDirName(10)) + reopened, err := NewCompositeStateStore(config.StateStoreConfig{ + Backend: "pebbledb", + AsyncWriteBuffer: 0, + KeepRecent: 100000, + EVMSplit: true, + SeparateEVMSubDBs: true, + DBDirectory: filepath.Join(snapDir, "cosmos", "pebbledb"), + EVMDBDirectory: filepath.Join(snapDir, "evm", "pebbledb"), + }, t.TempDir()) + require.NoError(t, err) + defer reopened.Close() + + require.Equal(t, int64(6), reopened.cosmosStore.GetEarliestVersion()) + // EVMStateStore reports the lowest of its sub-DBs, so this reads 6 only if + // all of them were stamped. + require.Equal(t, int64(6), reopened.evmStore.GetEarliestVersion()) +} + // The reason the barrier has to be a message in every queue rather than a wait: // a block that only touches storage keys is enqueued only on the storage sub-DB, // so the idle sub-DBs never observe that version and no amount of waiting would diff --git a/sei-db/state_db/ss/cosmos/store.go b/sei-db/state_db/ss/cosmos/store.go index 02aeaf915c..d617e75037 100644 --- a/sei-db/state_db/ss/cosmos/store.go +++ b/sei-db/state_db/ss/cosmos/store.go @@ -80,16 +80,22 @@ func (s *CosmosStateStore) Close() error { func (s *CosmosStateStore) SupportsCheckpoint() bool { _, checkpointable := s.db.(types.Checkpointable) _, barrier := s.db.(types.DrainBarrier) - _, versionSetter := s.db.(types.CheckpointVersionSetter) - return checkpointable && barrier && versionSetter + _, markerSetter := s.db.(types.CheckpointMarkerSetter) + return checkpointable && barrier && markerSetter } func (s *CosmosStateStore) ScheduleCheckpoint(destDir string, shouldRun func() bool, done func(error)) { types.ScheduleCheckpoint(s.db, destDir, shouldRun, done) } -func (s *CosmosStateStore) SetCheckpointVersion(destDir string, version int64) error { - return types.SetCheckpointVersion(s.db, destDir, version) +func (s *CosmosStateStore) SetCheckpointMarkers(destDir string, latest, earliest int64) error { + return types.SetCheckpointMarkers(s.db, destDir, latest, earliest) +} + +// HighestEarliestVersion has one database to report on, so it agrees with +// GetEarliestVersion. +func (s *CosmosStateStore) HighestEarliestVersion() int64 { + return s.db.GetEarliestVersion() } func (s *CosmosStateStore) WaitForPendingWrites() { diff --git a/sei-db/state_db/ss/evm/db_test.go b/sei-db/state_db/ss/evm/db_test.go index 4b7682f302..3f307edbba 100644 --- a/sei-db/state_db/ss/evm/db_test.go +++ b/sei-db/state_db/ss/evm/db_test.go @@ -31,6 +31,34 @@ func openTestStore(t *testing.T) types.StateStore { return store } +// Prune runs the sub-DBs in parallel and each writes its own earliest marker +// when it finishes, so mid-pass they disagree. The two readings answer different +// questions: GetEarliestVersion reports what the store as a whole can still be +// asked for, while HighestEarliestVersion reports the floor every sub-DB can +// honor, which is what a snapshot has to be stamped with. +func TestHighestEarliestVersionReportsTheFurthestPrunedSubDB(t *testing.T) { + dir := t.TempDir() + cfg := testConfig() + cfg.SeparateEVMSubDBs = true + + store, err := NewEVMStateStore(dir, cfg) + require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) + require.Greater(t, len(store.managedDBs), 1) + + require.Zero(t, store.GetEarliestVersion()) + require.Zero(t, store.HighestEarliestVersion()) + + require.NoError(t, store.subDBs[StoreStorage].SetEarliestVersion(40, false)) + require.Zero(t, store.GetEarliestVersion(), "an unpruned sub-DB still reads 0") + require.Equal(t, int64(40), store.HighestEarliestVersion()) + + // Once the pass finishes, the two agree again. + require.NoError(t, store.SetEarliestVersion(40, false)) + require.Equal(t, int64(40), store.GetEarliestVersion()) + require.Equal(t, int64(40), store.HighestEarliestVersion()) +} + func TestEVMStateStoreDefaultUsesUnifiedDB(t *testing.T) { dir := t.TempDir() cfg := testConfig() diff --git a/sei-db/state_db/ss/evm/store.go b/sei-db/state_db/ss/evm/store.go index daba3ad631..81b77cec5b 100644 --- a/sei-db/state_db/ss/evm/store.go +++ b/sei-db/state_db/ss/evm/store.go @@ -380,7 +380,7 @@ func (s *EVMStateStore) SupportsCheckpoint() bool { if _, barrier := db.(types.DrainBarrier); !barrier { return false } - if _, versionSetter := db.(types.CheckpointVersionSetter); !versionSetter { + if _, markerSetter := db.(types.CheckpointMarkerSetter); !markerSetter { return false } } @@ -433,23 +433,37 @@ func (s *EVMStateStore) ScheduleCheckpoint(destDir string, shouldRun func() bool } } -func (s *EVMStateStore) SetCheckpointVersion(destDir string, version int64) error { +func (s *EVMStateStore) SetCheckpointMarkers(destDir string, latest, earliest int64) error { if !s.separateDBs { db := s.primaryDB() if db == nil { return errors.New("EVM state store has no managed DB to stamp") } - return types.SetCheckpointVersion(db, destDir, version) + return types.SetCheckpointMarkers(db, destDir, latest, earliest) } for _, storeType := range AllEVMStoreTypes() { dest := filepath.Join(destDir, StoreTypeName(storeType)) - if err := types.SetCheckpointVersion(s.subDBs[storeType], dest, version); err != nil { - return fmt.Errorf("set EVM sub-DB %s checkpoint version: %w", StoreTypeName(storeType), err) + if err := types.SetCheckpointMarkers(s.subDBs[storeType], dest, latest, earliest); err != nil { + return fmt.Errorf("set EVM sub-DB %s checkpoint markers: %w", StoreTypeName(storeType), err) } } return nil } +// HighestEarliestVersion reports the furthest any sub-DB has been pruned. Prune +// runs the sub-DBs in parallel and each writes its own marker when it finishes, +// so mid-pass they disagree, and only the highest is a floor every sub-DB can +// honor. +func (s *EVMStateStore) HighestEarliestVersion() int64 { + var highest int64 + for _, db := range s.managedDBs { + if v := db.GetEarliestVersion(); v > highest { + highest = v + } + } + return highest +} + func (s *EVMStateStore) WaitForPendingWrites() { for _, db := range s.managedDBs { if w, ok := db.(interface{ WaitForPendingWrites() }); ok { From 50ba6e3c8ce7e5dc9d4bdb1b99c3fa3f64350961 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Wed, 12 Aug 2026 00:14:46 -0400 Subject: [PATCH 07/13] fix(seidb): convert the checkpoint marker where its bound is in view Extracting the marker write moved the int64 to uint64 conversion away from the non-negative check that makes it sound, which gosec reported as G115. The helper now takes the encoded value, so the conversion sits in the function that rejects a negative version. Co-authored-by: Cursor --- sei-db/db_engine/pebbledb/mvcc/db.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sei-db/db_engine/pebbledb/mvcc/db.go b/sei-db/db_engine/pebbledb/mvcc/db.go index faedc39c60..ec7046f6f7 100644 --- a/sei-db/db_engine/pebbledb/mvcc/db.go +++ b/sei-db/db_engine/pebbledb/mvcc/db.go @@ -317,9 +317,10 @@ func (db *Database) SetCheckpointMarkers(destDir string, latest, earliest int64) return fmt.Errorf("open checkpoint %q to set markers: %w", destDir, err) } - setErr := setCheckpointMarker(checkpoint, latestVersionKey, latest) + // Converted here, where the non-negative check above is in view. + setErr := setCheckpointMarker(checkpoint, latestVersionKey, uint64(latest)) if setErr == nil { - setErr = setCheckpointMarker(checkpoint, earliestVersionKey, earliest) + setErr = setCheckpointMarker(checkpoint, earliestVersionKey, uint64(earliest)) } closeErr := checkpoint.Close() if setErr != nil { @@ -331,9 +332,9 @@ func (db *Database) SetCheckpointMarkers(destDir string, latest, earliest int64) return errors.Join(setErr, closeErr) } -func setCheckpointMarker(checkpoint *pebble.DB, key string, version int64) error { +func setCheckpointMarker(checkpoint *pebble.DB, key string, version uint64) error { var marker [VersionSize]byte - binary.LittleEndian.PutUint64(marker[:], uint64(version)) + binary.LittleEndian.PutUint64(marker[:], version) return checkpoint.Set([]byte(key), marker[:], pebble.Sync) } From 5d1e675ca14141dccba19a136077a6572569530e Mon Sep 17 00:00:00 2001 From: blindchaser Date: Wed, 12 Aug 2026 01:09:18 -0400 Subject: [PATCH 08/13] docs(seidb): state both halves of the prune interaction together The earlier note recorded only the data-level effect of an unordered prune and called it harmless, which is true of the data and not of the version markers. Both now sit in one paragraph, so the reason the markers are stamped is next to the reason the data does not need to be. Co-authored-by: Cursor --- sei-db/state_db/ss/composite/snapshot.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/sei-db/state_db/ss/composite/snapshot.go b/sei-db/state_db/ss/composite/snapshot.go index 93b7ceb761..5db461c1a5 100644 --- a/sei-db/state_db/ss/composite/snapshot.go +++ b/sei-db/state_db/ss/composite/snapshot.go @@ -49,15 +49,15 @@ import ( // call ScheduleSnapshot. The rootmulti commit path owns the trigger for every // block, populated or empty, and is the only caller of ScheduleSnapshot. // -// Because pruning is outside the barrier and advances one database at a time, -// the version markers a snapshot publishes are stamped at publication rather -// than inherited from each checkpoint. Otherwise a snapshot taken during a -// prune pass would pair a pruned tree with an unpruned one and would be -// rejected on reopen. See stampedEarliest. -// Because pruning bypasses the barrier, a checkpoint can capture a partially -// applied prune — the same state a crash mid-prune leaves on the live DB. -// Reads at the label version are unaffected; only historical reads near the -// pruning horizon can see it. +// Pruning is the one writer nothing orders a snapshot against, and it has two +// separate effects. In the data, a checkpoint can capture a partially applied +// prune — the same state a crash mid-prune leaves on the live DB. That one is +// harmless: reads at the label are unaffected, and only historical reads near +// the pruning horizon can see it. In the version markers it is not harmless, +// because pruning advances the databases one at a time and a snapshot would +// pair a pruned tree with an unpruned one, which is rejected on reopen. The +// markers are therefore stamped at publication rather than inherited from each +// checkpoint. See stampedEarliest. // // SS rollback is not part of this feature, and the two do not compose yet. A // rollback leaves lastRequested at the pre-rollback high-water mark, so the From 183f87b3d6730cad35aab084051fdd6acc4845d6 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Thu, 13 Aug 2026 10:50:50 -0400 Subject: [PATCH 09/13] fix(seidb): report honest SS earliest floor Use the highest member earliest marker as the public SS floor so snapshots can inherit per-DB markers without cross-tree reconciliation, and advance prune markers before deleting history so checkpoint boundaries stay honest. Co-authored-by: Cursor --- .github/pr-summary.md | 23 +++++ sei-db/db_engine/pebbledb/mvcc/db.go | 37 ++++---- .../db_engine/pebbledb/mvcc/db_ascending.go | 14 +-- sei-db/db_engine/pebbledb/mvcc/db_test.go | 12 +-- sei-db/db_engine/pebbledb/mvcc/prune_test.go | 24 +++++ sei-db/db_engine/types/types.go | 33 +++---- sei-db/state_db/ss/composite/recovery_test.go | 24 +++-- sei-db/state_db/ss/composite/snapshot.go | 57 ++---------- sei-db/state_db/ss/composite/snapshot_test.go | 89 +++++++++++++------ sei-db/state_db/ss/composite/store.go | 29 +++--- sei-db/state_db/ss/cosmos/store.go | 14 +-- sei-db/state_db/ss/evm/db_test.go | 16 ++-- sei-db/state_db/ss/evm/store.go | 35 ++------ 13 files changed, 212 insertions(+), 195 deletions(-) create mode 100644 .github/pr-summary.md diff --git a/.github/pr-summary.md b/.github/pr-summary.md new file mode 100644 index 0000000000..53cd0a4daf --- /dev/null +++ b/.github/pr-summary.md @@ -0,0 +1,23 @@ +## Summary + +Add opt-in, exact-version Pebble checkpoints for the State Store. This PR contains snapshot generation and retention only; State Store rollback remains separate. + +- `sei-db/db_engine/types/types.go` and `sei-db/db_engine/pebbledb/mvcc/db.go`: add an ordered drain barrier to the single-writer FIFO. A snapshot barrier runs after version H and before H+1, supports shutdown cancellation, and stamps only the latest-version marker in the checkpoint copy. +- `sei-db/state_db/ss/cosmos/store.go` and `sei-db/state_db/ss/evm/store.go`: expose checkpoint scheduling for Cosmos, unified EVM, and separate EVM sub-databases. Composite publication waits for every required checkpoint. +- `sei-db/state_db/ss/evm/store.go`, `sei-db/state_db/ss/composite/store.go`, and MVCC prune paths: report the highest earliest-version marker across managed databases, allow divergent member floors after recovery, and advance prune's earliest marker before deleting history so snapshots can safely inherit per-DB markers. +- `sei-db/state_db/ss/composite/snapshot.go`: publish checkpoints through a temporary directory, durable rename, and atomic `current` symlink. Startup removes stale staging data, restores `current`, resumes from the newest snapshot, and enforces retention even after a failed publish. +- `sei-db/state_db/ss/composite/snapshot.go`: reject startup when any live SS database cannot hardlink into the snapshot root. Custom Cosmos directories place the root in a sibling `-snapshots` directory. Known limitation: all enabled Cosmos and EVM databases must use one filesystem. +- `sei-db/state_db/ss/composite/store.go` and `sei-cosmos/storev2/rootmulti/store.go`: trigger snapshots from one place, `rootmulti.flush`, for both populated and empty blocks. `ApplyChangesetAsync` schedules nothing, so callers outside the commit path do not inherit a trigger, and a state store that cannot schedule fails the build rather than losing snapshots silently. State-sync import and direct version-marker writes cannot publish a partial snapshot. +- `sei-db/config/*.go`, `app/seidb.go`, and `sei-cosmos/server/config/config.go`: keep snapshots default-off. When enabled, SS mirrors SC's effective block interval, minimum time interval, and retention settings. SS and SC apply independent in-flight gates, so they do not guarantee identical retained heights. +- `sei-db/state_db/ss/composite/snapshot.go` and `snapshot_metrics.go`: allow one snapshot in flight, cancel queued work during close, persist apparent-size metadata outside the publish lock, and export attempt, skip, outcome, duration, in-flight, height, retained-count, and apparent-byte metrics. +- `sei-db/common/utils/path.go`: define the shared default snapshot directory name. Managed snapshot paths have no lease; consumers must coordinate with generation and pruning. + +## Test plan + +- `sei-db/db_engine/pebbledb/mvcc/db_test.go` and `prune_test.go`: checkpoint copies preserve exact latest markers, inherit earliest markers, queued checkpoints cancel before work starts, and prune advances the earliest marker before delete work while preserving pruning coverage. +- `sei-db/state_db/ss/composite/snapshot_test.go`: cover exact labels, exclusion of versions written after the boundary while the write path keeps going, Cosmos-only and EVM-split layouts, idle EVM sub-databases, empty blocks, state-sync isolation, inherited per-store earliest markers, and latest labels across EVM sub-databases. +- `sei-db/state_db/ss/evm/db_test.go` and `sei-db/state_db/ss/composite/recovery_test.go`: cover max earliest-floor reporting and allow recovery with divergent member floors. +- `sei-cosmos/storev2/rootmulti/store_test.go`: cover the commit-path wiring, so a boundary produces a snapshot whether the block carried changesets or not. +- `sei-db/state_db/ss/composite/snapshot_test.go`: cover one-in-flight and minimum-time gates, close during barrier scheduling, custom directories, cross-filesystem rejection, restart resumption, stale staging cleanup, out-of-order publication, failed-publication retention, and persisted size metadata. +- `sei-db/config/ss_config_test.go`, `app/config_fuzz_test.go`, and configuration goldens: cover default-off rollout and effective SC cadence mirroring. +- Verified with `go test -race ./sei-db/db_engine/pebbledb/mvcc ./sei-db/state_db/ss/... ./evmrpc ./sei-cosmos/storev2/...`. diff --git a/sei-db/db_engine/pebbledb/mvcc/db.go b/sei-db/db_engine/pebbledb/mvcc/db.go index ec7046f6f7..c00bc19855 100644 --- a/sei-db/db_engine/pebbledb/mvcc/db.go +++ b/sei-db/db_engine/pebbledb/mvcc/db.go @@ -299,15 +299,11 @@ func (db *Database) Checkpoint(destDir string) error { return nil } -// SetCheckpointMarkers writes the version range into a completed checkpoint -// without changing the live database markers. Both are written under one open -// so a checkpoint is never left describing half a range. -func (db *Database) SetCheckpointMarkers(destDir string, latest, earliest int64) error { - if latest < 0 || earliest < 0 { - return fmt.Errorf("versions must be non-negative") - } - if earliest > latest { - return fmt.Errorf("earliest version %d is above latest version %d", earliest, latest) +// SetCheckpointVersion writes the logical latest version into a completed +// checkpoint without changing the live database marker. +func (db *Database) SetCheckpointVersion(destDir string, version int64) error { + if version < 0 { + return fmt.Errorf("version must be non-negative") } opts := newPebbleOptions(db.config, nil) @@ -318,16 +314,13 @@ func (db *Database) SetCheckpointMarkers(destDir string, latest, earliest int64) } // Converted here, where the non-negative check above is in view. - setErr := setCheckpointMarker(checkpoint, latestVersionKey, uint64(latest)) - if setErr == nil { - setErr = setCheckpointMarker(checkpoint, earliestVersionKey, uint64(earliest)) - } + setErr := setCheckpointMarker(checkpoint, latestVersionKey, uint64(version)) closeErr := checkpoint.Close() if setErr != nil { - setErr = fmt.Errorf("set checkpoint markers latest=%d earliest=%d: %w", latest, earliest, setErr) + setErr = fmt.Errorf("set checkpoint version %d: %w", version, setErr) } if closeErr != nil { - closeErr = fmt.Errorf("close checkpoint after setting markers: %w", closeErr) + closeErr = fmt.Errorf("close checkpoint after setting version: %w", closeErr) } return errors.Join(setErr, closeErr) } @@ -707,6 +700,10 @@ func (db *Database) pruneDescending(version int64) (_err error) { }() earliestVersion := version + 1 // we increment by 1 to include the provided version + prevEarliestVersion := db.GetEarliestVersion() + if err := db.SetEarliestVersion(earliestVersion, false); err != nil { + return err + } itr, err := db.storage.NewIter(nil) if err != nil { @@ -753,8 +750,11 @@ func (db *Database) pruneDescending(version int64) (_err error) { prevStore = storeKey updated, ok := db.storeKeyDirty.Load(storeKey) versionUpdated, typeOk := updated.(int64) - // Skip a store's keys if version it was last updated is less than last prune height - if !ok || (typeOk && versionUpdated < db.GetEarliestVersion()) { + // The marker is advanced before deletes so checkpoints never claim + // history that the prune has already dropped. The skip heuristic must + // still compare against the pre-prune marker; otherwise this pass would + // skip stores whose latest update is at or below the prune height. + if !ok || (typeOk && versionUpdated < prevEarliestVersion) { itr.SeekGE(storePrefix(storeKey + "0")) continue } @@ -825,9 +825,6 @@ func (db *Database) pruneDescending(version int64) (_err error) { } db.operationMetrics.AddRead(scanReads) - if err := db.SetEarliestVersion(earliestVersion, false); err != nil { - return err - } return db.compactPrunedRange(firstDeletedKey, lastDeletedKey) } diff --git a/sei-db/db_engine/pebbledb/mvcc/db_ascending.go b/sei-db/db_engine/pebbledb/mvcc/db_ascending.go index 4075f9eea1..82e8496029 100644 --- a/sei-db/db_engine/pebbledb/mvcc/db_ascending.go +++ b/sei-db/db_engine/pebbledb/mvcc/db_ascending.go @@ -108,6 +108,10 @@ func (db *Database) pruneAscending(version int64) (_err error) { }() earliestVersion := version + 1 // we increment by 1 to include the provided version + prevEarliestVersion := db.GetEarliestVersion() + if err := db.SetEarliestVersion(earliestVersion, false); err != nil { + return err + } itr, err := db.storage.NewIter(nil) if err != nil { @@ -154,8 +158,11 @@ func (db *Database) pruneAscending(version int64) (_err error) { prevStore = storeKey updated, ok := db.storeKeyDirty.Load(storeKey) versionUpdated, typeOk := updated.(int64) - // Skip a store's keys if version it was last updated is less than last prune height - if !ok || (typeOk && versionUpdated < db.GetEarliestVersion()) { + // The marker is advanced before deletes so checkpoints never claim + // history that the prune has already dropped. The skip heuristic must + // still compare against the pre-prune marker; otherwise this pass would + // skip stores whose latest update is at or below the prune height. + if !ok || (typeOk && versionUpdated < prevEarliestVersion) { itr.SeekGE(storePrefix(storeKey + "0")) continue } @@ -224,9 +231,6 @@ func (db *Database) pruneAscending(version int64) (_err error) { } db.operationMetrics.AddRead(scanReads) - if err := db.SetEarliestVersion(earliestVersion, false); err != nil { - return err - } return db.compactPrunedRange(firstDeletedKey, lastDeletedKey) } diff --git a/sei-db/db_engine/pebbledb/mvcc/db_test.go b/sei-db/db_engine/pebbledb/mvcc/db_test.go index 709d2096e8..febcf5cfcc 100644 --- a/sei-db/db_engine/pebbledb/mvcc/db_test.go +++ b/sei-db/db_engine/pebbledb/mvcc/db_test.go @@ -66,10 +66,9 @@ func TestVersionedCheckpointPreservesFutureLiveMarker(t *testing.T) { done <- err }) require.NoError(t, <-done) - // The caller decides both markers. Neither has to match what the copy - // inherited: the label comes from the barrier, and the earliest version is - // reconciled across every tree in the snapshot. - require.NoError(t, types.SetCheckpointMarkers(store, dest, 5, 5)) + // The caller stamps only the label. Earliest is inherited from the + // checkpointed DB because prune advances it before deleting history. + require.NoError(t, types.SetCheckpointVersion(store, dest, 5)) require.Equal(t, int64(10), store.GetLatestVersion()) require.Equal(t, int64(4), store.GetEarliestVersion()) @@ -84,10 +83,7 @@ func TestVersionedCheckpointPreservesFutureLiveMarker(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { require.NoError(t, checkpoint.Close()) }) require.Equal(t, int64(5), checkpoint.GetLatestVersion()) - require.Equal(t, int64(5), checkpoint.GetEarliestVersion()) - - require.Error(t, types.SetCheckpointMarkers(store, dest, 5, 6), - "an earliest version above the label describes an empty range") + require.Equal(t, int64(4), checkpoint.GetEarliestVersion()) } func TestScheduledCheckpointCanBeCanceledAtBarrier(t *testing.T) { diff --git a/sei-db/db_engine/pebbledb/mvcc/prune_test.go b/sei-db/db_engine/pebbledb/mvcc/prune_test.go index 6334760534..5809585f11 100644 --- a/sei-db/db_engine/pebbledb/mvcc/prune_test.go +++ b/sei-db/db_engine/pebbledb/mvcc/prune_test.go @@ -119,4 +119,28 @@ func TestPruneDescendingOrder_DeletesOldVersions(t *testing.T) { require.ElementsMatch(t, []int64{140}, rawVersionsForKey(t, db, store, k2)) }) + t.Run("idle store still prunes against previous earliest marker", func(t *testing.T) { + db := newTestDB(t, true) + + applyVersion(t, db, store, 50, key, []byte("v50")) + applyVersion(t, db, store, 100, key, []byte("v100")) + + require.NoError(t, db.Prune(150)) + + versions := rawVersionsForKey(t, db, store, key) + require.ElementsMatch(t, []int64{100}, versions, + "prune must not use the just-advanced marker to skip this store") + }) + +} + +func TestPruneAdvancesEarliestBeforeDeletingHistory(t *testing.T) { + db := newTestDB(t, true) + + require.NoError(t, db.storage.Set([]byte("invalid-mvcc-key"), []byte("value"), defaultWriteOpts)) + + err := db.Prune(10) + require.Error(t, err) + require.Equal(t, int64(11), db.GetEarliestVersion(), + "earliest marker must advance before a later prune failure") } diff --git a/sei-db/db_engine/types/types.go b/sei-db/db_engine/types/types.go index 58bc64e178..92f6cc8f70 100644 --- a/sei-db/db_engine/types/types.go +++ b/sei-db/db_engine/types/types.go @@ -122,16 +122,16 @@ type Checkpointable interface { Checkpoint(destDir string) error } -// CheckpointMarkerSetter writes the logical version range into a completed +// CheckpointVersionSetter writes the logical latest version into a completed // checkpoint without changing the live database. // -// Both markers are stamped rather than inherited from the checkpoint. The -// latest marker has to be stamped because a checkpoint is a copy of a database -// whose marker may already have moved on. The earliest marker has to be stamped -// because pruning advances it outside the write queue and one database at a -// time, so two checkpoints of the same instant can disagree about it. -type CheckpointMarkerSetter interface { - SetCheckpointMarkers(destDir string, latest, earliest int64) error +// The latest marker has to be stamped because a checkpoint is a copy of a +// database whose marker may already have moved on. The earliest marker is +// inherited from the checkpoint; pruning advances it before deleting history, so +// every checkpoint boundary either sees the old marker with old data or the new +// marker with data that is at least as deep as advertised. +type CheckpointVersionSetter interface { + SetCheckpointVersion(destDir string, version int64) error } // DrainBarrier is an optional capability for engines that apply changesets from @@ -145,12 +145,7 @@ type DrainBarrier interface { type CheckpointScheduler interface { SupportsCheckpoint() bool ScheduleCheckpoint(destDir string, shouldRun func() bool, done func(error)) - SetCheckpointMarkers(destDir string, latest, earliest int64) error - // HighestEarliestVersion reports the largest earliest version among the - // databases this store owns. GetEarliestVersion reports the smallest, which - // is the right answer for serving reads but the wrong one for stamping a - // snapshot: it would claim a range a partly pruned database cannot serve. - HighestEarliestVersion() int64 + SetCheckpointVersion(destDir string, version int64) error } // ErrCheckpointCanceled reports that a queued checkpoint was canceled before @@ -179,14 +174,14 @@ func ScheduleCheckpoint(db StateStore, destDir string, shouldRun func() bool, do }) } -// SetCheckpointMarkers makes a completed checkpoint self-describing without +// SetCheckpointVersion makes a completed checkpoint self-describing without // changing the live database. -func SetCheckpointMarkers(db StateStore, destDir string, latest, earliest int64) error { - setter, ok := db.(CheckpointMarkerSetter) +func SetCheckpointVersion(db StateStore, destDir string, version int64) error { + setter, ok := db.(CheckpointVersionSetter) if !ok { - return fmt.Errorf("state store backend %T cannot set checkpoint markers", db) + return fmt.Errorf("state store backend %T cannot set checkpoint version", db) } - return setter.SetCheckpointMarkers(destDir, latest, earliest) + return setter.SetCheckpointVersion(destDir, version) } // --------------------------------------------------------------------------- diff --git a/sei-db/state_db/ss/composite/recovery_test.go b/sei-db/state_db/ss/composite/recovery_test.go index 9688b3afac..926e5d4ef7 100644 --- a/sei-db/state_db/ss/composite/recovery_test.go +++ b/sei-db/state_db/ss/composite/recovery_test.go @@ -78,23 +78,35 @@ func TestEVMSSPreRecoveryAfterStateSync(t *testing.T) { require.Contains(t, err.Error(), "EVM SS is empty") } -// TestEVMSSPostRecoveryEarliestMismatch: diverging earliest versions must abort startup. +// TestEVMSSPostRecoveryEarliestMismatch: diverging earliest versions are allowed +// because the composite reports the highest member floor. func TestEVMSSPostRecoveryEarliestMismatch(t *testing.T) { cosmos := &fakeStateStore{latest: 100, earliest: 50} evm := &fakeStateStore{latest: 100, earliest: 75} cs := newCompositeStateStoreWithStores(cosmos, evm, config.StateStoreConfig{EVMSplit: true}) - err := cs.validateEVMSSPostRecovery() - require.Error(t, err) - require.Contains(t, err.Error(), "earliest version") + cs.validateEVMSSPostRecovery() // Matching earliest → pass. evm.earliest = 50 - require.NoError(t, cs.validateEVMSSPostRecovery()) + cs.validateEVMSSPostRecovery() // Both zero → pass (fresh DBs). cosmos.earliest = 0 evm.earliest = 0 - require.NoError(t, cs.validateEVMSSPostRecovery()) + cs.validateEVMSSPostRecovery() +} + +func TestCompositeGetEarliestVersionReportsHighestMemberFloor(t *testing.T) { + cosmos := &fakeStateStore{latest: 100, earliest: 50} + evm := &fakeStateStore{latest: 100, earliest: 75} + cs := newCompositeStateStoreWithStores(cosmos, evm, config.StateStoreConfig{EVMSplit: true}) + require.Equal(t, int64(75), cs.GetEarliestVersion()) + + cosmos.earliest = 90 + require.Equal(t, int64(90), cs.GetEarliestVersion()) + + cs.evmStore = nil + require.Equal(t, int64(90), cs.GetEarliestVersion()) } // fakeStateStore stubs latest/earliest for validator tests. diff --git a/sei-db/state_db/ss/composite/snapshot.go b/sei-db/state_db/ss/composite/snapshot.go index 5db461c1a5..ba9810c747 100644 --- a/sei-db/state_db/ss/composite/snapshot.go +++ b/sei-db/state_db/ss/composite/snapshot.go @@ -49,15 +49,12 @@ import ( // call ScheduleSnapshot. The rootmulti commit path owns the trigger for every // block, populated or empty, and is the only caller of ScheduleSnapshot. // -// Pruning is the one writer nothing orders a snapshot against, and it has two -// separate effects. In the data, a checkpoint can capture a partially applied -// prune — the same state a crash mid-prune leaves on the live DB. That one is -// harmless: reads at the label are unaffected, and only historical reads near -// the pruning horizon can see it. In the version markers it is not harmless, -// because pruning advances the databases one at a time and a snapshot would -// pair a pruned tree with an unpruned one, which is rejected on reopen. The -// markers are therefore stamped at publication rather than inherited from each -// checkpoint. See stampedEarliest. +// Pruning is the one writer nothing orders a snapshot against. A checkpoint can +// capture a partially applied prune — the same state a crash mid-prune leaves on +// the live DB. This is safe for a snapshot because pruning advances each DB's +// earliest marker before deleting history, so the checkpoint never claims a +// range the DB has already dropped. Reopening a snapshot with different member +// floors is allowed; the composite reports the highest floor any member carries. // // SS rollback is not part of this feature, and the two do not compose yet. A // rollback leaves lastRequested at the pre-rollback high-water mark, so the @@ -432,12 +429,11 @@ func (m *snapshotManager) startPublish( _ = os.RemoveAll(tmpDir) return } - earliest := m.stampedEarliest(version) for _, target := range targets { - if err := target.store.SetCheckpointMarkers(target.dest, version, earliest); err != nil { + if err := target.store.SetCheckpointVersion(target.dest, version); err != nil { recordSnapshotCompletion(start, "failure") - logger.Error("failed to set state store snapshot markers", - "version", version, "earliest", earliest, "dir", target.dest, "error", err) + logger.Error("failed to set state store snapshot version", + "version", version, "dir", target.dest, "error", err) _ = os.RemoveAll(tmpDir) return } @@ -450,41 +446,6 @@ func (m *snapshotManager) startPublish( }() } -// stampedEarliest is the earliest version the published snapshot claims to -// serve. Every tree in one snapshot gets this same value. -// -// A prune pass is the reason it cannot be left to each checkpoint. Pruning -// bypasses the apply queues, so no barrier orders a snapshot against it, and it -// advances the databases one at a time: composite.Prune finishes EVM, including -// its compaction, before it starts the Cosmos scan. On a large store that gap -// is minutes long, and a snapshot taken inside it would capture a Cosmos tree -// that has not been pruned next to an EVM tree that has. Reopening that -// snapshot fails the earliest-version agreement check in NewCompositeStateStore -// and the snapshot is scrap. -// -// The value is the highest earliest version reached anywhere, because a -// snapshot must not promise a version one of its trees has already dropped. -// Reading it here rather than at checkpoint time can only overshoot what a tree -// holds, since an earliest marker never moves backward outside a restore, and -// overshooting is the safe direction: the snapshot serves less than it has -// rather than claiming more. It is clamped to the label so the pair stays -// ordered when retention is short enough for pruning to overtake the boundary. -func (m *snapshotManager) stampedEarliest(label int64) int64 { - earliest := m.cosmosScheduler.HighestEarliestVersion() - if m.evmScheduler != nil { - if evmEarliest := m.evmScheduler.HighestEarliestVersion(); evmEarliest > earliest { - earliest = evmEarliest - } - } - if earliest > label { - return label - } - if earliest < 0 { - return 0 - } - return earliest -} - // publish moves a finished checkpoint into place and reports whether the whole // publication succeeded. Retention runs either way. // diff --git a/sei-db/state_db/ss/composite/snapshot_test.go b/sei-db/state_db/ss/composite/snapshot_test.go index 3852f53306..11e3665880 100644 --- a/sei-db/state_db/ss/composite/snapshot_test.go +++ b/sei-db/state_db/ss/composite/snapshot_test.go @@ -26,7 +26,7 @@ func (*noBarrierStateStore) Checkpoint(string) error { return nil } -func (*noBarrierStateStore) SetCheckpointMarkers(string, int64, int64) error { +func (*noBarrierStateStore) SetCheckpointVersion(string, int64) error { return nil } @@ -59,14 +59,10 @@ func (s *controlledSnapshotScheduler) ScheduleCheckpoint( } } -func (*controlledSnapshotScheduler) SetCheckpointMarkers(string, int64, int64) error { +func (*controlledSnapshotScheduler) SetCheckpointVersion(string, int64) error { return nil } -func (*controlledSnapshotScheduler) HighestEarliestVersion() int64 { - return 0 -} - func bankChangeset(key, value string) []*proto.NamedChangeSet { return []*proto.NamedChangeSet{ { @@ -530,21 +526,19 @@ func TestSnapshotExcludesVersionsWrittenAfterTheBoundary(t *testing.T) { require.Equal(t, []byte{byte(label + 2)}, val) } -// A prune pass advances each database's earliest marker on its own schedule and -// bypasses the barrier entirely, so the two SS trees disagree for as long as the -// pass runs — minutes on a large store. A snapshot taken in that window must -// still reopen, which is why the published markers are stamped rather than -// inherited from whatever each checkpoint happened to capture. -func TestSnapshotStampsOneEarliestVersionAcrossTrees(t *testing.T) { +// A snapshot inherits each database's earliest marker. The composite is allowed +// to reopen with different member floors because it reports the highest one. +func TestSnapshotInheritsPerStoreEarliestMarkers(t *testing.T) { store, root := setupSnapshotStore(t, 10, 5, false) for v := int64(1); v <= 9; v++ { writeBlock(t, store, v) } - // Mid-prune: EVM has advanced its earliest marker, Cosmos has not been - // reached yet. composite.Prune visits EVM first, so this is the real order. + settle(t, store) + require.NoError(t, store.cosmosStore.SetEarliestVersion(2, false)) require.NoError(t, store.evmStore.SetEarliestVersion(5, false)) - require.Equal(t, int64(0), store.cosmosStore.GetEarliestVersion()) + require.Equal(t, int64(2), store.cosmosStore.GetEarliestVersion()) + require.Equal(t, int64(5), store.evmStore.GetEarliestVersion()) writeBlock(t, store, 10) settle(t, store) @@ -560,27 +554,52 @@ func TestSnapshotStampsOneEarliestVersionAcrossTrees(t *testing.T) { DBDirectory: filepath.Join(snapDir, "cosmos", "pebbledb"), EVMDBDirectory: filepath.Join(snapDir, "evm", "pebbledb"), }, t.TempDir()) - require.NoError(t, err, "a snapshot taken during a prune pass must reopen") + require.NoError(t, err, "a snapshot with different member floors must reopen") + defer reopened.Close() + + require.Equal(t, int64(2), reopened.cosmosStore.GetEarliestVersion()) + require.Equal(t, int64(5), reopened.GetEarliestVersion()) + require.Equal(t, int64(5), reopened.evmStore.GetEarliestVersion()) +} + +func TestSnapshotInheritsEarliestMarkerAfterPrune(t *testing.T) { + store, root := setupSnapshotStore(t, 10, 5, false) + + for v := int64(1); v <= 9; v++ { + writeBlock(t, store, v) + } + settle(t, store) + require.NoError(t, store.Prune(4)) + + writeBlock(t, store, 10) + settle(t, store) + + snapDir := filepath.Join(root, SnapshotDirName(10)) + reopened, err := NewCompositeStateStore(config.StateStoreConfig{ + Backend: "pebbledb", + AsyncWriteBuffer: 0, + KeepRecent: 100000, + EVMSplit: true, + DBDirectory: filepath.Join(snapDir, "cosmos", "pebbledb"), + EVMDBDirectory: filepath.Join(snapDir, "evm", "pebbledb"), + }, t.TempDir()) + require.NoError(t, err) defer reopened.Close() - // The stamp is the highest earliest version any tree had reached, so the - // snapshot never claims to serve a version one of its trees has dropped. require.Equal(t, int64(5), reopened.GetEarliestVersion()) require.Equal(t, int64(5), reopened.cosmosStore.GetEarliestVersion()) require.Equal(t, int64(5), reopened.evmStore.GetEarliestVersion()) } -// With separate sub-DBs the stamp has to reach every one of them, not just the -// tree root, or the reopened store reports the lowest sub-DB and the check that -// guards a mixed pair fails again. -func TestSnapshotStampsEveryEVMSubDB(t *testing.T) { +// With separate sub-DBs the latest label has to reach every one of them, not +// just the sub-DBs that took writes, or the snapshot is not self-describing at +// its exact boundary. +func TestSnapshotSetsLatestVersionEveryEVMSubDB(t *testing.T) { store, root := setupSnapshotStore(t, 10, 5, true) for v := int64(1); v <= 9; v++ { writeBlock(t, store, v) } - require.NoError(t, store.evmStore.SetEarliestVersion(6, false)) - require.Equal(t, int64(0), store.cosmosStore.GetEarliestVersion()) writeBlock(t, store, 10) settle(t, store) @@ -596,12 +615,24 @@ func TestSnapshotStampsEveryEVMSubDB(t *testing.T) { EVMDBDirectory: filepath.Join(snapDir, "evm", "pebbledb"), }, t.TempDir()) require.NoError(t, err) - defer reopened.Close() - require.Equal(t, int64(6), reopened.cosmosStore.GetEarliestVersion()) - // EVMStateStore reports the lowest of its sub-DBs, so this reads 6 only if - // all of them were stamped. - require.Equal(t, int64(6), reopened.evmStore.GetEarliestVersion()) + require.Equal(t, int64(10), reopened.evmStore.GetLatestVersion()) + require.NoError(t, reopened.Close()) + + evmRoot := filepath.Join(snapDir, "evm", "pebbledb") + for _, storeType := range evm.AllEVMStoreTypes() { + subDir := filepath.Join(evmRoot, evm.StoreTypeName(storeType)) + subDB, err := NewCompositeStateStore(config.StateStoreConfig{ + Backend: "pebbledb", + AsyncWriteBuffer: 0, + KeepRecent: 100000, + UseDefaultComparer: true, + DBDirectory: subDir, + }, t.TempDir()) + require.NoError(t, err) + require.Equal(t, int64(10), subDB.GetLatestVersion(), "sub-DB %s latest marker", evm.StoreTypeName(storeType)) + require.NoError(t, subDB.Close()) + } } // The reason the barrier has to be a message in every queue rather than a wait: diff --git a/sei-db/state_db/ss/composite/store.go b/sei-db/state_db/ss/composite/store.go index dc73e5d118..70326b70da 100644 --- a/sei-db/state_db/ss/composite/store.go +++ b/sei-db/state_db/ss/composite/store.go @@ -110,11 +110,7 @@ func NewCompositeStateStore( return nil, fmt.Errorf("failed to recover state store: %w", err) } - // Mismatched earliest versions = DBs from different snapshots; reads would diverge. - if err := cs.validateEVMSSPostRecovery(); err != nil { - _ = cs.Close() - return nil, err - } + cs.validateEVMSSPostRecovery() if ssConfig.SnapshotInterval > 0 { snapshotRoot := utils.GetStateStoreSnapshotsPath(homeDir) @@ -177,20 +173,23 @@ func (s *CompositeStateStore) validateEVMSSPreRecovery() error { return nil } -// validateEVMSSPostRecovery rejects mismatched earliest versions between the two SS DBs. -func (s *CompositeStateStore) validateEVMSSPostRecovery() error { +// validateEVMSSPostRecovery reports mismatched earliest versions between SS DBs. +// Divergence is safe because GetEarliestVersion reports the highest member +// floor, which is the first version every routed store can serve. +func (s *CompositeStateStore) validateEVMSSPostRecovery() { if s.evmStore == nil { - return nil + return } cosmosEarliest := s.cosmosStore.GetEarliestVersion() evmEarliest := s.evmStore.GetEarliestVersion() if cosmosEarliest != evmEarliest && (cosmosEarliest > 0 || evmEarliest > 0) { - return fmt.Errorf( - "EVM SS earliest version %d does not match Cosmos SS earliest version %d: state sync the EVM SS DB, or set evm-ss-split=false", - evmEarliest, cosmosEarliest, + logger.Warn( + "EVM SS earliest version does not match Cosmos SS earliest version; serving the highest floor", + "evmEarliest", evmEarliest, + "cosmosEarliest", cosmosEarliest, + "reportedEarliest", max(cosmosEarliest, evmEarliest), ) } - return nil } func (s *CompositeStateStore) StartPruning() { @@ -243,7 +242,11 @@ func (s *CompositeStateStore) GetLatestVersion() int64 { } func (s *CompositeStateStore) GetEarliestVersion() int64 { - return s.cosmosStore.GetEarliestVersion() + earliest := s.cosmosStore.GetEarliestVersion() + if s.evmStore != nil { + earliest = max(earliest, s.evmStore.GetEarliestVersion()) + } + return earliest } func (s *CompositeStateStore) Close() error { diff --git a/sei-db/state_db/ss/cosmos/store.go b/sei-db/state_db/ss/cosmos/store.go index d617e75037..02aeaf915c 100644 --- a/sei-db/state_db/ss/cosmos/store.go +++ b/sei-db/state_db/ss/cosmos/store.go @@ -80,22 +80,16 @@ func (s *CosmosStateStore) Close() error { func (s *CosmosStateStore) SupportsCheckpoint() bool { _, checkpointable := s.db.(types.Checkpointable) _, barrier := s.db.(types.DrainBarrier) - _, markerSetter := s.db.(types.CheckpointMarkerSetter) - return checkpointable && barrier && markerSetter + _, versionSetter := s.db.(types.CheckpointVersionSetter) + return checkpointable && barrier && versionSetter } func (s *CosmosStateStore) ScheduleCheckpoint(destDir string, shouldRun func() bool, done func(error)) { types.ScheduleCheckpoint(s.db, destDir, shouldRun, done) } -func (s *CosmosStateStore) SetCheckpointMarkers(destDir string, latest, earliest int64) error { - return types.SetCheckpointMarkers(s.db, destDir, latest, earliest) -} - -// HighestEarliestVersion has one database to report on, so it agrees with -// GetEarliestVersion. -func (s *CosmosStateStore) HighestEarliestVersion() int64 { - return s.db.GetEarliestVersion() +func (s *CosmosStateStore) SetCheckpointVersion(destDir string, version int64) error { + return types.SetCheckpointVersion(s.db, destDir, version) } func (s *CosmosStateStore) WaitForPendingWrites() { diff --git a/sei-db/state_db/ss/evm/db_test.go b/sei-db/state_db/ss/evm/db_test.go index 3f307edbba..69dc3ea2b6 100644 --- a/sei-db/state_db/ss/evm/db_test.go +++ b/sei-db/state_db/ss/evm/db_test.go @@ -31,12 +31,9 @@ func openTestStore(t *testing.T) types.StateStore { return store } -// Prune runs the sub-DBs in parallel and each writes its own earliest marker -// when it finishes, so mid-pass they disagree. The two readings answer different -// questions: GetEarliestVersion reports what the store as a whole can still be -// asked for, while HighestEarliestVersion reports the floor every sub-DB can -// honor, which is what a snapshot has to be stamped with. -func TestHighestEarliestVersionReportsTheFurthestPrunedSubDB(t *testing.T) { +// GetEarliestVersion reports the highest sub-DB floor, which is the earliest +// version every routed sub-DB can serve. +func TestGetEarliestVersionReportsTheFurthestPrunedSubDB(t *testing.T) { dir := t.TempDir() cfg := testConfig() cfg.SeparateEVMSubDBs = true @@ -47,16 +44,13 @@ func TestHighestEarliestVersionReportsTheFurthestPrunedSubDB(t *testing.T) { require.Greater(t, len(store.managedDBs), 1) require.Zero(t, store.GetEarliestVersion()) - require.Zero(t, store.HighestEarliestVersion()) require.NoError(t, store.subDBs[StoreStorage].SetEarliestVersion(40, false)) - require.Zero(t, store.GetEarliestVersion(), "an unpruned sub-DB still reads 0") - require.Equal(t, int64(40), store.HighestEarliestVersion()) + require.Equal(t, int64(40), store.GetEarliestVersion()) - // Once the pass finishes, the two agree again. + // Once the pass finishes, the reported floor stays the same. require.NoError(t, store.SetEarliestVersion(40, false)) require.Equal(t, int64(40), store.GetEarliestVersion()) - require.Equal(t, int64(40), store.HighestEarliestVersion()) } func TestEVMStateStoreDefaultUsesUnifiedDB(t *testing.T) { diff --git a/sei-db/state_db/ss/evm/store.go b/sei-db/state_db/ss/evm/store.go index 81b77cec5b..19d80cac36 100644 --- a/sei-db/state_db/ss/evm/store.go +++ b/sei-db/state_db/ss/evm/store.go @@ -154,16 +154,13 @@ func (s *EVMStateStore) SetLatestVersion(version int64) error { } func (s *EVMStateStore) GetEarliestVersion() int64 { - var minVersion int64 = -1 + var maxVersion int64 for _, db := range s.managedDBs { - if v := db.GetEarliestVersion(); minVersion < 0 || v < minVersion { - minVersion = v + if v := db.GetEarliestVersion(); v > maxVersion { + maxVersion = v } } - if minVersion < 0 { - return 0 - } - return minVersion + return maxVersion } func (s *EVMStateStore) SetEarliestVersion(version int64, ignoreVersion bool) error { @@ -380,7 +377,7 @@ func (s *EVMStateStore) SupportsCheckpoint() bool { if _, barrier := db.(types.DrainBarrier); !barrier { return false } - if _, markerSetter := db.(types.CheckpointMarkerSetter); !markerSetter { + if _, versionSetter := db.(types.CheckpointVersionSetter); !versionSetter { return false } } @@ -433,37 +430,23 @@ func (s *EVMStateStore) ScheduleCheckpoint(destDir string, shouldRun func() bool } } -func (s *EVMStateStore) SetCheckpointMarkers(destDir string, latest, earliest int64) error { +func (s *EVMStateStore) SetCheckpointVersion(destDir string, version int64) error { if !s.separateDBs { db := s.primaryDB() if db == nil { return errors.New("EVM state store has no managed DB to stamp") } - return types.SetCheckpointMarkers(db, destDir, latest, earliest) + return types.SetCheckpointVersion(db, destDir, version) } for _, storeType := range AllEVMStoreTypes() { dest := filepath.Join(destDir, StoreTypeName(storeType)) - if err := types.SetCheckpointMarkers(s.subDBs[storeType], dest, latest, earliest); err != nil { - return fmt.Errorf("set EVM sub-DB %s checkpoint markers: %w", StoreTypeName(storeType), err) + if err := types.SetCheckpointVersion(s.subDBs[storeType], dest, version); err != nil { + return fmt.Errorf("set EVM sub-DB %s checkpoint version: %w", StoreTypeName(storeType), err) } } return nil } -// HighestEarliestVersion reports the furthest any sub-DB has been pruned. Prune -// runs the sub-DBs in parallel and each writes its own marker when it finishes, -// so mid-pass they disagree, and only the highest is a floor every sub-DB can -// honor. -func (s *EVMStateStore) HighestEarliestVersion() int64 { - var highest int64 - for _, db := range s.managedDBs { - if v := db.GetEarliestVersion(); v > highest { - highest = v - } - } - return highest -} - func (s *EVMStateStore) WaitForPendingWrites() { for _, db := range s.managedDBs { if w, ok := db.(interface{ WaitForPendingWrites() }); ok { From 06df0ff36b23bc739fb43568a7b918817636f4d4 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Thu, 13 Aug 2026 11:28:48 -0400 Subject: [PATCH 10/13] fix(seidb): keep prune whole when the earliest marker moves first Raising the earliest-version marker ahead of the deletes fixed the marker a checkpoint inherits, but it moved two failure modes with it. A lost compare-and-swap used to cost only the marker write, because the call sat after the deletes. From the front of the pass it cancelled the whole pass, reclaiming nothing; the racing writer is the state-sync restore path. advanceEarliestVersion retries it, and still returns a persistence failure, because deleting under a marker that only moved in memory would advertise dropped versions after a restart. A pass that failed partway left the marker ahead of its own deletes, and the next pass read that marker as its skip baseline and dropped any store idle since. The rows stayed on disk with no read able to reach them. pruneIncomplete makes the next pass in the same process rescan; the crash window that outlives the flag is recorded on pruneDescending. Also record in the snapshot.go header that this file is what the planned per-SS restructure has to move, and why its count-based retention cannot meet StorageGarbageCollector until then; correct the requestSnapshot comment, which claimed the caller waits for none of the work when it does wait for the staging directories; and drop the pr-summary.md artifact. Co-authored-by: Cursor --- .github/pr-summary.md | 23 ------- sei-db/db_engine/pebbledb/mvcc/db.go | 64 +++++++++++++++++-- .../db_engine/pebbledb/mvcc/db_ascending.go | 25 ++++++-- sei-db/db_engine/pebbledb/mvcc/prune_test.go | 41 ++++++++++++ sei-db/state_db/ss/composite/snapshot.go | 20 +++++- 5 files changed, 136 insertions(+), 37 deletions(-) delete mode 100644 .github/pr-summary.md diff --git a/.github/pr-summary.md b/.github/pr-summary.md deleted file mode 100644 index 53cd0a4daf..0000000000 --- a/.github/pr-summary.md +++ /dev/null @@ -1,23 +0,0 @@ -## Summary - -Add opt-in, exact-version Pebble checkpoints for the State Store. This PR contains snapshot generation and retention only; State Store rollback remains separate. - -- `sei-db/db_engine/types/types.go` and `sei-db/db_engine/pebbledb/mvcc/db.go`: add an ordered drain barrier to the single-writer FIFO. A snapshot barrier runs after version H and before H+1, supports shutdown cancellation, and stamps only the latest-version marker in the checkpoint copy. -- `sei-db/state_db/ss/cosmos/store.go` and `sei-db/state_db/ss/evm/store.go`: expose checkpoint scheduling for Cosmos, unified EVM, and separate EVM sub-databases. Composite publication waits for every required checkpoint. -- `sei-db/state_db/ss/evm/store.go`, `sei-db/state_db/ss/composite/store.go`, and MVCC prune paths: report the highest earliest-version marker across managed databases, allow divergent member floors after recovery, and advance prune's earliest marker before deleting history so snapshots can safely inherit per-DB markers. -- `sei-db/state_db/ss/composite/snapshot.go`: publish checkpoints through a temporary directory, durable rename, and atomic `current` symlink. Startup removes stale staging data, restores `current`, resumes from the newest snapshot, and enforces retention even after a failed publish. -- `sei-db/state_db/ss/composite/snapshot.go`: reject startup when any live SS database cannot hardlink into the snapshot root. Custom Cosmos directories place the root in a sibling `-snapshots` directory. Known limitation: all enabled Cosmos and EVM databases must use one filesystem. -- `sei-db/state_db/ss/composite/store.go` and `sei-cosmos/storev2/rootmulti/store.go`: trigger snapshots from one place, `rootmulti.flush`, for both populated and empty blocks. `ApplyChangesetAsync` schedules nothing, so callers outside the commit path do not inherit a trigger, and a state store that cannot schedule fails the build rather than losing snapshots silently. State-sync import and direct version-marker writes cannot publish a partial snapshot. -- `sei-db/config/*.go`, `app/seidb.go`, and `sei-cosmos/server/config/config.go`: keep snapshots default-off. When enabled, SS mirrors SC's effective block interval, minimum time interval, and retention settings. SS and SC apply independent in-flight gates, so they do not guarantee identical retained heights. -- `sei-db/state_db/ss/composite/snapshot.go` and `snapshot_metrics.go`: allow one snapshot in flight, cancel queued work during close, persist apparent-size metadata outside the publish lock, and export attempt, skip, outcome, duration, in-flight, height, retained-count, and apparent-byte metrics. -- `sei-db/common/utils/path.go`: define the shared default snapshot directory name. Managed snapshot paths have no lease; consumers must coordinate with generation and pruning. - -## Test plan - -- `sei-db/db_engine/pebbledb/mvcc/db_test.go` and `prune_test.go`: checkpoint copies preserve exact latest markers, inherit earliest markers, queued checkpoints cancel before work starts, and prune advances the earliest marker before delete work while preserving pruning coverage. -- `sei-db/state_db/ss/composite/snapshot_test.go`: cover exact labels, exclusion of versions written after the boundary while the write path keeps going, Cosmos-only and EVM-split layouts, idle EVM sub-databases, empty blocks, state-sync isolation, inherited per-store earliest markers, and latest labels across EVM sub-databases. -- `sei-db/state_db/ss/evm/db_test.go` and `sei-db/state_db/ss/composite/recovery_test.go`: cover max earliest-floor reporting and allow recovery with divergent member floors. -- `sei-cosmos/storev2/rootmulti/store_test.go`: cover the commit-path wiring, so a boundary produces a snapshot whether the block carried changesets or not. -- `sei-db/state_db/ss/composite/snapshot_test.go`: cover one-in-flight and minimum-time gates, close during barrier scheduling, custom directories, cross-filesystem rejection, restart resumption, stale staging cleanup, out-of-order publication, failed-publication retention, and persisted size metadata. -- `sei-db/config/ss_config_test.go`, `app/config_fuzz_test.go`, and configuration goldens: cover default-off rollout and effective SC cadence mirroring. -- Verified with `go test -race ./sei-db/db_engine/pebbledb/mvcc ./sei-db/state_db/ss/... ./evmrpc ./sei-cosmos/storev2/...`. diff --git a/sei-db/db_engine/pebbledb/mvcc/db.go b/sei-db/db_engine/pebbledb/mvcc/db.go index c00bc19855..bb77093f04 100644 --- a/sei-db/db_engine/pebbledb/mvcc/db.go +++ b/sei-db/db_engine/pebbledb/mvcc/db.go @@ -58,6 +58,11 @@ const ( // so deleted data accumulates and slows every subsequent prune scan. Allowing // Pebble to burst up to a few compactions clears that backlog. maxConcurrentCompactions = 4 + + // earliestVersionAdvanceAttempts bounds the retries a prune pass makes when + // it raises the earliest-version marker before deleting. See + // Database.advanceEarliestVersion. + earliestVersionAdvanceAttempts = 3 ) var ( @@ -85,6 +90,12 @@ type Database struct { // Used in pruning to skip over stores that have not been updated recently storeKeyDirty sync.Map + // pruneIncomplete records that a pass raised the earliest-version marker and + // then failed before it finished deleting. The next pass cannot use that + // marker as its skip baseline: rows below it are still on disk, and a store + // that has gone idle since would be skipped for as long as it stays idle. + pruneIncomplete atomic.Bool + // Changelog used to support async write streamHandler wal.ChangelogWAL @@ -412,6 +423,29 @@ func (db *Database) GetEarliestVersion() int64 { return db.earliestVersion.Load() } +// advanceEarliestVersion raises the earliest-version marker to target for a +// prune pass that has not deleted anything yet. +// +// SetEarliestVersion fails its compare-and-swap when another writer moves the +// marker at the same moment; in practice that writer is the state-sync restore +// path. Retrying resolves it, and the pass has to keep going rather than return +// the error: this call runs ahead of the deletes, so abandoning the pass here +// reclaims nothing. A marker another writer already raised past target is not a +// failure — SetEarliestVersion reports success for it. +// +// Persistence failures are still returned. Deleting history under a marker that +// only moved in memory would advertise, after a restart, versions the pass has +// already dropped. +func (db *Database) advanceEarliestVersion(target int64) error { + var err error + for range earliestVersionAdvanceAttempts { + if err = db.SetEarliestVersion(target, false); err == nil { + return nil + } + } + return err +} + // Retrieves earliest version from db, if not found, return 0 func retrieveEarliestVersion(db *pebble.DB) (int64, error) { return retrieveVersionKey(db, earliestVersionKey) @@ -682,6 +716,11 @@ func (db *Database) getDescending(storeKey string, targetVersion int64, key []by // NOTE: There is a rare case when a module's keys are skipped during pruning even though // it has been updated. This occurs when that module's keys are updated in between pruning runs, the node after is restarted. // This is not a large issue given the next time that module is updated, it will be properly pruned thereafter. +// NOTE: the marker is raised before the deletes, so a pass that fails partway +// leaves rows below the marker on disk. pruneIncomplete makes the next pass in +// the same process rescan every store to reach them. A crash inside that window +// loses the flag, and those rows stay on disk — unreachable by any read, since +// the marker bounds reads too — until the store is written to again. func (db *Database) pruneDescending(version int64) (_err error) { // Defensive check: ensure database is not closed if db.storage == nil { @@ -700,10 +739,22 @@ func (db *Database) pruneDescending(version int64) (_err error) { }() earliestVersion := version + 1 // we increment by 1 to include the provided version - prevEarliestVersion := db.GetEarliestVersion() - if err := db.SetEarliestVersion(earliestVersion, false); err != nil { + skipBelow := db.GetEarliestVersion() + if err := db.advanceEarliestVersion(earliestVersion); err != nil { return err } + if db.pruneIncomplete.Load() { + // A previous pass raised the marker and then stopped short of its + // deletes, so the marker no longer bounds what is on disk. Scan every + // store to reach the rows it left behind. + skipBelow = 0 + } + db.pruneIncomplete.Store(true) + defer func() { + if _err == nil { + db.pruneIncomplete.Store(false) + } + }() itr, err := db.storage.NewIter(nil) if err != nil { @@ -751,10 +802,11 @@ func (db *Database) pruneDescending(version int64) (_err error) { updated, ok := db.storeKeyDirty.Load(storeKey) versionUpdated, typeOk := updated.(int64) // The marker is advanced before deletes so checkpoints never claim - // history that the prune has already dropped. The skip heuristic must - // still compare against the pre-prune marker; otherwise this pass would - // skip stores whose latest update is at or below the prune height. - if !ok || (typeOk && versionUpdated < prevEarliestVersion) { + // history that the prune has already dropped. skipBelow is the marker + // as it stood before this pass raised it; comparing against the raised + // value would skip every store whose latest update is at or below the + // prune height. + if !ok || (typeOk && versionUpdated < skipBelow) { itr.SeekGE(storePrefix(storeKey + "0")) continue } diff --git a/sei-db/db_engine/pebbledb/mvcc/db_ascending.go b/sei-db/db_engine/pebbledb/mvcc/db_ascending.go index 82e8496029..6932d06fd2 100644 --- a/sei-db/db_engine/pebbledb/mvcc/db_ascending.go +++ b/sei-db/db_engine/pebbledb/mvcc/db_ascending.go @@ -108,10 +108,22 @@ func (db *Database) pruneAscending(version int64) (_err error) { }() earliestVersion := version + 1 // we increment by 1 to include the provided version - prevEarliestVersion := db.GetEarliestVersion() - if err := db.SetEarliestVersion(earliestVersion, false); err != nil { + skipBelow := db.GetEarliestVersion() + if err := db.advanceEarliestVersion(earliestVersion); err != nil { return err } + if db.pruneIncomplete.Load() { + // A previous pass raised the marker and then stopped short of its + // deletes, so the marker no longer bounds what is on disk. Scan every + // store to reach the rows it left behind. + skipBelow = 0 + } + db.pruneIncomplete.Store(true) + defer func() { + if _err == nil { + db.pruneIncomplete.Store(false) + } + }() itr, err := db.storage.NewIter(nil) if err != nil { @@ -159,10 +171,11 @@ func (db *Database) pruneAscending(version int64) (_err error) { updated, ok := db.storeKeyDirty.Load(storeKey) versionUpdated, typeOk := updated.(int64) // The marker is advanced before deletes so checkpoints never claim - // history that the prune has already dropped. The skip heuristic must - // still compare against the pre-prune marker; otherwise this pass would - // skip stores whose latest update is at or below the prune height. - if !ok || (typeOk && versionUpdated < prevEarliestVersion) { + // history that the prune has already dropped. skipBelow is the marker + // as it stood before this pass raised it; comparing against the raised + // value would skip every store whose latest update is at or below the + // prune height. + if !ok || (typeOk && versionUpdated < skipBelow) { itr.SeekGE(storePrefix(storeKey + "0")) continue } diff --git a/sei-db/db_engine/pebbledb/mvcc/prune_test.go b/sei-db/db_engine/pebbledb/mvcc/prune_test.go index 5809585f11..c6d7474a43 100644 --- a/sei-db/db_engine/pebbledb/mvcc/prune_test.go +++ b/sei-db/db_engine/pebbledb/mvcc/prune_test.go @@ -144,3 +144,44 @@ func TestPruneAdvancesEarliestBeforeDeletingHistory(t *testing.T) { require.Equal(t, int64(11), db.GetEarliestVersion(), "earliest marker must advance before a later prune failure") } + +// TestPruneAfterFailedPassRescansIdleStores covers the other half of raising the +// marker first: the pass that follows a failure cannot use that marker as its +// skip baseline. store1 goes idle at version 100, below the raised marker, so +// skipping it would leave v50 on disk with no read able to reach it. +func TestPruneAfterFailedPassRescansIdleStores(t *testing.T) { + const store = "store1" + key := []byte("k") + db := newTestDB(t, true) + + applyVersion(t, db, store, 50, key, []byte("v50")) + applyVersion(t, db, store, 100, key, []byte("v100")) + + // "invalid-mvcc-key" sorts ahead of every "s/k:" store key, so the pass + // fails after raising the marker and before deleting anything. + badKey := []byte("invalid-mvcc-key") + require.NoError(t, db.storage.Set(badKey, []byte("value"), defaultWriteOpts)) + require.Error(t, db.Prune(150)) + require.Equal(t, int64(151), db.GetEarliestVersion()) + require.ElementsMatch(t, []int64{50, 100}, rawVersionsForKey(t, db, store, key), + "the failed pass must not have deleted anything") + + require.NoError(t, db.storage.Delete(badKey, defaultWriteOpts)) + require.NoError(t, db.Prune(150)) + + require.ElementsMatch(t, []int64{100}, rawVersionsForKey(t, db, store, key), + "the pass after a failure must rescan a store the raised marker would skip") +} + +// TestAdvanceEarliestVersionAcceptsAHigherMarker pins the outcome a prune pass +// sees when another writer moves the marker past its target. Raising the marker +// now runs ahead of the deletes, so reporting that as a failure would cost the +// whole pass rather than just the marker write. +func TestAdvanceEarliestVersionAcceptsAHigherMarker(t *testing.T) { + db := newTestDB(t, true) + + require.NoError(t, db.SetEarliestVersion(200, false)) + require.NoError(t, db.advanceEarliestVersion(151)) + require.Equal(t, int64(200), db.GetEarliestVersion(), + "the target must not lower a marker another writer raised past it") +} diff --git a/sei-db/state_db/ss/composite/snapshot.go b/sei-db/state_db/ss/composite/snapshot.go index ba9810c747..70517ed71e 100644 --- a/sei-db/state_db/ss/composite/snapshot.go +++ b/sei-db/state_db/ss/composite/snapshot.go @@ -69,6 +69,19 @@ import ( // a path remaining present across a retention pass. Until a lease API exists, // consumers must stop the node or use external coordination that prevents // pruning before they open or copy a snapshot. +// +// This file is the layer the planned per-SS restructure has to move. The +// lifecycle here — layout, retention, the current symlink, staging and +// publication, restart recovery — is reachable only as a method on +// *CompositeStateStore, and startSnapshotManager requires a checkpointable +// Cosmos store, so an EVM-only store cannot use it as written. The agreed +// direction is for each SS to own its own snapshot creation and retention behind +// gc.PrunableStore, with the composite reduced to fan-out, which also removes the +// second retention path this file adds: prune here is count-based and has no +// ExternalPruning stand-down, so pointing StorageGarbageCollector at SS before +// then would give a store two independent pruners. GetRollbackFloor is the reason +// this waits on the rollback work — count-based retention can delete the snapshot +// a rollback needs, which is the same gap the paragraph above records. const ( // SnapshotsDirName is the directory under data/state_store that holds // online snapshots. @@ -336,8 +349,11 @@ func (m *snapshotManager) finishSnapshot() { // version on the backends and has not enqueued anything above it, so a barrier // placed in each apply queue now captures that backend with everything up to // version applied and nothing after it. The backends reach their barriers -// independently and at different wall-clock times, and the caller waits for -// none of it — enqueueing a barrier costs what enqueueing a changeset costs. +// independently and at different wall-clock times, and the caller waits for none +// of the checkpointing — enqueueing a barrier costs what enqueueing a changeset +// costs. The caller does wait for the staging directories below: one Stat, one +// RemoveAll and one MkdirAll per target, on the commit path and ahead of the SC +// apply. func (m *snapshotManager) requestSnapshot(version int64, start time.Time) error { name := SnapshotDirName(version) finalDir := filepath.Join(m.root, name) From de05dd5dc0f37d19b86de19eece6640edd8ad3c1 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Thu, 13 Aug 2026 12:10:55 -0400 Subject: [PATCH 11/13] refactor(seidb): move checkpoint scheduling out of the engine types Review follow-ups that do not change behavior. CheckpointScheduler, ScheduleCheckpoint, SetCheckpointVersion and ErrCheckpointCanceled move to sei-db/management/checkpoint_scheduler.go. Checkpointable, DrainBarrier and CheckpointVersionSetter stay in db_engine/types: the engines implement them, and SC and dbcache use Checkpointable too. The split is capability with the engine, decision of when a checkpoint runs above it. types.go says where the rest went. Shorten the app.toml text for ss-snapshot-enable to the constraints an operator can act on. The detail it carried moves into the SnapshotEnable Go doc rather than out of the tree: pebbledb-only, the hardlink requirement, what a custom Cosmos SS directory does to the snapshot root, and the apply-goroutine occupancy behind the backpressure. Say on EVMStateStore.ScheduleCheckpoint how five sub-DBs reach the same block without agreeing on anything: each barrier lands after the target block and before the next one in its own queue, and SetCheckpointVersion labels them all with that block. Log the reason a boundary was skipped. A skipped boundary is why an expected snapshot is missing, so it should not be a metric alone. Co-authored-by: Cursor --- sei-db/config/ss_config.go | 18 +++++- sei-db/config/toml.go | 35 ++++------- sei-db/db_engine/pebbledb/mvcc/db_test.go | 9 +-- sei-db/db_engine/types/types.go | 48 ++------------- sei-db/management/checkpoint_scheduler.go | 59 +++++++++++++++++++ sei-db/state_db/ss/composite/snapshot.go | 19 +++--- sei-db/state_db/ss/composite/snapshot_test.go | 3 +- sei-db/state_db/ss/cosmos/store.go | 5 +- sei-db/state_db/ss/evm/store.go | 23 +++++--- 9 files changed, 127 insertions(+), 92 deletions(-) create mode 100644 sei-db/management/checkpoint_scheduler.go diff --git a/sei-db/config/ss_config.go b/sei-db/config/ss_config.go index 03b22aed6a..54b9cfa4d2 100644 --- a/sei-db/config/ss_config.go +++ b/sei-db/config/ss_config.go @@ -66,13 +66,25 @@ type StateStoreConfig struct { UseDefaultComparer bool `mapstructure:"use-default-comparer"` // SnapshotEnable controls whether the state store takes periodic online - // snapshots. Snapshots are Pebble checkpoints (hardlink trees), so each - // retained snapshot pins the SSTs it references and prevents compaction + // snapshots. Snapshots are Pebble checkpoints (hardlink trees), so the + // backend must be pebbledb and every SS database must be able to hardlink + // into the snapshot root. Startup fails on either rather than running + // without snapshots. A custom Cosmos SS directory moves the snapshot root + // beside that directory, which keeps the link inside one filesystem. + // + // Taking a snapshot occupies each backend's SS apply goroutine for the WAL + // flush, the filesystem sync, and the checkpoint. No data is copied up + // front, but a queue that fills during that window applies write + // backpressure. + // + // Each retained snapshot pins the SSTs it references and prevents compaction // from reclaiming them. Steady-state disk overhead is therefore the // compaction churn accumulated over SnapshotInterval blocks, per retained // snapshot — significant on a multi-TB state store. Managed snapshots have // no lease in this release, so consumers must quiesce generation and pruning - // before using a snapshot directory. + // before using a snapshot directory. Attempts, skips, outcomes, duration, + // in-flight state, height, count, and apparent bytes are exported as + // ss_snapshot_* metrics. // defaults to false SnapshotEnable bool `mapstructure:"snapshot-enable"` diff --git a/sei-db/config/toml.go b/sei-db/config/toml.go index 9966cfc120..b993158618 100644 --- a/sei-db/config/toml.go +++ b/sei-db/config/toml.go @@ -140,29 +140,18 @@ ss-import-num-workers = {{ .StateStore.ImportNumWorkers }} # Applies when ss-backend = "pebbledb". Default: false. ss-enable-read-write-metrics = {{ .StateStore.EnableReadWriteMetrics }} -# SnapshotEnable controls whether the state store takes periodic online -# snapshots. The cadence is not configurable here: it mirrors the state-commit -# snapshot interval, minimum time interval, and retention settings. SC and SS -# apply their in-flight gates independently, so a skipped boundary can differ. -# Snapshots are PebbleDB checkpoints, i.e. hardlink trees, so ss-backend must be -# pebbledb. Enabling this on any other backend fails startup instead of running -# without snapshots. Creating one blocks each backend's SS apply worker for the -# full WAL flush, filesystem sync, and checkpoint operation; a full async queue -# then applies write backpressure. It does not copy data up front. Startup also -# rejects snapshot configurations where a live SS database and the snapshot root -# cannot hardlink to each other. Each enabled Cosmos and EVM SS database must -# therefore use the same filesystem. A custom Cosmos SS directory moves the -# snapshot root beside that directory. -# Retained snapshots pin referenced SSTs, so compaction cannot reclaim them. -# Expect steady-state disk overhead on the order of the compaction churn over -# one snapshot interval per retained snapshot, which is substantial on a -# multi-TB state store. -# Managed snapshot directories have no lease in this release. Do not pack an -# archive or serve state sync directly from them while the node is running: -# retention can remove a directory during use. Stop the node, or use external -# coordination that prevents pruning, before consuming a snapshot. Snapshot -# attempts, skips, outcomes, duration, in-flight state, height, count, and -# apparent bytes are exported through ss_snapshot_* metrics. Default: false. +# SnapshotEnable turns on periodic online state-store snapshots. The cadence is +# not configurable here: it mirrors the state-commit snapshot settings. +# Two configurations fail startup rather than run without snapshots: an +# ss-backend other than "pebbledb", and SS databases that cannot hardlink into +# the snapshot root, which needs every SS database and that root on one +# filesystem. +# Each retained snapshot pins the SST files it references against compaction, so +# budget the write churn of one snapshot interval per retained snapshot. This is +# substantial on a multi-TB state store. +# Snapshot directories have no lease in this release: stop the node, or prevent +# pruning by other means, before you copy one. +# Cost and progress are exported as ss_snapshot_* metrics. Default: false. ss-snapshot-enable = {{ .StateStore.SnapshotEnable }} # EVMDBDirectory defines the directory for the optional EVM state-store DB(s). diff --git a/sei-db/db_engine/pebbledb/mvcc/db_test.go b/sei-db/db_engine/pebbledb/mvcc/db_test.go index febcf5cfcc..ea914203e5 100644 --- a/sei-db/db_engine/pebbledb/mvcc/db_test.go +++ b/sei-db/db_engine/pebbledb/mvcc/db_test.go @@ -11,6 +11,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/config" sstest "github.com/sei-protocol/sei-chain/sei-db/db_engine/test" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" + "github.com/sei-protocol/sei-chain/sei-db/management" ) func TestStorageTestSuite(t *testing.T) { @@ -62,13 +63,13 @@ func TestVersionedCheckpointPreservesFutureLiveMarker(t *testing.T) { dest := filepath.Join(t.TempDir(), "snapshot") done := make(chan error, 1) - types.ScheduleCheckpoint(store, dest, nil, func(err error) { + management.ScheduleCheckpoint(store, dest, nil, func(err error) { done <- err }) require.NoError(t, <-done) // The caller stamps only the label. Earliest is inherited from the // checkpointed DB because prune advances it before deleting history. - require.NoError(t, types.SetCheckpointVersion(store, dest, 5)) + require.NoError(t, management.SetCheckpointVersion(store, dest, 5)) require.Equal(t, int64(10), store.GetLatestVersion()) require.Equal(t, int64(4), store.GetEarliestVersion()) @@ -96,10 +97,10 @@ func TestScheduledCheckpointCanBeCanceledAtBarrier(t *testing.T) { dest := filepath.Join(t.TempDir(), "snapshot") done := make(chan error, 1) - types.ScheduleCheckpoint(store, dest, func() bool { return false }, func(err error) { + management.ScheduleCheckpoint(store, dest, func() bool { return false }, func(err error) { done <- err }) - require.ErrorIs(t, <-done, types.ErrCheckpointCanceled) + require.ErrorIs(t, <-done, management.ErrCheckpointCanceled) require.NoDirExists(t, dest) } diff --git a/sei-db/db_engine/types/types.go b/sei-db/db_engine/types/types.go index 92f6cc8f70..24856dc539 100644 --- a/sei-db/db_engine/types/types.go +++ b/sei-db/db_engine/types/types.go @@ -1,8 +1,6 @@ package types import ( - "errors" - "fmt" "io" "github.com/sei-protocol/sei-chain/sei-db/proto" @@ -141,48 +139,10 @@ type DrainBarrier interface { ScheduleAtDrain(fn func()) } -// CheckpointScheduler coordinates checkpoints for stores with in-flight writes. -type CheckpointScheduler interface { - SupportsCheckpoint() bool - ScheduleCheckpoint(destDir string, shouldRun func() bool, done func(error)) - SetCheckpointVersion(destDir string, version int64) error -} - -// ErrCheckpointCanceled reports that a queued checkpoint was canceled before -// it started. -var ErrCheckpointCanceled = errors.New("state store checkpoint canceled") - -// ScheduleCheckpoint checkpoints an engine after all writes already enqueued -// on it have been applied. -func ScheduleCheckpoint(db StateStore, destDir string, shouldRun func() bool, done func(error)) { - cp, ok := db.(Checkpointable) - if !ok { - done(fmt.Errorf("state store backend %T does not support checkpoints", db)) - return - } - barrier, ok := db.(DrainBarrier) - if !ok { - done(fmt.Errorf("state store backend %T does not support ordered checkpoint barriers", db)) - return - } - barrier.ScheduleAtDrain(func() { - if shouldRun != nil && !shouldRun() { - done(ErrCheckpointCanceled) - return - } - done(cp.Checkpoint(destDir)) - }) -} - -// SetCheckpointVersion makes a completed checkpoint self-describing without -// changing the live database. -func SetCheckpointVersion(db StateStore, destDir string, version int64) error { - setter, ok := db.(CheckpointVersionSetter) - if !ok { - return fmt.Errorf("state store backend %T cannot set checkpoint version", db) - } - return setter.SetCheckpointVersion(destDir, version) -} +// The three interfaces above are engine capabilities. Deciding when a checkpoint +// runs, and what version it is labeled with, is coordination rather than engine +// behavior and lives in sei-db/management: CheckpointScheduler, +// ScheduleCheckpoint, SetCheckpointVersion and ErrCheckpointCanceled. // --------------------------------------------------------------------------- // SS DB layer diff --git a/sei-db/management/checkpoint_scheduler.go b/sei-db/management/checkpoint_scheduler.go new file mode 100644 index 0000000000..8e68bb92b9 --- /dev/null +++ b/sei-db/management/checkpoint_scheduler.go @@ -0,0 +1,59 @@ +// Package management holds the coordination layer above the DB engines: work +// that decides when an engine-level operation runs, rather than how the engine +// performs it. +package management + +import ( + "errors" + "fmt" + + "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" +) + +// CheckpointScheduler coordinates checkpoints for stores with in-flight writes. +// +// The engine-side capabilities this builds on — types.Checkpointable, +// types.DrainBarrier and types.CheckpointVersionSetter — stay with the engines +// that implement them. What lives here is the decision of when a checkpoint runs +// and what version it is labeled with. +type CheckpointScheduler interface { + SupportsCheckpoint() bool + ScheduleCheckpoint(destDir string, shouldRun func() bool, done func(error)) + SetCheckpointVersion(destDir string, version int64) error +} + +// ErrCheckpointCanceled reports that a queued checkpoint was canceled before +// it started. +var ErrCheckpointCanceled = errors.New("state store checkpoint canceled") + +// ScheduleCheckpoint checkpoints an engine after all writes already enqueued +// on it have been applied. +func ScheduleCheckpoint(db types.StateStore, destDir string, shouldRun func() bool, done func(error)) { + cp, ok := db.(types.Checkpointable) + if !ok { + done(fmt.Errorf("state store backend %T does not support checkpoints", db)) + return + } + barrier, ok := db.(types.DrainBarrier) + if !ok { + done(fmt.Errorf("state store backend %T does not support ordered checkpoint barriers", db)) + return + } + barrier.ScheduleAtDrain(func() { + if shouldRun != nil && !shouldRun() { + done(ErrCheckpointCanceled) + return + } + done(cp.Checkpoint(destDir)) + }) +} + +// SetCheckpointVersion makes a completed checkpoint self-describing without +// changing the live database. +func SetCheckpointVersion(db types.StateStore, destDir string, version int64) error { + setter, ok := db.(types.CheckpointVersionSetter) + if !ok { + return fmt.Errorf("state store backend %T cannot set checkpoint version", db) + } + return setter.SetCheckpointVersion(destDir, version) +} diff --git a/sei-db/state_db/ss/composite/snapshot.go b/sei-db/state_db/ss/composite/snapshot.go index 70517ed71e..34cf372cf8 100644 --- a/sei-db/state_db/ss/composite/snapshot.go +++ b/sei-db/state_db/ss/composite/snapshot.go @@ -14,7 +14,7 @@ import ( "time" "github.com/sei-protocol/sei-chain/sei-db/common/utils" - "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" + "github.com/sei-protocol/sei-chain/sei-db/management" ) // Online state-store snapshots. Every SnapshotInterval blocks the store takes a @@ -149,8 +149,8 @@ type snapshotManager struct { keepRecent int minTime time.Duration - cosmosScheduler types.CheckpointScheduler - evmScheduler types.CheckpointScheduler + cosmosScheduler management.CheckpointScheduler + evmScheduler management.CheckpointScheduler snapshotSizes map[int64]int64 mu sync.Mutex @@ -173,7 +173,7 @@ type snapshotManager struct { } type checkpointTarget struct { - store types.CheckpointScheduler + store management.CheckpointScheduler dest string } @@ -185,13 +185,13 @@ func (s *CompositeStateStore) startSnapshotManager(root string, sourceDirs []str if s.config.SnapshotInterval <= 0 { return nil } - cosmosScheduler, ok := s.cosmosStore.(types.CheckpointScheduler) + cosmosScheduler, ok := s.cosmosStore.(management.CheckpointScheduler) if !ok || !cosmosScheduler.SupportsCheckpoint() { return fmt.Errorf("cosmos backend %q does not support checkpoints", s.config.Backend) } - var evmScheduler types.CheckpointScheduler + var evmScheduler management.CheckpointScheduler if s.evmStore != nil { - evmScheduler, ok = s.evmStore.(types.CheckpointScheduler) + evmScheduler, ok = s.evmStore.(management.CheckpointScheduler) if !ok || !evmScheduler.SupportsCheckpoint() { return fmt.Errorf("EVM backend %q does not support checkpoints", s.config.Backend) } @@ -315,6 +315,9 @@ func (m *snapshotManager) maybeSnapshot(version int64) { if !accepted { if skipReason != "" { recordSnapshotSkipped(skipReason) + // A skipped boundary is the reason a snapshot an operator expected is + // not on disk, so name the gate rather than leaving only a metric. + logger.Info("skipping state store snapshot", "version", version, "reason", skipReason) } return } @@ -436,7 +439,7 @@ func (m *snapshotManager) startPublish( defer m.publishing.Done() defer m.finishSnapshot() if checkpointErr != nil { - if errors.Is(checkpointErr, types.ErrCheckpointCanceled) { + if errors.Is(checkpointErr, management.ErrCheckpointCanceled) { recordSnapshotCompletion(start, "canceled") } else { recordSnapshotCompletion(start, "failure") diff --git a/sei-db/state_db/ss/composite/snapshot_test.go b/sei-db/state_db/ss/composite/snapshot_test.go index 11e3665880..565a9fa45a 100644 --- a/sei-db/state_db/ss/composite/snapshot_test.go +++ b/sei-db/state_db/ss/composite/snapshot_test.go @@ -8,6 +8,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/config" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" + "github.com/sei-protocol/sei-chain/sei-db/management" "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/cosmos" "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/evm" @@ -50,7 +51,7 @@ func (s *controlledSnapshotScheduler) ScheduleCheckpoint( } s.pending <- func() { if !shouldRun() { - done(types.ErrCheckpointCanceled) + done(management.ErrCheckpointCanceled) return } s.checkpointCalls++ diff --git a/sei-db/state_db/ss/cosmos/store.go b/sei-db/state_db/ss/cosmos/store.go index 02aeaf915c..c512335e49 100644 --- a/sei-db/state_db/ss/cosmos/store.go +++ b/sei-db/state_db/ss/cosmos/store.go @@ -4,6 +4,7 @@ import ( dbm "github.com/tendermint/tm-db" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" + "github.com/sei-protocol/sei-chain/sei-db/management" "github.com/sei-protocol/sei-chain/sei-db/proto" ) @@ -85,11 +86,11 @@ func (s *CosmosStateStore) SupportsCheckpoint() bool { } func (s *CosmosStateStore) ScheduleCheckpoint(destDir string, shouldRun func() bool, done func(error)) { - types.ScheduleCheckpoint(s.db, destDir, shouldRun, done) + management.ScheduleCheckpoint(s.db, destDir, shouldRun, done) } func (s *CosmosStateStore) SetCheckpointVersion(destDir string, version int64) error { - return types.SetCheckpointVersion(s.db, destDir, version) + return management.SetCheckpointVersion(s.db, destDir, version) } func (s *CosmosStateStore) WaitForPendingWrites() { diff --git a/sei-db/state_db/ss/evm/store.go b/sei-db/state_db/ss/evm/store.go index 19d80cac36..4c20aa6ceb 100644 --- a/sei-db/state_db/ss/evm/store.go +++ b/sei-db/state_db/ss/evm/store.go @@ -12,6 +12,7 @@ import ( commonevm "github.com/sei-protocol/sei-chain/sei-db/common/keys" "github.com/sei-protocol/sei-chain/sei-db/config" "github.com/sei-protocol/sei-chain/sei-db/db_engine/types" + "github.com/sei-protocol/sei-chain/sei-db/management" "github.com/sei-protocol/sei-chain/sei-db/proto" "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/backend" ) @@ -384,9 +385,17 @@ func (s *EVMStateStore) SupportsCheckpoint() bool { return len(s.managedDBs) > 0 } -// ScheduleCheckpoint places one barrier on each managed apply queue. A sub-DB -// that did not receive a change at the target block stays at its last version, -// which is the correct state for that block. +// ScheduleCheckpoint places one barrier on each managed apply queue. +// +// Every sub-DB checkpoints at the same block without the sub-DBs having to agree +// on anything. The caller runs this after it has enqueued the target block on +// every sub-DB and before it enqueues any later block, so each barrier lands at +// the same point in its own queue: after that block and before the next one. A +// sub-DB then checkpoints its own state as of that block. Wall-clock times +// differ, and no lock is shared. A sub-DB that received no change at the target +// block stays at its last version, which is that sub-DB's correct state for the +// block. SetCheckpointVersion afterwards labels every sub-DB with the same +// block, so a reopened snapshot reports one version rather than five. func (s *EVMStateStore) ScheduleCheckpoint(destDir string, shouldRun func() bool, done func(error)) { if !s.separateDBs { db := s.primaryDB() @@ -397,7 +406,7 @@ func (s *EVMStateStore) ScheduleCheckpoint(destDir string, shouldRun func() bool done(errors.New("EVM state store has no managed DB to checkpoint")) return } - types.ScheduleCheckpoint(db, destDir, shouldRun, done) + management.ScheduleCheckpoint(db, destDir, shouldRun, done) return } @@ -415,7 +424,7 @@ func (s *EVMStateStore) ScheduleCheckpoint(destDir string, shouldRun func() bool for _, storeType := range storeTypes { name := StoreTypeName(storeType) dest := filepath.Join(destDir, name) - types.ScheduleCheckpoint(s.subDBs[storeType], dest, shouldRun, func(err error) { + management.ScheduleCheckpoint(s.subDBs[storeType], dest, shouldRun, func(err error) { mu.Lock() if err != nil && firstErr == nil { firstErr = fmt.Errorf("checkpoint EVM sub-DB %s: %w", name, err) @@ -436,11 +445,11 @@ func (s *EVMStateStore) SetCheckpointVersion(destDir string, version int64) erro if db == nil { return errors.New("EVM state store has no managed DB to stamp") } - return types.SetCheckpointVersion(db, destDir, version) + return management.SetCheckpointVersion(db, destDir, version) } for _, storeType := range AllEVMStoreTypes() { dest := filepath.Join(destDir, StoreTypeName(storeType)) - if err := types.SetCheckpointVersion(s.subDBs[storeType], dest, version); err != nil { + if err := management.SetCheckpointVersion(s.subDBs[storeType], dest, version); err != nil { return fmt.Errorf("set EVM sub-DB %s checkpoint version: %w", StoreTypeName(storeType), err) } } From 144059b93368c83c39c7d12215aa55e5d3672f3c Mon Sep 17 00:00:00 2001 From: blindchaser Date: Thu, 13 Aug 2026 14:59:53 -0400 Subject: [PATCH 12/13] docs(seidb): describe SS snapshots as rollback inputs State store snapshots are rollback replay start points, not standalone export artifacts. Record the intended SC FlatKV-style restore model: restore from a state-store snapshot and replay the state WAL forward, while state sync rebuilds SS from the SC snapshot stream. Update the snapshot manager header and SnapshotEnable docs to match that model, keep the no-lease warning scoped to future node-external tools, and clarify that per-SS restructuring should mirror FlatKV once rollback can honor the floor it reports. Co-authored-by: Cursor --- sei-db/config/ss_config.go | 9 ++-- sei-db/config/toml.go | 4 +- sei-db/state_db/ss/composite/snapshot.go | 64 +++++++++++++++--------- 3 files changed, 46 insertions(+), 31 deletions(-) diff --git a/sei-db/config/ss_config.go b/sei-db/config/ss_config.go index 54b9cfa4d2..567faf9138 100644 --- a/sei-db/config/ss_config.go +++ b/sei-db/config/ss_config.go @@ -80,10 +80,11 @@ type StateStoreConfig struct { // Each retained snapshot pins the SSTs it references and prevents compaction // from reclaiming them. Steady-state disk overhead is therefore the // compaction churn accumulated over SnapshotInterval blocks, per retained - // snapshot — significant on a multi-TB state store. Managed snapshots have - // no lease in this release, so consumers must quiesce generation and pruning - // before using a snapshot directory. Attempts, skips, outcomes, duration, - // in-flight state, height, count, and apparent bytes are exported as + // snapshot — significant on a multi-TB state store. Managed snapshots are + // rollback restore points, not an archive format. They have no lease in this + // release, so node-external tools must not resolve a snapshot path and open it + // later without first adding a hold mechanism. Attempts, skips, outcomes, + // duration, in-flight state, height, count, and apparent bytes are exported as // ss_snapshot_* metrics. // defaults to false SnapshotEnable bool `mapstructure:"snapshot-enable"` diff --git a/sei-db/config/toml.go b/sei-db/config/toml.go index b993158618..6847fd3650 100644 --- a/sei-db/config/toml.go +++ b/sei-db/config/toml.go @@ -149,8 +149,8 @@ ss-enable-read-write-metrics = {{ .StateStore.EnableReadWriteMetrics }} # Each retained snapshot pins the SST files it references against compaction, so # budget the write churn of one snapshot interval per retained snapshot. This is # substantial on a multi-TB state store. -# Snapshot directories have no lease in this release: stop the node, or prevent -# pruning by other means, before you copy one. +# Snapshot directories are rollback restore points, not an archive format. +# They have no lease, so do not build tools that resolve one and open it later. # Cost and progress are exported as ss_snapshot_* metrics. Default: false. ss-snapshot-enable = {{ .StateStore.SnapshotEnable }} diff --git a/sei-db/state_db/ss/composite/snapshot.go b/sei-db/state_db/ss/composite/snapshot.go index 34cf372cf8..cd9be00f4a 100644 --- a/sei-db/state_db/ss/composite/snapshot.go +++ b/sei-db/state_db/ss/composite/snapshot.go @@ -25,6 +25,12 @@ import ( // applies backpressure until the checkpoint finishes. The result is an // immutable, crash-consistent image of the query store. // +// These snapshots are an input to SS rollback, not an export format. The +// intended restore model matches SC FlatKV: restore from an SS snapshot, then +// replay the state WAL forward to the target height. State sync imports the SC +// snapshot stream and rebuilds SS from that stream; it does not consume these +// SS snapshot directories. +// // On-disk layout under the snapshot root. By default the root is // /data/state_store/snapshots. A custom Cosmos SS directory moves it // to the sibling -snapshots directory so Pebble can use hardlinks. @@ -37,12 +43,15 @@ import ( // / (when EVM sub-DBs are separate) // // Snapshots are eligible at the same interval boundaries and minimum time -// cadence as state commit. Each layer applies its in-flight gate independently, -// so a skipped boundary can differ. For every accepted SS snapshot, the label -// is exact: it is the version the write path had just handed to the backends -// when the snapshot was requested. Placing a barrier in each backend's apply -// queue — rather than sampling what the backends had applied — makes that label -// exact without the request having to wait. See requestSnapshot. +// cadence as state commit. This composite implementation uses one trigger and +// one current link for all member stores, so its member snapshots share a label. +// That same label is a property of this layout, not a rollback requirement: +// rollback can replay the state WAL from each store's own nearest snapshot. For +// every accepted SS snapshot, the label is exact: it is the version the write +// path had just handed to the backends when the snapshot was requested. Placing +// a barrier in each backend's apply queue — rather than sampling what the +// backends had applied — makes that label exact without the request having to +// wait. See requestSnapshot. // // The barrier orders only the async block-commit queues. Import, recovery, // pruning, and direct version-marker writes bypass those queues and must not @@ -56,32 +65,37 @@ import ( // range the DB has already dropped. Reopening a snapshot with different member // floors is allowed; the composite reports the highest floor any member carries. // -// SS rollback is not part of this feature, and the two do not compose yet. A -// rollback leaves lastRequested at the pre-rollback high-water mark, so the -// re-executed boundaries are read as repeats and skipped, and the already -// published snapshot-NNNNN directories keep labels that belong to the abandoned -// chain. Nothing in the layout tells a consumer of current that this happened, -// so the snapshot root must be cleared by hand after a rollback. State-syncing -// to a height below existing snapshots in a reused home directory has the same -// shape and needs the same manual clearing. +// SS rollback is not implemented in this feature. When it is added, it should +// use these snapshots the same way SC FlatKV does: restore from a snapshot +// boundary, then replay the state WAL forward. Until then, rolling back or +// state-syncing to a lower height in a reused home directory leaves two stale +// facts behind: lastRequested still carries the old high-water mark, so repeated +// boundaries can be skipped, and already published snapshot-NNNNN directories +// keep labels from the abandoned chain. Clear the snapshot root by hand in that +// case. // -// Managed snapshot directories have no lease. A live consumer must not rely on -// a path remaining present across a retention pass. Until a lease API exists, -// consumers must stop the node or use external coordination that prevents -// pruning before they open or copy a snapshot. +// Managed snapshot directories have no lease because they are not a node-external +// consumption API. Retention may remove any snapshot that rollback does not need. +// If a future tool opens or copies these directories directly, it must first add +// a lease or other hold mechanism. // // This file is the layer the planned per-SS restructure has to move. The // lifecycle here — layout, retention, the current symlink, staging and // publication, restart recovery — is reachable only as a method on // *CompositeStateStore, and startSnapshotManager requires a checkpointable // Cosmos store, so an EVM-only store cannot use it as written. The agreed -// direction is for each SS to own its own snapshot creation and retention behind -// gc.PrunableStore, with the composite reduced to fan-out, which also removes the -// second retention path this file adds: prune here is count-based and has no -// ExternalPruning stand-down, so pointing StorageGarbageCollector at SS before -// then would give a store two independent pruners. GetRollbackFloor is the reason -// this waits on the rollback work — count-based retention can delete the snapshot -// a rollback needs, which is the same gap the paragraph above records. +// direction is for each SS to own its own snapshot root, current link, creation, +// and retention behind gc.PrunableStore, mirroring SC FlatKV. Composite mode can +// then fan out to Cosmos SS and EVM SS, while Giga can use EVM SS directly. That +// also removes the second retention path this file adds: prune here is +// count-based and has no ExternalPruning stand-down, so pointing +// StorageGarbageCollector at SS before then would give a store two independent +// pruners. The shape waits on rollback not because GetRollbackFloor is unknown; +// SC FlatKV already defines that floor. It waits because gc.PrunableStore must +// not report a floor above what the store can actually restore to. The current +// link semantics should change with that work too: this implementation points to +// the newest published snapshot, while FlatKV's current link points to the +// active snapshot that open/rollback clones and replays from. const ( // SnapshotsDirName is the directory under data/state_store that holds // online snapshots. From 9d0670194aad8fbc66e63a879c21d6cda132b04c Mon Sep 17 00:00:00 2001 From: blindchaser Date: Thu, 13 Aug 2026 15:23:23 -0400 Subject: [PATCH 13/13] fix(seidb): persist the earliest marker before publishing it Serialize earliest-version writers and update the in-memory marker only after Pebble accepts the metadata write. This prevents a failed persistence attempt from looking successful on a later same-height prune and allowing history deletion under a marker that will regress after restart. Add a read-only Pebble regression test that proves the persistence error is returned and the in-memory marker stays unchanged. Co-authored-by: Cursor --- sei-db/db_engine/pebbledb/mvcc/db.go | 57 ++++++++------------ sei-db/db_engine/pebbledb/mvcc/prune_test.go | 23 ++++++++ 2 files changed, 44 insertions(+), 36 deletions(-) diff --git a/sei-db/db_engine/pebbledb/mvcc/db.go b/sei-db/db_engine/pebbledb/mvcc/db.go index bb77093f04..d4ea4c2f94 100644 --- a/sei-db/db_engine/pebbledb/mvcc/db.go +++ b/sei-db/db_engine/pebbledb/mvcc/db.go @@ -58,11 +58,6 @@ const ( // so deleted data accumulates and slows every subsequent prune scan. Allowing // Pebble to burst up to a few compactions clears that backlog. maxConcurrentCompactions = 4 - - // earliestVersionAdvanceAttempts bounds the retries a prune pass makes when - // it raises the earliest-version marker before deleting. See - // Database.advanceEarliestVersion. - earliestVersionAdvanceAttempts = 3 ) var ( @@ -76,7 +71,8 @@ type Database struct { asyncWriteWG sync.WaitGroup config config.StateStoreConfig // Earliest version for db after pruning - earliestVersion atomic.Int64 + earliestVersion atomic.Int64 + earliestVersionMu sync.Mutex // Latest version for db latestVersion atomic.Int64 // descending indicates whether this DB uses descending-version MVCC @@ -401,21 +397,21 @@ func (db *Database) SetEarliestVersion(version int64, ignoreVersion bool) error if version < 0 { return fmt.Errorf("version must be non-negative") } + db.earliestVersionMu.Lock() + defer db.earliestVersionMu.Unlock() + earliestVersion := db.earliestVersion.Load() - if version > earliestVersion || ignoreVersion { - swapped := db.earliestVersion.CompareAndSwap(earliestVersion, version) - if swapped { - var ts [VersionSize]byte - binary.LittleEndian.PutUint64(ts[:], uint64(version)) - err := db.storage.Set([]byte(earliestVersionKey), ts[:], defaultWriteOpts) - if err == nil { - db.operationMetrics.AddWrite(1) - } - return err - } else { - return fmt.Errorf("failed to set earliest version to: %d", version) - } + if version <= earliestVersion && !ignoreVersion { + return nil } + + var ts [VersionSize]byte + binary.LittleEndian.PutUint64(ts[:], uint64(version)) + if err := db.storage.Set([]byte(earliestVersionKey), ts[:], defaultWriteOpts); err != nil { + return err + } + db.earliestVersion.Store(version) + db.operationMetrics.AddWrite(1) return nil } @@ -426,24 +422,13 @@ func (db *Database) GetEarliestVersion() int64 { // advanceEarliestVersion raises the earliest-version marker to target for a // prune pass that has not deleted anything yet. // -// SetEarliestVersion fails its compare-and-swap when another writer moves the -// marker at the same moment; in practice that writer is the state-sync restore -// path. Retrying resolves it, and the pass has to keep going rather than return -// the error: this call runs ahead of the deletes, so abandoning the pass here -// reclaims nothing. A marker another writer already raised past target is not a -// failure — SetEarliestVersion reports success for it. -// -// Persistence failures are still returned. Deleting history under a marker that -// only moved in memory would advertise, after a restart, versions the pass has -// already dropped. +// SetEarliestVersion serializes competing writers and changes the in-memory +// marker only after Pebble accepts the metadata write. A persistence failure is +// therefore returned with both markers unchanged. Deleting history under a +// marker that only moved in memory would advertise, after a restart, versions +// the pass has already dropped. func (db *Database) advanceEarliestVersion(target int64) error { - var err error - for range earliestVersionAdvanceAttempts { - if err = db.SetEarliestVersion(target, false); err == nil { - return nil - } - } - return err + return db.SetEarliestVersion(target, false) } // Retrieves earliest version from db, if not found, return 0 diff --git a/sei-db/db_engine/pebbledb/mvcc/prune_test.go b/sei-db/db_engine/pebbledb/mvcc/prune_test.go index c6d7474a43..c7909c426c 100644 --- a/sei-db/db_engine/pebbledb/mvcc/prune_test.go +++ b/sei-db/db_engine/pebbledb/mvcc/prune_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/cockroachdb/pebble/v2" + "github.com/cockroachdb/pebble/v2/vfs" "github.com/stretchr/testify/require" "github.com/sei-protocol/sei-chain/sei-db/config" @@ -185,3 +186,25 @@ func TestAdvanceEarliestVersionAcceptsAHigherMarker(t *testing.T) { require.Equal(t, int64(200), db.GetEarliestVersion(), "the target must not lower a marker another writer raised past it") } + +// TestAdvanceEarliestVersionReturnsPersistenceFailure pins that Pebble must +// accept the metadata write before the in-memory marker moves. Otherwise a +// later call with the same target would see the target in memory, return nil, +// and let pruning delete history under a marker that was never persisted. +func TestAdvanceEarliestVersionReturnsPersistenceFailure(t *testing.T) { + fs := vfs.NewMem() + storage, err := pebble.Open("db", &pebble.Options{FS: fs}) + require.NoError(t, err) + require.NoError(t, storage.Close()) + + storage, err = pebble.Open("db", &pebble.Options{FS: fs, ReadOnly: true}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, storage.Close()) }) + + db := &Database{storage: storage} + err = db.advanceEarliestVersion(151) + + require.ErrorIs(t, err, pebble.ErrReadOnly) + require.Zero(t, db.GetEarliestVersion(), + "a failed metadata write must not move the in-memory marker") +}