feat(seidb): add state store snapshots

app/config_fuzz_test.go

@@ -23,11 +23,12 @@ import (
 //
 //   - parseSCConfigs guards almost every read with `if v := opts.Get(k); v != nil`,
 //     so a key absent from an older app.toml keeps its non-zero in-code default.
-//   - parseSSConfigs guards nothing. Every read is a bare cast of a possibly-nil
-//     value, so an absent key resolves to the zero value and overwrites the
-//     default. ss-keep-recent becomes 0 (keep everything, unbounded disk growth),
+//   - parseSSConfigs leaves every legacy read unguarded. Each is a bare cast of a
+//     possibly-nil value, so an absent key resolves to the zero value and overwrites
+//     the default. ss-keep-recent becomes 0 (keep everything, unbounded disk growth),
 //     ss-async-write-buffer becomes 0 (synchronous writes), ss-backend becomes ""
-//     and ss-enable becomes false.
+//     and ss-enable becomes false. ss-snapshot-enable is the one guarded read, so an
+//     app.toml written before SS snapshots existed keeps the in-code default.
 //
 // Neither reader returns an error, so nothing about the second case is visible at
 // boot. It is recorded here as behavior rather than reported as a defect: the
@@ -84,8 +85,8 @@ var scKeys = []configtest.KeySpec{
 	},
 }
 
-// ssKeys is the [state-store] read-site manifest. Every row is unguarded and
-// unchecked — the section has no presence checks at all.
+// ssKeys is the [state-store] read-site manifest. Every row is unchecked;
+// SnapshotEnable is guarded while the legacy rows remain unguarded.
 //
 // StateStoreConfig also carries KeepLastVersion and UseDefaultComparer, which are
 // absent here because parseSSConfigs reads neither: they hold their in-code
@@ -112,6 +113,10 @@ var ssKeys = []configtest.KeySpec{
 	{Key: FlagSSImportNumWorkers, Path: "ImportNumWorkers", Cast: configtest.CastInt, Unguarded: true},
 	{Key: FlagSSDirectory, Path: "DBDirectory", Cast: configtest.CastString, Unguarded: true},
 	{Key: FlagSSReadWriteMetrics, Path: "EnableReadWriteMetrics", Cast: configtest.CastBool, Unguarded: true},
+	{
+		Key: FlagSSSnapshotEnable, Path: "SnapshotEnable", Cast: configtest.CastBool,
+		Why: "guarded so app.toml files created before SS snapshots keep the default-off rollout",
+	},
 	{Key: FlagEVMSSDirectory, Path: "EVMDBDirectory", Cast: configtest.CastString, Unguarded: true},
 	{Key: FlagEVMSSSeparateDBs, Path: "SeparateEVMSubDBs", Cast: configtest.CastBool, Unguarded: true},
 	{Key: FlagEVMSSSplit, Path: "EVMSplit", Cast: configtest.CastBool, Unguarded: true},
@@ -253,9 +258,8 @@ func FuzzParseSCConfigs(f *testing.F) {
 }
 
 // FuzzParseSSConfigs drives every [state-store] key through arbitrary raw values.
-// Because the whole section is unguarded, the property being pinned for a nil
-// value is the clobber itself: the resolved field must equal the cast's zero, not
-// the in-code default.
+// For legacy unguarded rows, a nil value must clobber the field to the cast's
+// zero. Guarded rows such as SnapshotEnable must retain their in-code default.
 func FuzzParseSSConfigs(f *testing.F) {
 	seeds := configtest.NewSeeds(f, fuzzing.ConfigValue)
 
@@ -277,7 +281,8 @@ func FuzzParseSSConfigs(f *testing.F) {
 	seeds.AddRow(uint(3), fuzzing.KindInt64, "", int64(200000), false)
 	seeds.AddRow(uint(3), fuzzing.KindNil, "", int64(0), false) // nil clobbers KeepRecent to 0
 	seeds.AddRow(uint(6), fuzzing.KindString, "/var/lib/sei/ss", int64(0), false)
-	seeds.AddRow(uint(10), fuzzing.KindBoolString, "", int64(0), true)
+	seeds.AddRow(uint(8), fuzzing.KindBool, "", int64(0), true) // explicit snapshot opt-in; the default is off
+	seeds.AddRow(uint(11), fuzzing.KindBoolString, "", int64(0), true)
 
 	// The clobber cuts both ways for the four rows below. Because the section is unguarded,
 	// an absent key resolves them to their cast's zero, and so does the malformed seed on an
@@ -287,7 +292,7 @@ func FuzzParseSSConfigs(f *testing.F) {
 	seeds.AddRow(uint(4), fuzzing.KindInt64, "", int64(1800), false) // prune every 30 min rather than the default 600s
 	seeds.AddRow(uint(5), fuzzing.KindInt64, "", int64(4), false)    // four import workers rather than the default 1
 	seeds.AddRow(uint(7), fuzzing.KindBool, "", int64(0), true)      // pebbledb read/write metrics on; the default is off
-	seeds.AddRow(uint(9), fuzzing.KindBool, "", int64(0), true)      // EVM state in its own sub-DBs; the default is shared
+	seeds.AddRow(uint(10), fuzzing.KindBool, "", int64(0), true)     // EVM state in its own sub-DBs; the default is shared
 
 	configtest.CheckEveryRowHasADiscriminatingSeed(f, "state-store", readSS, ssKeys, seeds)
 
@@ -520,8 +525,9 @@ func TestParseSCConfigsAbsentBaseline(t *testing.T) {
 
 // TestParseSSConfigsAbsentBaselineIsZeroClobbered records the clobber in full: an
 // app.toml with no [state-store] section resolves to a config in which every
-// operator-visible knob has been overwritten with a zero value, including the two
-// that change the node's disk behavior without any log line.
+// unguarded operator-visible knob has been overwritten with a zero value, including
+// the two that change the node's disk behavior without any log line. SnapshotEnable
+// is the one field that survives, because its read is guarded.
 func TestParseSSConfigsAbsentBaselineIsZeroClobbered(t *testing.T) {
 	got := parseSSConfigs(configtest.AppOpts{})
 
@@ -673,6 +679,13 @@ func TestManifestNamesEveryField(t *testing.T) {
 			// manager would otherwise try to map a key onto.
 			"KeepLastVersion",
 			"UseDefaultComparer",
+			// The three below are unreachable for a different reason, and the distinction is the
+			// point: they are tagged mapstructure:"-" so no key can bind them even in principle,
+			// and AlignSSSnapshotWithSC derives all three at runtime from the state-commit cadence.
+			// ss-snapshot-enable is the only SS-side knob, and it has a row of its own above.
+			"SnapshotInterval",
+			"SnapshotKeepRecent",
+			"SnapshotMinTimeInterval",
 		)
 	})
 	t.Run("light_invariance", func(t *testing.T) {

app/seidb.go

@@ -48,6 +48,7 @@ const (
 	FlagSSPruneInterval     = "state-store.ss-prune-interval"
 	FlagSSImportNumWorkers  = "state-store.ss-import-num-workers"
 	FlagSSReadWriteMetrics  = "state-store.ss-enable-read-write-metrics"
+	FlagSSSnapshotEnable    = "state-store.ss-snapshot-enable"
 
 	// EVM SS optimization (embedded in SS config, controlled via write/read mode)
 	FlagEVMSSDirectory   = "state-store.evm-ss-db-directory"
@@ -204,6 +205,12 @@ func parseSSConfigs(appOpts servertypes.AppOptions) config.StateStoreConfig {
 	ssConfig.DBDirectory = cast.ToString(appOpts.Get(FlagSSDirectory))
 	ssConfig.EnableReadWriteMetrics = cast.ToBool(appOpts.Get(FlagSSReadWriteMetrics))
 
+	// An absent key is an app.toml rendered before SS snapshots existed. Keep
+	// the in-code default (off) rather than relying on a nil cast.
+	if v := appOpts.Get(FlagSSSnapshotEnable); v != nil {
+		ssConfig.SnapshotEnable = cast.ToBool(v)
+	}
+
 	// EVM optimization fields (embedded in SS config)
 	ssConfig.EVMDBDirectory = cast.ToString(appOpts.Get(FlagEVMSSDirectory))
 	ssConfig.SeparateEVMSubDBs = cast.ToBool(appOpts.Get(FlagEVMSSSeparateDBs))

app/testdata/state-store.golden

@@ -8,6 +8,10 @@ ImportNumWorkers = int(1)
 EnableReadWriteMetrics = bool(false)
 KeepLastVersion = bool(true)
 UseDefaultComparer = bool(false)
+SnapshotEnable = bool(false)
+SnapshotInterval = int64(0)
+SnapshotKeepRecent = int(0)
+SnapshotMinTimeInterval = time.Duration(0s)
 EVMSplit = bool(false)
 EVMDBDirectory = string("")
 SeparateEVMSubDBs = bool(false)

app/testdata/state-store.keys.golden

@@ -6,6 +6,7 @@
 "state-store.ss-import-num-workers"
 "state-store.ss-db-directory"
 "state-store.ss-enable-read-write-metrics"
+"state-store.ss-snapshot-enable"
 "state-store.evm-ss-db-directory"
 "state-store.evm-ss-separate-dbs"
 "state-store.evm-ss-split"

sei-cosmos/server/config/config.go

@@ -508,6 +508,13 @@ func GetConfig(v *viper.Viper) (Config, error) {
 		memIAVLConfig.SnapshotPrefetchThreshold = v.GetFloat64("state-commit.sc-snapshot-prefetch-threshold")
 	}
 
+	// Absent key means an app.toml rendered before SS snapshots existed, which
+	// should keep the in-code default (off) rather than rely on viper's zero.
+	ssSnapshotEnable := config.DefaultStateStoreConfig().SnapshotEnable
+	if v.IsSet("state-store.ss-snapshot-enable") {
+		ssSnapshotEnable = v.GetBool("state-store.ss-snapshot-enable")
+	}
+
 	// Apply the in-code default when the key is absent so that nodes upgrading
 	// with an older app.toml (which lacks this key) are still bounded rather
 	// than running with unlimited connections.
@@ -636,6 +643,7 @@ func GetConfig(v *viper.Viper) (Config, error) {
 			EnableReadWriteMetrics: v.GetBool(
 				"state-store.ss-enable-read-write-metrics",
 			),
+			SnapshotEnable:    ssSnapshotEnable,
 			EVMSplit:          v.GetBool("state-store.evm-ss-split"),
 			EVMDBDirectory:    v.GetString("state-store.evm-ss-db-directory"),
 			SeparateEVMSubDBs: v.GetBool("state-store.evm-ss-separate-dbs"),

sei-cosmos/server/config/testdata/server_config.golden

@@ -140,6 +140,10 @@ StateStore.ImportNumWorkers = int(1)
 StateStore.EnableReadWriteMetrics = bool(false)
 StateStore.KeepLastVersion = bool(true)
 StateStore.UseDefaultComparer = bool(false)
+StateStore.SnapshotEnable = bool(false)
+StateStore.SnapshotInterval = int64(0)
+StateStore.SnapshotKeepRecent = int(0)
+StateStore.SnapshotMinTimeInterval = time.Duration(0s)
 StateStore.EVMSplit = bool(false)
 StateStore.EVMDBDirectory = string("")
 StateStore.SeparateEVMSubDBs = bool(false)

sei-cosmos/storev2/rootmulti/store.go

@@ -37,6 +37,7 @@ import (
 	"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashlog"
 	sctypes "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/types"
 	"github.com/sei-protocol/sei-chain/sei-db/state_db/ss"
+	sscomposite "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/composite"
 	abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types"
 	dbm "github.com/tendermint/tm-db"
 )
@@ -48,10 +49,24 @@ var (
 	_ types.Queryable        = (*Store)(nil)
 )
 
+// stateStoreSnapshotScheduler is the commit path's half of the SS snapshot
+// contract: flush tells the state store which version it has just finished
+// enqueueing, and the store decides whether that version is a boundary.
+type stateStoreSnapshotScheduler interface {
+	ScheduleSnapshot(version int64)
+}
+
+// ss.NewStateStore returns the interface, so the capability is resolved by type
+// assertion at startup. This pins the only implementation, so wrapping the state
+// store without carrying the method through fails the build here rather than
+// silently ending SS snapshots at runtime.
+var _ stateStoreSnapshotScheduler = (*sscomposite.CompositeStateStore)(nil)
+
 type Store struct {
 	mtx            sync.RWMutex
 	scStore        sctypes.Committer
 	ssStore        seidbtypes.StateStore
+	ssSnapshots    stateStoreSnapshotScheduler
 	lastCommitInfo *types.CommitInfo
 	storesParams   map[types.StoreKey]storeParams
 	storeKeys      map[string]types.StoreKey
@@ -136,6 +151,7 @@ func NewStore(
 		scDir:              scDir,
 	}
 	if ssConfig.Enable {
+		config.AlignSSSnapshotWithSC(scConfig, &ssConfig)
 		ssStore, err := ss.NewStateStore(homeDir, ssConfig)
 		if err != nil {
 			panic(err)
@@ -150,6 +166,16 @@ func NewStore(
 			panic("Enabling SS store without state sync could cause data corruption")
 		}
 		store.ssStore = ssStore
+		scheduler, ok := ssStore.(stateStoreSnapshotScheduler)
+		if !ok {
+			// Unreachable while CompositeStateStore is the only implementation,
+			// which the assertion above pins. Log rather than drop silently, so
+			// a wrapper that loses the method is visible as a boot line instead
+			// of as snapshots that never appear.
+			logger.Error("state store does not schedule snapshots; SS snapshots are disabled",
+				"type", fmt.Sprintf("%T", ssStore))
+		}
+		store.ssSnapshots = scheduler
 	}
 	return store
 
@@ -255,6 +281,14 @@ func (rs *Store) flush() error {
 			telemetry.SetGauge(float32(currentVersion), "storeV2", "ss", "version")
 		}
 	}
+	// Both branches above have finished handing currentVersion to SS and have
+	// enqueued nothing above it, which is what makes an SS snapshot label exact.
+	// Triggering here rather than inside either branch keeps populated and empty
+	// blocks on one path. A repeat within the same block (flush runs twice, and
+	// the second pass sees an empty changeset) is ignored by the state store.
+	if rs.ssSnapshots != nil {
+		rs.ssSnapshots.ScheduleSnapshot(currentVersion)
+	}
 	return rs.scStore.ApplyChangeSets(changeSets)
 }
 

sei-cosmos/storev2/rootmulti/store_test.go

@@ -3,6 +3,7 @@ package rootmulti
 import (
 	"context"
 	"fmt"
+	"path/filepath"
 	"sync"
 	"testing"
 	"time"
@@ -11,6 +12,7 @@ import (
 	"github.com/sei-protocol/sei-chain/sei-cosmos/store/types"
 	"github.com/sei-protocol/sei-chain/sei-cosmos/storev2/state"
 	"github.com/sei-protocol/sei-chain/sei-db/config"
+	sscomposite "github.com/sei-protocol/sei-chain/sei-db/state_db/ss/composite"
 	abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/time/rate"
@@ -135,6 +137,55 @@ func TestSCSS_WriteAndHistoricalRead(t *testing.T) {
 	require.Equal(t, valV1, resp.Value)
 }
 
+// flush owns the SS snapshot trigger for every block, so a boundary must be
+// scheduled whether or not the block carried changesets. The composite package
+// cannot pin this: its tests call ScheduleSnapshot themselves, so a regression
+// in either branch of flush is invisible there.
+func TestFlushSchedulesSSSnapshotAtABoundary(t *testing.T) {
+	for _, tc := range []struct {
+		name         string
+		writeAtBlock int64
+	}{
+		{name: "boundary block is populated", writeAtBlock: 2},
+		{name: "boundary block is empty", writeAtBlock: 1},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			home := t.TempDir()
+			scCfg := config.DefaultStateCommitConfig()
+			scCfg.Enable = true
+			scCfg.MemIAVLConfig.AsyncCommitBuffer = 0
+			// SS mirrors the SC cadence, so this is what puts the SS boundary at 2.
+			scCfg.MemIAVLConfig.SnapshotInterval = 2
+			scCfg.MemIAVLConfig.SnapshotKeepRecent = 1
+
+			ssCfg := config.DefaultStateStoreConfig()
+			ssCfg.Enable = true
+			ssCfg.SnapshotEnable = true
+
+			store := NewStore(home, scCfg, ssCfg, []string{})
+			defer func() { _ = store.Close() }()
+			require.NotNil(t, store.ssSnapshots, "SS snapshot capability was not resolved")
+
+			key := types.NewKVStoreKey("bank")
+			store.MountStoreWithDB(key, types.StoreTypeIAVL, nil)
+			require.NoError(t, store.LoadLatestVersion())
+
+			for block := int64(1); block <= 2; block++ {
+				if block == tc.writeAtBlock {
+					store.GetStoreByName("bank").(types.KVStore).Set([]byte("k"), []byte("v"))
+				}
+				require.Equal(t, block, store.Commit(true).Version)
+			}
+
+			root := filepath.Join(home, "data", "state_store", sscomposite.SnapshotsDirName)
+			require.Eventually(t, func() bool {
+				versions, err := sscomposite.ListSnapshotVersions(root)
+				return err == nil && len(versions) == 1 && versions[0] == 2
+			}, 10*time.Second, 20*time.Millisecond, "boundary did not produce an SS snapshot")
+		})
+	}
+}
+
 // TestCacheMultiStoreWithVersion_OnlyUsesSSStores verifies that CacheMultiStoreWithVersion
 // serves SS stores when enabled, and falls back to SC when SS is disabled, for
 // height=0 (latest) and explicit latest height.

sei-db/common/utils/path.go

@@ -7,6 +7,8 @@ import (
 	"strings"
 )
 
+const StateStoreSnapshotsDirName = "snapshots"
+
 // DirExists returns true if path exists and is a directory.
 func DirExists(path string) bool {
 	info, err := os.Stat(path)
@@ -63,6 +65,11 @@ func GetEVMStateStorePath(homePath string, backend string) string {
 	return filepath.Join(homePath, "data", "state_store", "evm", backend)
 }
 
+// GetStateStoreSnapshotsPath returns the path for online state-store snapshots.
+func GetStateStoreSnapshotsPath(homePath string) string {
+	return filepath.Join(homePath, "data", "state_store", StateStoreSnapshotsDirName)
+}
+
 // GetReceiptStorePath returns the path for the receipt store.
 // New nodes use data/ledger/receipt/{backend}; existing nodes with
 // data/receipt.db continue using the legacy path for backward compatibility.

sei-db/config/sc_config.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config"
 	"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl"
@@ -16,6 +17,34 @@ const (
 	legacySCWriteModeCosmosOnly = "cosmos_only"
 )
 
+// EffectiveMemIAVLSnapshotCadence resolves memIAVL's snapshot cadence the way
+// Options.FillDefaults resolves it at OpenDB, so a caller mirroring the cadence
+// onto another backend sees the values memIAVL will actually run with rather
+// than the raw config. A zero means "unset" here, not "disabled": memIAVL heals
+// it to the default, so mirroring the raw zero would silently disable snapshots
+// on the mirroring backend.
+func EffectiveMemIAVLSnapshotCadence(cfg memiavl.Config) (interval, keepRecent uint32) {
+	interval = cfg.SnapshotInterval
+	if interval == 0 {
+		interval = memiavl.DefaultSnapshotInterval
+	}
+	keepRecent = cfg.SnapshotKeepRecent
+	if keepRecent == 0 {
+		keepRecent = memiavl.DefaultSnapshotKeepRecent
+	}
+	return interval, keepRecent
+}
+
+// EffectiveMemIAVLSnapshotMinTimeInterval resolves the minimum wall-clock
+// interval the same way memIAVL Options.FillDefaults does.
+func EffectiveMemIAVLSnapshotMinTimeInterval(cfg memiavl.Config) time.Duration {
+	seconds := cfg.SnapshotMinTimeInterval
+	if seconds == 0 {
+		seconds = memiavl.DefaultSnapshotMinTimeInterval
+	}
+	return time.Duration(seconds) * time.Second
+}
+
 // StateCommitConfig defines configuration for the state commit (SC) layer.
 type StateCommitConfig struct {
 	// Enable defines if the state-commit (SeiDB) should be enabled.