State sync support

baseapp/abci.go

@@ -262,6 +262,27 @@ func (app *BaseApp) Commit() (res abci.ResponseCommit) {
 		app.halt()
 	}
 
+	if app.snapshotInterval > 0 && uint64(header.Height)%app.snapshotInterval == 0 {
+		go func() {
+			app.logger.Info("Taking state snapshot", "height", header.Height)
+			err := app.snapshot(uint64(header.Height))
+			if err != nil {
+				app.logger.Error("Failed to take snapshot", "height", header.Height, "err", err.Error())
+				return
+			}
+			app.logger.Info("Finished taking state snapshot", "height", header.Height)
+			if app.snapshotRetention > 0 {
+				app.logger.Info("Pruning state snapshots")
+				pruned, err := app.snapshotStore.Prune(app.snapshotRetention)
+				if err != nil {
+					app.logger.Error("Failed to prune snapshots", "err", err.Error())
+					return
+				}
+				app.logger.Info(fmt.Sprintf("Pruned %v state snapshots", pruned))
+			}
+		}()
+	}
+
 	return abci.ResponseCommit{
 		Data: commitID.Hash,
 	}

baseapp/baseapp.go

@@ -14,6 +14,7 @@ import (
 	dbm "github.com/tendermint/tm-db"
 
 	"github.com/cosmos/cosmos-sdk/store"
+	"github.com/cosmos/cosmos-sdk/store/snapshot"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
 )
@@ -26,6 +27,9 @@ const (
 
 	// MainStoreKey is the string representation of the main store
 	MainStoreKey = "main"
+
+	// snapshotFormat is the currently used snapshot format
+	snapshotFormat = 1
 )
 
 var (
@@ -70,6 +74,11 @@ type BaseApp struct { // nolint: maligned
 	idPeerFilter   sdk.PeerFilter   // filter peers by node ID
 	fauxMerkleMode bool             // if true, IAVL MountStores uses MountStoresDB for simulation speed.
 
+	// snapshot storage, i.e. dumps of app state at certain intervals
+	snapshotStore     *snapshot.Store
+	snapshotInterval  uint64 // interval (in blocks) between snapshots (0 to disable)
+	snapshotRetention uint32 // number of snapshots to keep (0 for all)
+
 	// volatile states:
 	//
 	// checkState is set on InitChain and reset on Commit
@@ -623,3 +632,18 @@ func (app *BaseApp) runMsgs(ctx sdk.Context, msgs []sdk.Msg, mode runTxMode) (*s
 		Events: events,
 	}, nil
 }
+
+// snapshot takes a snapshot
+func (app *BaseApp) snapshot(height uint64) error {
+	if app.snapshotStore == nil {
+		return errors.New("no snapshot store configured")
+	}
+	if app.snapshotStore.Active() {
+		return errors.New("a snapshot is already in progress")
+	}
+	chunks, err := app.cms.Snapshot(height)
+	if err != nil {
+		return err
+	}
+	return app.snapshotStore.Save(height, snapshotFormat, chunks)
+}

baseapp/options.go

@@ -7,6 +7,7 @@ import (
 	dbm "github.com/tendermint/tm-db"
 
 	"github.com/cosmos/cosmos-sdk/store"
+	"github.com/cosmos/cosmos-sdk/store/snapshot"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 )
 
@@ -44,6 +45,16 @@ func SetInterBlockCache(cache sdk.MultiStorePersistentCache) func(*BaseApp) {
 	return func(app *BaseApp) { app.setInterBlockCache(cache) }
 }
 
+// SetSnapshotDB sets the snapshot store.
+func SetSnapshotStore(s *snapshot.Store) func(*BaseApp) {
+	return func(app *BaseApp) { app.SetSnapshotStore(s) }
+}
+
+// SetSnapshotPolicy sets the snapshot policy.
+func SetSnapshotPolicy(interval uint64, retention uint32) func(*BaseApp) {
+	return func(app *BaseApp) { app.SetSnapshotPolicy(interval, retention) }
+}
+
 func (app *BaseApp) SetName(name string) {
 	if app.sealed {
 		panic("SetName() on sealed BaseApp")
@@ -143,3 +154,21 @@ func (app *BaseApp) SetRouter(router sdk.Router) {
 	}
 	app.router = router
 }
+
+// SetSnapshotDB sets the snapshot database and data directory. This database must
+// be independent of the app state.
+func (app *BaseApp) SetSnapshotStore(s *snapshot.Store) {
+	if app.sealed {
+		panic("SetSnapshotDB() on sealed BaseApp")
+	}
+	app.snapshotStore = s
+}
+
+// SetSnapshotPolicy sets the snapshotting policy. 0 disables snapshotting.
+func (app *BaseApp) SetSnapshotPolicy(interval uint64, retention uint32) {
+	if app.sealed {
+		panic("SetSnapshotPolicy on sealed BaseApp")
+	}
+	app.snapshotInterval = interval
+	app.snapshotRetention = retention
+}

go.mod

@@ -35,4 +35,9 @@ require (
 
 replace github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.2-alpha.regen.1
 
+// FIXME Use local iavl with export/import API
+replace github.com/tendermint/iavl => ../iavl
+
+replace github.com/tendermint/tm-db => ../tm-db
+
 go 1.14

go.sum

@@ -137,12 +137,14 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
 github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk=
 github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -388,7 +390,6 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
-github.com/spf13/viper v1.6.1/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k=
 github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E=
 github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k=
 github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
@@ -408,8 +409,6 @@ github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d h1:gZZadD8H+fF+n9CmNhYL1Y0dJB+kLOmKd7FbPJLeGHs=
 github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
-github.com/tecbot/gorocksdb v0.0.0-20191017175515-d217d93fd4c5 h1:gVwAW5OwaZlDB5/CfqcGFM9p9C+KxvQKyNOltQ8orj0=
-github.com/tecbot/gorocksdb v0.0.0-20191017175515-d217d93fd4c5/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8=
 github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok=
 github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8=
 github.com/tendermint/btcd v0.1.1 h1:0VcxPfflS2zZ3RiOAHkBiFUcPvbtRj5O7zHmcJWHV7s=
@@ -419,9 +418,9 @@ github.com/tendermint/crypto v0.0.0-20191022145703-50d29ede1e15/go.mod h1:z4YtwM
 github.com/tendermint/go-amino v0.14.1/go.mod h1:i/UKE5Uocn+argJJBb12qTZsCDBcAYMbR92AaJVmKso=
 github.com/tendermint/go-amino v0.15.1 h1:D2uk35eT4iTsvJd9jWIetzthE5C0/k2QmMFkCN+4JgQ=
 github.com/tendermint/go-amino v0.15.1/go.mod h1:TQU0M1i/ImAo+tYpZi73AU3V/dKeCoMC9Sphe2ZwGME=
-github.com/tendermint/iavl v0.13.0 h1:r2sINvNFlJsLlLhGoqlqPlREWYkuK26BvMfkBt+XQnA=
-github.com/tendermint/iavl v0.13.0/go.mod h1:7nSUPdrsHEZ2nNZa+9gaIrcJciWd1jCQZXtcyARU82k=
 github.com/tendermint/tendermint v0.33.0/go.mod h1:s5UoymnPIY+GcA3mMte4P9gpMP8vS7UH7HBXikT1pHI=
+github.com/tendermint/tendermint v0.33.1 h1:8f68LUBz8yhISZvaLFP4siXXrLWsWeoYfelbdNtmvm4=
+github.com/tendermint/tendermint v0.33.1/go.mod h1:fBOKyrlXOETqQ+heL8x/TZgSdmItON54csyabvktBp0=
 github.com/tendermint/tendermint v0.33.2 h1:NzvRMTuXJxqSsFed2J7uHmMU5N1CVzSpfi3nCc882KY=
 github.com/tendermint/tendermint v0.33.2/go.mod h1:25DqB7YvV1tN3tHsjWoc2vFtlwICfrub9XO6UBO+4xk=
 github.com/tendermint/tm-db v0.4.0 h1:iPbCcLbf4nwDFhS39Zo1lpdS1X/cT9CkTlUx17FHQgA=

server/constructors.go

@@ -11,13 +11,14 @@ import (
 	tmtypes "github.com/tendermint/tendermint/types"
 	dbm "github.com/tendermint/tm-db"
 
+	"github.com/cosmos/cosmos-sdk/store/snapshot"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 )
 
 type (
 	// AppCreator is a function that allows us to lazily initialize an
 	// application using various configurations.
-	AppCreator func(log.Logger, dbm.DB, io.Writer) abci.Application
+	AppCreator func(log.Logger, dbm.DB, *snapshot.Store, io.Writer) abci.Application
 
 	// AppExporter is a function that dumps all app state to
 	// JSON-serializable structure and returns the current validator set.
@@ -30,6 +31,12 @@ func openDB(rootDir string) (dbm.DB, error) {
 	return db, err
 }
 
+func openSnapshotStore(db dbm.DB, rootDir string) (*snapshot.Store, error) {
+	db = dbm.NewPrefixDB(db, []byte("s/_snapshots/"))
+	dir := filepath.Join(rootDir, "data", "snapshots")
+	return snapshot.New(db, dir)
+}
+
 func openTraceWriter(traceWriterFile string) (w io.Writer, err error) {
 	if traceWriterFile != "" {
 		w, err = os.OpenFile(

server/mock/store.go

@@ -99,6 +99,14 @@ func (ms multiStore) SetInterBlockCache(_ sdk.MultiStorePersistentCache) {
 	panic("not implemented")
 }
 
+func (kv multiStore) Snapshot(height uint64) (<-chan io.ReadCloser, error) {
+	panic("not implemented")
+}
+
+func (kv multiStore) Restore(height uint64, chunks <-chan io.ReadCloser) error {
+	panic("not implemented")
+}
+
 var _ sdk.KVStore = kvStore{}
 
 type kvStore struct {

server/start.go

@@ -139,12 +139,17 @@ func startStandAlone(ctx *Context, appCreator AppCreator) error {
 	if err != nil {
 		return err
 	}
+	ss, err := openSnapshotStore(db, home)
+	if err != nil {
+		return err
+	}
+
 	traceWriter, err := openTraceWriter(traceWriterFile)
 	if err != nil {
 		return err
 	}
 
-	app := appCreator(ctx.Logger, db, traceWriter)
+	app := appCreator(ctx.Logger, db, ss, traceWriter)
 
 	svr, err := server.NewServer(addr, "socket", app)
 	if err != nil {
@@ -180,12 +185,17 @@ func startInProcess(ctx *Context, appCreator AppCreator) (*node.Node, error) {
 		return nil, err
 	}
 
+	ss, err := openSnapshotStore(db, home)
+	if err != nil {
+		return nil, err
+	}
+
 	traceWriter, err := openTraceWriter(traceWriterFile)
 	if err != nil {
 		return nil, err
 	}
 
-	app := appCreator(ctx.Logger, db, traceWriter)
+	app := appCreator(ctx.Logger, db, ss, traceWriter)
 
 	nodeKey, err := p2p.LoadOrGenNodeKey(cfg.NodeKeyFile())
 	if err != nil {

store/iavl/store.go

@@ -233,6 +233,28 @@ func (st *Store) ReverseIterator(start, end []byte) types.Iterator {
 	return newIAVLIterator(iTree, start, end, false)
 }
 
+// Exports the IAVL store at the given version, returning an iavl.Exporter for the tree.
+func (st *Store) Export(version int64) (*iavl.Exporter, error) {
+	istore, err := st.GetImmutable(version)
+	if err != nil {
+		return nil, fmt.Errorf("iavl export failed for version %v: %w", version, err)
+	}
+	tree, ok := istore.tree.(*immutableTree)
+	if !ok || tree == nil {
+		return nil, fmt.Errorf("iavl export failed: unable to fetch tree for version %v", version)
+	}
+	return tree.Export(), nil
+}
+
+// Import imports an IAVL tree at the given version, returning an iavl.Importer for importing.
+func (st *Store) Import(version int64) (*iavl.Importer, error) {
+	tree, ok := st.tree.(*iavl.MutableTree)
+	if !ok {
+		return nil, errors.New("iavl import failed: unable to find mutable tree")
+	}
+	return tree.Import(version)
+}
+
 // Handle gatest the latest height, if height is 0
 func getHeight(tree Tree, req abci.RequestQuery) int64 {
 	height := req.Height