From 3f5b66e719ea4ef0c66755982faa1898bdad6e76 Mon Sep 17 00:00:00 2001
From: Yawning Angel <yawning@schwanenlied.me>
Date: Tue, 19 May 2020 12:09:11 +0000
Subject: [PATCH 1/4] go/storage: Add support for read-only mode

It is safer if tooling can access on-disk state of an offline node in a
way that ensures that writes will be rejected.  This adds support for
doing so by rejecting writes at the API level and by configuring the
backing database to open the DB for read-only access.
---
 go/storage/api/api.go               |  6 ++++++
 go/storage/database/database.go     | 19 +++++++++++++++++++
 go/storage/mkvs/db/api/api.go       |  5 +++++
 go/storage/mkvs/db/badger/badger.go | 16 ++++++++++++++++
 4 files changed, 46 insertions(+)

diff --git a/go/storage/api/api.go b/go/storage/api/api.go
index 7f3c5f53e44..56860c48afe 100644
--- a/go/storage/api/api.go
+++ b/go/storage/api/api.go
@@ -69,6 +69,8 @@ var (
 	ErrRootNotFound = nodedb.ErrRootNotFound
 	// ErrRootMustFollowOld indicates that the passed new root does not follow old root.
 	ErrRootMustFollowOld = nodedb.ErrRootMustFollowOld
+	// ErrReadOnly indicates that the storage backend is read-only.
+	ErrReadOnly = nodedb.ErrReadOnly
 
 	// ReceiptSignatureContext is the signature context used for verifying MKVS receipts.
 	ReceiptSignatureContext = signature.NewContext("oasis-core/storage: receipt", signature.WithChainSeparation())
@@ -105,6 +107,9 @@ type Config struct { // nolint: maligned
 
 	// MemoryOnly will make the storage memory-only (if the backend supports it).
 	MemoryOnly bool
+
+	// ReadOnly will make the storage read-only.
+	ReadOnly bool
 }
 
 // ToNodeDB converts from a Config to a node DB Config.
@@ -115,6 +120,7 @@ func (cfg *Config) ToNodeDB() *nodedb.Config {
 		MaxCacheSize:     cfg.MaxCacheSize,
 		NoFsync:          cfg.NoFsync,
 		MemoryOnly:       cfg.MemoryOnly,
+		ReadOnly:         cfg.ReadOnly,
 		DiscardWriteLogs: cfg.DiscardWriteLogs,
 	}
 }
diff --git a/go/storage/database/database.go b/go/storage/database/database.go
index 65176c9b046..4d607753b8c 100644
--- a/go/storage/database/database.go
+++ b/go/storage/database/database.go
@@ -44,6 +44,8 @@ type databaseBackend struct {
 
 	signer signature.Signer
 	initCh chan struct{}
+
+	readOnly bool
 }
 
 // New constructs a new database backed storage Backend instance.
@@ -92,10 +94,15 @@ func New(cfg *api.Config) (api.Backend, error) {
 		rootCache:    rootCache,
 		signer:       cfg.Signer,
 		initCh:       initCh,
+		readOnly:     cfg.ReadOnly,
 	}, nil
 }
 
 func (ba *databaseBackend) Apply(ctx context.Context, request *api.ApplyRequest) ([]*api.Receipt, error) {
+	if ba.readOnly {
+		return nil, fmt.Errorf("storage/database: failed to Apply: %w", api.ErrReadOnly)
+	}
+
 	newRoot, err := ba.rootCache.Apply(
 		ctx,
 		request.Namespace,
@@ -114,6 +121,10 @@ func (ba *databaseBackend) Apply(ctx context.Context, request *api.ApplyRequest)
 }
 
 func (ba *databaseBackend) ApplyBatch(ctx context.Context, request *api.ApplyBatchRequest) ([]*api.Receipt, error) {
+	if ba.readOnly {
+		return nil, fmt.Errorf("storage/database: failed to ApplyBatch: %w", api.ErrReadOnly)
+	}
+
 	newRoots := make([]hash.Hash, 0, len(request.Ops))
 	for _, op := range request.Ops {
 		newRoot, err := ba.rootCache.Apply(ctx, request.Namespace, op.SrcRound, op.SrcRoot, request.DstRound, op.DstRoot, op.WriteLog)
@@ -128,6 +139,10 @@ func (ba *databaseBackend) ApplyBatch(ctx context.Context, request *api.ApplyBat
 }
 
 func (ba *databaseBackend) Merge(ctx context.Context, request *api.MergeRequest) ([]*api.Receipt, error) {
+	if ba.readOnly {
+		return nil, fmt.Errorf("storage/database: failed to Merge: %w", api.ErrReadOnly)
+	}
+
 	newRoot, err := ba.rootCache.Merge(ctx, request.Namespace, request.Round, request.Base, request.Others)
 	if err != nil {
 		return nil, fmt.Errorf("storage/database: failed to Merge: %w", err)
@@ -138,6 +153,10 @@ func (ba *databaseBackend) Merge(ctx context.Context, request *api.MergeRequest)
 }
 
 func (ba *databaseBackend) MergeBatch(ctx context.Context, request *api.MergeBatchRequest) ([]*api.Receipt, error) {
+	if ba.readOnly {
+		return nil, fmt.Errorf("storage/database: failed to MergeBatch: %w", api.ErrReadOnly)
+	}
+
 	newRoots := make([]hash.Hash, 0, len(request.Ops))
 	for _, op := range request.Ops {
 		newRoot, err := ba.rootCache.Merge(ctx, request.Namespace, request.Round, op.Base, op.Others)
diff --git a/go/storage/mkvs/db/api/api.go b/go/storage/mkvs/db/api/api.go
index c5bdf532b6a..6819d491746 100644
--- a/go/storage/mkvs/db/api/api.go
+++ b/go/storage/mkvs/db/api/api.go
@@ -43,6 +43,8 @@ var (
 	ErrBadNamespace = errors.New(ModuleName, 10, "mkvs: bad namespace")
 	// ErrNotEarliest indicates that the given version is not the earliest version.
 	ErrNotEarliest = errors.New(ModuleName, 11, "mkvs: version is not the earliest version")
+	// ErrReadOnly indicates that a write operation failed due to a read-only database.
+	ErrReadOnly = errors.New(ModuleName, 12, "mkvs: read-only database")
 )
 
 // Config is the node database backend configuration.
@@ -56,6 +58,9 @@ type Config struct { // nolint: maligned
 	// MemoryOnly will make the storage memory-only (if the backend supports it).
 	MemoryOnly bool
 
+	// ReadOnly will make the storage read-only.
+	ReadOnly bool
+
 	// Namespace is the namespace contained within the database.
 	Namespace common.Namespace
 
diff --git a/go/storage/mkvs/db/badger/badger.go b/go/storage/mkvs/db/badger/badger.go
index adc12f8ee80..52928dc2bee 100644
--- a/go/storage/mkvs/db/badger/badger.go
+++ b/go/storage/mkvs/db/badger/badger.go
@@ -53,6 +53,7 @@ func New(cfg *api.Config) (api.NodeDB, error) {
 	db := &badgerNodeDB{
 		logger:           logging.GetLogger("mkvs/db/badger"),
 		namespace:        cfg.Namespace,
+		readOnly:         cfg.ReadOnly,
 		discardWriteLogs: cfg.DiscardWriteLogs,
 	}
 
@@ -64,6 +65,7 @@ func New(cfg *api.Config) (api.NodeDB, error) {
 	opts = opts.WithTruncate(true)
 	opts = opts.WithCompression(options.Snappy)
 	opts = opts.WithMaxCacheSize(cfg.MaxCacheSize)
+	opts = opts.WithReadOnly(cfg.ReadOnly)
 
 	if cfg.MemoryOnly {
 		db.logger.Warn("using memory-only mode, data will not be persisted")
@@ -94,6 +96,7 @@ type badgerNodeDB struct { // nolint: maligned
 
 	namespace common.Namespace
 
+	readOnly         bool
 	discardWriteLogs bool
 
 	db *badger.DB
@@ -403,6 +406,10 @@ func (d *badgerNodeDB) HasRoot(root node.Root) bool {
 }
 
 func (d *badgerNodeDB) Finalize(ctx context.Context, version uint64, roots []hash.Hash) error { // nolint: gocyclo
+	if d.readOnly {
+		return api.ErrReadOnly
+	}
+
 	d.metaUpdateLock.Lock()
 	defer d.metaUpdateLock.Unlock()
 
@@ -559,6 +566,10 @@ func (d *badgerNodeDB) Finalize(ctx context.Context, version uint64, roots []has
 }
 
 func (d *badgerNodeDB) Prune(ctx context.Context, version uint64) error {
+	if d.readOnly {
+		return api.ErrReadOnly
+	}
+
 	d.metaUpdateLock.Lock()
 	defer d.metaUpdateLock.Unlock()
 
@@ -742,6 +753,11 @@ func (ba *badgerBatch) RemoveNodes(nodes []node.Node) error {
 }
 
 func (ba *badgerBatch) Commit(root node.Root) error {
+	// XXX: Ideally this would fail at batch creation.
+	if ba.db.readOnly {
+		return api.ErrReadOnly
+	}
+
 	ba.db.metaUpdateLock.Lock()
 	defer ba.db.metaUpdateLock.Unlock()
 

From e5d5402474473c6b95c553b67e94f6c41de648f9 Mon Sep 17 00:00:00 2001
From: Yawning Angel <yawning@schwanenlied.me>
Date: Tue, 19 May 2020 12:10:44 +0000
Subject: [PATCH 2/4] go/consensus/tendermint/abci: Move storage init into a
 helper

The dump tool was duplicating a considerable amount of this, so refactor
it out into a helper to minimize code duplication.
---
 go/consensus/tendermint/abci/mux.go   |  3 ++
 go/consensus/tendermint/abci/state.go | 58 ++++++++++++++++++++-------
 2 files changed, 46 insertions(+), 15 deletions(-)

diff --git a/go/consensus/tendermint/abci/mux.go b/go/consensus/tendermint/abci/mux.go
index 93bf2673927..2c74e61fd49 100644
--- a/go/consensus/tendermint/abci/mux.go
+++ b/go/consensus/tendermint/abci/mux.go
@@ -72,6 +72,9 @@ type ApplicationConfig struct {
 
 	// MemoryOnlyStorage forces in-memory storage to be used for the state storage.
 	MemoryOnlyStorage bool
+
+	// ReadOnlyStorage forces read-only access for the state storage.
+	ReadOnlyStorage bool
 }
 
 // TransactionAuthHandler is the interface for ABCI applications that handle
diff --git a/go/consensus/tendermint/abci/state.go b/go/consensus/tendermint/abci/state.go
index ff3f5e62395..07ed38730ef 100644
--- a/go/consensus/tendermint/abci/state.go
+++ b/go/consensus/tendermint/abci/state.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"os"
 	"path/filepath"
 	"sync"
 	"time"
@@ -359,16 +360,31 @@ func (s *applicationState) pruneWorker() {
 	}
 }
 
-func newApplicationState(ctx context.Context, cfg *ApplicationConfig) (*applicationState, error) {
+// InitStateStorage initializes the internal ABCI state storage.
+func InitStateStorage(ctx context.Context, cfg *ApplicationConfig) (storage.LocalBackend, storage.NodeDB, *storage.Root, error) {
 	baseDir := filepath.Join(cfg.DataDir, appStateDir)
-	if err := common.Mkdir(baseDir); err != nil {
-		return nil, fmt.Errorf("failed to create application state directory: %w", err)
+	switch cfg.ReadOnlyStorage {
+	case true:
+		// Note: I'm not sure what badger does when given a path that
+		// doesn't actually contain a database, when it's set to
+		// read-only.  Hopefully it's something sensible.
+		fi, err := os.Lstat(baseDir)
+		if err != nil {
+			return nil, nil, nil, fmt.Errorf("failed to stat application state directory: %w", err)
+		}
+		if !fi.Mode().IsDir() {
+			return nil, nil, nil, fmt.Errorf("application state path is not a directory: %v", fi.Mode())
+		}
+	default:
+		if err := common.Mkdir(baseDir); err != nil {
+			return nil, nil, nil, fmt.Errorf("failed to create application state directory: %w", err)
+		}
 	}
 
 	switch cfg.StorageBackend {
 	case storageDB.BackendNameBadgerDB:
 	default:
-		return nil, fmt.Errorf("unsupported storage backend: %s", cfg.StorageBackend)
+		return nil, nil, nil, fmt.Errorf("unsupported storage backend: %s", cfg.StorageBackend)
 	}
 
 	db, err := storageDB.New(&storage.Config{
@@ -378,9 +394,10 @@ func newApplicationState(ctx context.Context, cfg *ApplicationConfig) (*applicat
 		DiscardWriteLogs: true,
 		NoFsync:          true, // This is safe as Tendermint will replay on crash.
 		MemoryOnly:       cfg.MemoryOnlyStorage,
+		ReadOnly:         cfg.ReadOnlyStorage,
 	})
 	if err != nil {
-		return nil, err
+		return nil, nil, nil, err
 	}
 	ldb := db.(storage.LocalBackend)
 	ndb := ldb.NodeDB()
@@ -396,20 +413,20 @@ func newApplicationState(ctx context.Context, cfg *ApplicationConfig) (*applicat
 	// Figure out the latest version/hash if any, and use that as the block height/hash.
 	latestVersion, err := ndb.GetLatestVersion(ctx)
 	if err != nil {
-		return nil, err
+		return nil, nil, nil, err
 	}
 	roots, err := ndb.GetRootsForVersion(ctx, latestVersion)
 	if err != nil {
-		return nil, err
+		return nil, nil, nil, err
 	}
-	stateRoot := storage.Root{
+	stateRoot := &storage.Root{
 		Version: latestVersion,
 	}
 	switch len(roots) {
 	case 0:
 		// No roots -- empty database.
 		if latestVersion != 0 {
-			return nil, fmt.Errorf("state: no roots at non-zero height, corrupted database?")
+			return nil, nil, nil, fmt.Errorf("state: no roots at non-zero height, corrupted database?")
 		}
 		stateRoot.Hash.Empty()
 	case 1:
@@ -417,12 +434,25 @@ func newApplicationState(ctx context.Context, cfg *ApplicationConfig) (*applicat
 		stateRoot.Hash = roots[0]
 	default:
 		// More roots -- should not happen for our use case.
-		return nil, fmt.Errorf("state: more than one root, corrupted database?")
+		return nil, nil, nil, fmt.Errorf("state: more than one root, corrupted database?")
 	}
 
+	ok = true
+
+	return ldb, ndb, stateRoot, nil
+}
+
+func newApplicationState(ctx context.Context, cfg *ApplicationConfig) (*applicationState, error) {
+	// Initialize the state storage.
+	ldb, ndb, stateRoot, err := InitStateStorage(ctx, cfg)
+	if err != nil {
+		return nil, err
+	}
+	latestVersion := stateRoot.Version
+
 	// Use the node database directly to avoid going through the syncer interface.
-	deliverTxTree := mkvs.NewWithRoot(nil, ndb, stateRoot, mkvs.WithoutWriteLog())
-	checkTxTree := mkvs.NewWithRoot(nil, ndb, stateRoot, mkvs.WithoutWriteLog())
+	deliverTxTree := mkvs.NewWithRoot(nil, ndb, *stateRoot, mkvs.WithoutWriteLog())
+	checkTxTree := mkvs.NewWithRoot(nil, ndb, *stateRoot, mkvs.WithoutWriteLog())
 
 	// Initialize the state pruner.
 	statePruner, err := newStatePruner(&cfg.Pruning, ndb, latestVersion)
@@ -443,7 +473,7 @@ func newApplicationState(ctx context.Context, cfg *ApplicationConfig) (*applicat
 		cancelCtx:       cancelCtx,
 		deliverTxTree:   deliverTxTree,
 		checkTxTree:     checkTxTree,
-		stateRoot:       stateRoot,
+		stateRoot:       *stateRoot,
 		storage:         ldb,
 		statePruner:     statePruner,
 		prunerClosedCh:  make(chan struct{}),
@@ -462,8 +492,6 @@ func newApplicationState(ctx context.Context, cfg *ApplicationConfig) (*applicat
 		}
 	}
 
-	ok = true
-
 	go s.metricsWorker()
 	go s.pruneWorker()
 

From bb06f5335d0806b4e0eb4c0953269a2b01eb0229 Mon Sep 17 00:00:00 2001
From: Yawning Angel <yawning@schwanenlied.me>
Date: Tue, 19 May 2020 13:31:59 +0000
Subject: [PATCH 3/4] go/consensus/tendermint: Allow queries without a full
 ApplicaitonState

The timekeeping part of this (needed for the registry) will be somewhat
annoying to implement, and it's unfortunate that the ApplicationState is
now split into two interfaces, but I can't think of a better way to do
it, and this shouldn't be that bad even if we add more things.
---
 go/consensus/tendermint/abci/state/state.go   |  2 +-
 go/consensus/tendermint/api/state.go          | 24 +++++++++++-------
 go/consensus/tendermint/apps/beacon/query.go  | 13 +++++++---
 .../tendermint/apps/beacon/state/state.go     |  2 +-
 .../tendermint/apps/epochtime_mock/query.go   | 13 +++++++---
 .../tendermint/apps/epochtime_mock/state.go   |  2 +-
 .../tendermint/apps/keymanager/query.go       | 13 +++++++---
 .../tendermint/apps/keymanager/state/state.go |  2 +-
 .../tendermint/apps/registry/query.go         | 25 ++++++++++++-------
 .../tendermint/apps/registry/state/state.go   |  2 +-
 .../tendermint/apps/roothash/query.go         | 13 +++++++---
 .../tendermint/apps/roothash/state/state.go   |  2 +-
 .../tendermint/apps/scheduler/query.go        | 15 ++++++++---
 .../tendermint/apps/scheduler/state/state.go  |  2 +-
 go/consensus/tendermint/apps/staking/query.go | 13 +++++++---
 .../tendermint/apps/staking/state/state.go    |  2 +-
 16 files changed, 100 insertions(+), 45 deletions(-)

diff --git a/go/consensus/tendermint/abci/state/state.go b/go/consensus/tendermint/abci/state/state.go
index a1441ec4f3c..75480bbbb66 100644
--- a/go/consensus/tendermint/abci/state/state.go
+++ b/go/consensus/tendermint/abci/state/state.go
@@ -24,7 +24,7 @@ type ImmutableState struct {
 }
 
 // NewImmutableState creates a new immutable consensus backend state wrapper.
-func NewImmutableState(ctx context.Context, state api.ApplicationState, version int64) (*ImmutableState, error) {
+func NewImmutableState(ctx context.Context, state api.ApplicationQueryState, version int64) (*ImmutableState, error) {
 	is, err := api.NewImmutableState(ctx, state, version)
 	if err != nil {
 		return nil, err
diff --git a/go/consensus/tendermint/api/state.go b/go/consensus/tendermint/api/state.go
index aa7ffeda951..c2f19a1e66d 100644
--- a/go/consensus/tendermint/api/state.go
+++ b/go/consensus/tendermint/api/state.go
@@ -25,11 +25,7 @@ var (
 
 // ApplicationState is the overall past, present and future state of all multiplexed applications.
 type ApplicationState interface {
-	// Storage returns the storage backend.
-	Storage() storage.LocalBackend
-
-	// BlockHeight returns the last committed block height.
-	BlockHeight() int64
+	ApplicationQueryState
 
 	// BlockHash returns the last committed block hash.
 	BlockHash() []byte
@@ -49,9 +45,6 @@ type ApplicationState interface {
 	// GetBaseEpoch returns the base epoch.
 	GetBaseEpoch() (epochtime.EpochTime, error)
 
-	// GetEpoch returns epoch at block height.
-	GetEpoch(ctx context.Context, blockHeight int64) (epochtime.EpochTime, error)
-
 	// GetCurrentEpoch returns the epoch at the current block height.
 	GetCurrentEpoch(ctx context.Context) (epochtime.EpochTime, error)
 
@@ -69,6 +62,19 @@ type ApplicationState interface {
 	NewContext(mode ContextMode, now time.Time) *Context
 }
 
+// ApplicationQueryState is minimum methods required to service
+// ApplicationState queries.
+type ApplicationQueryState interface {
+	// Storage returns the storage backend.
+	Storage() storage.LocalBackend
+
+	// BlockHeight returns the last committed block height.
+	BlockHeight() int64
+
+	// GetEpoch returns epoch at block height.
+	GetEpoch(ctx context.Context, blockHeight int64) (epochtime.EpochTime, error)
+}
+
 // MockApplicationStateConfig is the configuration for the mock application state.
 type MockApplicationStateConfig struct {
 	BlockHeight int64
@@ -203,7 +209,7 @@ func (s *ImmutableState) Close() {
 }
 
 // NewImmutableState creates a new immutable state wrapper.
-func NewImmutableState(ctx context.Context, state ApplicationState, version int64) (*ImmutableState, error) {
+func NewImmutableState(ctx context.Context, state ApplicationQueryState, version int64) (*ImmutableState, error) {
 	if state == nil {
 		return nil, ErrNoState
 	}
diff --git a/go/consensus/tendermint/apps/beacon/query.go b/go/consensus/tendermint/apps/beacon/query.go
index 773f141128a..86fed9d6447 100644
--- a/go/consensus/tendermint/apps/beacon/query.go
+++ b/go/consensus/tendermint/apps/beacon/query.go
@@ -4,6 +4,7 @@ import (
 	"context"
 
 	beacon "github.com/oasislabs/oasis-core/go/beacon/api"
+	abciAPI "github.com/oasislabs/oasis-core/go/consensus/tendermint/api"
 	beaconState "github.com/oasislabs/oasis-core/go/consensus/tendermint/apps/beacon/state"
 )
 
@@ -15,12 +16,12 @@ type Query interface {
 
 // QueryFactory is the beacon query factory.
 type QueryFactory struct {
-	app *beaconApplication
+	state abciAPI.ApplicationQueryState
 }
 
 // QueryAt returns the beacon query interface for a specific height.
 func (sf *QueryFactory) QueryAt(ctx context.Context, height int64) (Query, error) {
-	state, err := beaconState.NewImmutableState(ctx, sf.app.state, height)
+	state, err := beaconState.NewImmutableState(ctx, sf.state, height)
 	if err != nil {
 		return nil, err
 	}
@@ -36,5 +37,11 @@ func (bq *beaconQuerier) Beacon(ctx context.Context) ([]byte, error) {
 }
 
 func (app *beaconApplication) QueryFactory() interface{} {
-	return &QueryFactory{app}
+	return &QueryFactory{app.state}
+}
+
+// NewQueryFactory returns a new QueryFactory backed by the given state
+// instance.
+func NewQueryFactory(state abciAPI.ApplicationQueryState) *QueryFactory {
+	return &QueryFactory{state}
 }
diff --git a/go/consensus/tendermint/apps/beacon/state/state.go b/go/consensus/tendermint/apps/beacon/state/state.go
index 6be0b63f8c6..79441303e2e 100644
--- a/go/consensus/tendermint/apps/beacon/state/state.go
+++ b/go/consensus/tendermint/apps/beacon/state/state.go
@@ -28,7 +28,7 @@ type ImmutableState struct {
 	is *abciAPI.ImmutableState
 }
 
-func NewImmutableState(ctx context.Context, state abciAPI.ApplicationState, version int64) (*ImmutableState, error) {
+func NewImmutableState(ctx context.Context, state abciAPI.ApplicationQueryState, version int64) (*ImmutableState, error) {
 	is, err := abciAPI.NewImmutableState(ctx, state, version)
 	if err != nil {
 		return nil, err
diff --git a/go/consensus/tendermint/apps/epochtime_mock/query.go b/go/consensus/tendermint/apps/epochtime_mock/query.go
index 5e27da7d0c2..583ed58b0e2 100644
--- a/go/consensus/tendermint/apps/epochtime_mock/query.go
+++ b/go/consensus/tendermint/apps/epochtime_mock/query.go
@@ -3,6 +3,7 @@ package epochtimemock
 import (
 	"context"
 
+	abciAPI "github.com/oasislabs/oasis-core/go/consensus/tendermint/api"
 	epochtime "github.com/oasislabs/oasis-core/go/epochtime/api"
 )
 
@@ -13,12 +14,12 @@ type Query interface {
 
 // QueryFactory is the mock epochtime query factory.
 type QueryFactory struct {
-	app *epochTimeMockApplication
+	state abciAPI.ApplicationQueryState
 }
 
 // QueryAt returns the mock epochtime query interface for a specific height.
 func (sf *QueryFactory) QueryAt(ctx context.Context, height int64) (Query, error) {
-	state, err := newImmutableState(ctx, sf.app.state, height)
+	state, err := newImmutableState(ctx, sf.state, height)
 	if err != nil {
 		return nil, err
 	}
@@ -34,5 +35,11 @@ func (eq *epochtimeMockQuerier) Epoch(ctx context.Context) (epochtime.EpochTime,
 }
 
 func (app *epochTimeMockApplication) QueryFactory() interface{} {
-	return &QueryFactory{app}
+	return &QueryFactory{app.state}
+}
+
+// NewQueryFactory returns a new QueryFactory backed by the given state
+// instance.
+func NewQueryFactory(state abciAPI.ApplicationQueryState) *QueryFactory {
+	return &QueryFactory{state}
 }
diff --git a/go/consensus/tendermint/apps/epochtime_mock/state.go b/go/consensus/tendermint/apps/epochtime_mock/state.go
index 04aa54c4008..589f18b7189 100644
--- a/go/consensus/tendermint/apps/epochtime_mock/state.go
+++ b/go/consensus/tendermint/apps/epochtime_mock/state.go
@@ -63,7 +63,7 @@ func (s *immutableState) getFutureEpoch(ctx context.Context) (*mockEpochTimeStat
 	return &state, nil
 }
 
-func newImmutableState(ctx context.Context, state abciAPI.ApplicationState, version int64) (*immutableState, error) {
+func newImmutableState(ctx context.Context, state abciAPI.ApplicationQueryState, version int64) (*immutableState, error) {
 	is, err := abciAPI.NewImmutableState(ctx, state, version)
 	if err != nil {
 		return nil, err
diff --git a/go/consensus/tendermint/apps/keymanager/query.go b/go/consensus/tendermint/apps/keymanager/query.go
index 8b6a965b341..830290b5d95 100644
--- a/go/consensus/tendermint/apps/keymanager/query.go
+++ b/go/consensus/tendermint/apps/keymanager/query.go
@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/oasislabs/oasis-core/go/common"
+	abciAPI "github.com/oasislabs/oasis-core/go/consensus/tendermint/api"
 	keymanagerState "github.com/oasislabs/oasis-core/go/consensus/tendermint/apps/keymanager/state"
 	keymanager "github.com/oasislabs/oasis-core/go/keymanager/api"
 )
@@ -17,12 +18,12 @@ type Query interface {
 
 // QueryFactory is the key manager query factory.
 type QueryFactory struct {
-	app *keymanagerApplication
+	state abciAPI.ApplicationQueryState
 }
 
 // QueryAt returns the key manager query interface for a specific height.
 func (sf *QueryFactory) QueryAt(ctx context.Context, height int64) (Query, error) {
-	state, err := keymanagerState.NewImmutableState(ctx, sf.app.state, height)
+	state, err := keymanagerState.NewImmutableState(ctx, sf.state, height)
 	if err != nil {
 		return nil, err
 	}
@@ -42,5 +43,11 @@ func (kq *keymanagerQuerier) Statuses(ctx context.Context) ([]*keymanager.Status
 }
 
 func (app *keymanagerApplication) QueryFactory() interface{} {
-	return &QueryFactory{app}
+	return &QueryFactory{app.state}
+}
+
+// NewQueryFactory returns a new QueryFactory backed by the given state
+// instance.
+func NewQueryFactory(state abciAPI.ApplicationQueryState) *QueryFactory {
+	return &QueryFactory{state}
 }
diff --git a/go/consensus/tendermint/apps/keymanager/state/state.go b/go/consensus/tendermint/apps/keymanager/state/state.go
index 682ba383bed..2dda20df70f 100644
--- a/go/consensus/tendermint/apps/keymanager/state/state.go
+++ b/go/consensus/tendermint/apps/keymanager/state/state.go
@@ -74,7 +74,7 @@ func (st *ImmutableState) Status(ctx context.Context, id common.Namespace) (*api
 	return &status, nil
 }
 
-func NewImmutableState(ctx context.Context, state abciAPI.ApplicationState, version int64) (*ImmutableState, error) {
+func NewImmutableState(ctx context.Context, state abciAPI.ApplicationQueryState, version int64) (*ImmutableState, error) {
 	is, err := abciAPI.NewImmutableState(ctx, state, version)
 	if err != nil {
 		return nil, err
diff --git a/go/consensus/tendermint/apps/registry/query.go b/go/consensus/tendermint/apps/registry/query.go
index 5e838533ede..7e08d9ed35c 100644
--- a/go/consensus/tendermint/apps/registry/query.go
+++ b/go/consensus/tendermint/apps/registry/query.go
@@ -8,6 +8,7 @@ import (
 	"github.com/oasislabs/oasis-core/go/common/crypto/signature"
 	"github.com/oasislabs/oasis-core/go/common/entity"
 	"github.com/oasislabs/oasis-core/go/common/node"
+	abciAPI "github.com/oasislabs/oasis-core/go/consensus/tendermint/api"
 	registryState "github.com/oasislabs/oasis-core/go/consensus/tendermint/apps/registry/state"
 	registry "github.com/oasislabs/oasis-core/go/registry/api"
 )
@@ -26,22 +27,22 @@ type Query interface {
 
 // QueryFactory is the registry query factory.
 type QueryFactory struct {
-	app *registryApplication
+	state abciAPI.ApplicationQueryState
 }
 
 // QueryAt returns the registry query interface for a specific height.
 func (sf *QueryFactory) QueryAt(ctx context.Context, height int64) (Query, error) {
-	state, err := registryState.NewImmutableState(ctx, sf.app.state, height)
+	state, err := registryState.NewImmutableState(ctx, sf.state, height)
 	if err != nil {
 		return nil, err
 	}
-	return &registryQuerier{sf.app, state, height}, nil
+	return &registryQuerier{sf.state, state, height}, nil
 }
 
 type registryQuerier struct {
-	app    *registryApplication
-	state  *registryState.ImmutableState
-	height int64
+	queryState abciAPI.ApplicationQueryState
+	state      *registryState.ImmutableState
+	height     int64
 }
 
 func (rq *registryQuerier) Entity(ctx context.Context, id signature.PublicKey) (*entity.Entity, error) {
@@ -53,7 +54,7 @@ func (rq *registryQuerier) Entities(ctx context.Context) ([]*entity.Entity, erro
 }
 
 func (rq *registryQuerier) Node(ctx context.Context, id signature.PublicKey) (*node.Node, error) {
-	epoch, err := rq.app.state.GetEpoch(ctx, rq.height)
+	epoch, err := rq.queryState.GetEpoch(ctx, rq.height)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get epoch: %w", err)
 	}
@@ -75,7 +76,7 @@ func (rq *registryQuerier) NodeStatus(ctx context.Context, id signature.PublicKe
 }
 
 func (rq *registryQuerier) Nodes(ctx context.Context) ([]*node.Node, error) {
-	epoch, err := rq.app.state.GetEpoch(ctx, rq.height)
+	epoch, err := rq.queryState.GetEpoch(ctx, rq.height)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get epoch: %w", err)
 	}
@@ -105,5 +106,11 @@ func (rq *registryQuerier) Runtimes(ctx context.Context) ([]*registry.Runtime, e
 }
 
 func (app *registryApplication) QueryFactory() interface{} {
-	return &QueryFactory{app}
+	return &QueryFactory{app.state}
+}
+
+// NewQueryFactory returns a new QueryFactory backed by the given state
+// instance.
+func NewQueryFactory(state abciAPI.ApplicationQueryState) *QueryFactory {
+	return &QueryFactory{state}
 }
diff --git a/go/consensus/tendermint/apps/registry/state/state.go b/go/consensus/tendermint/apps/registry/state/state.go
index c72d4aaf3ab..68a3b605831 100644
--- a/go/consensus/tendermint/apps/registry/state/state.go
+++ b/go/consensus/tendermint/apps/registry/state/state.go
@@ -530,7 +530,7 @@ func (s *ImmutableState) NodeBySubKey(ctx context.Context, key signature.PublicK
 	return s.Node(ctx, id)
 }
 
-func NewImmutableState(ctx context.Context, state abciAPI.ApplicationState, version int64) (*ImmutableState, error) {
+func NewImmutableState(ctx context.Context, state abciAPI.ApplicationQueryState, version int64) (*ImmutableState, error) {
 	is, err := abciAPI.NewImmutableState(ctx, state, version)
 	if err != nil {
 		return nil, err
diff --git a/go/consensus/tendermint/apps/roothash/query.go b/go/consensus/tendermint/apps/roothash/query.go
index 4f313b5fb6d..2db2d3f76e6 100644
--- a/go/consensus/tendermint/apps/roothash/query.go
+++ b/go/consensus/tendermint/apps/roothash/query.go
@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/oasislabs/oasis-core/go/common"
+	abciAPI "github.com/oasislabs/oasis-core/go/consensus/tendermint/api"
 	roothashState "github.com/oasislabs/oasis-core/go/consensus/tendermint/apps/roothash/state"
 	roothash "github.com/oasislabs/oasis-core/go/roothash/api"
 	"github.com/oasislabs/oasis-core/go/roothash/api/block"
@@ -18,12 +19,12 @@ type Query interface {
 
 // QueryFactory is the roothash query factory.
 type QueryFactory struct {
-	app *rootHashApplication
+	state abciAPI.ApplicationQueryState
 }
 
 // QueryAt returns the roothash query interface for a specific height.
 func (sf *QueryFactory) QueryAt(ctx context.Context, height int64) (Query, error) {
-	state, err := roothashState.NewImmutableState(ctx, sf.app.state, height)
+	state, err := roothashState.NewImmutableState(ctx, sf.state, height)
 	if err != nil {
 		return nil, err
 	}
@@ -51,5 +52,11 @@ func (rq *rootHashQuerier) GenesisBlock(ctx context.Context, id common.Namespace
 }
 
 func (app *rootHashApplication) QueryFactory() interface{} {
-	return &QueryFactory{app}
+	return &QueryFactory{app.state}
+}
+
+// NewQueryFactory returns a new QueryFactory backed by the given state
+// instance.
+func NewQueryFactory(state abciAPI.ApplicationQueryState) *QueryFactory {
+	return &QueryFactory{state}
 }
diff --git a/go/consensus/tendermint/apps/roothash/state/state.go b/go/consensus/tendermint/apps/roothash/state/state.go
index 34acf8e746c..94d9629005d 100644
--- a/go/consensus/tendermint/apps/roothash/state/state.go
+++ b/go/consensus/tendermint/apps/roothash/state/state.go
@@ -43,7 +43,7 @@ type ImmutableState struct {
 	is *abciAPI.ImmutableState
 }
 
-func NewImmutableState(ctx context.Context, state abciAPI.ApplicationState, version int64) (*ImmutableState, error) {
+func NewImmutableState(ctx context.Context, state abciAPI.ApplicationQueryState, version int64) (*ImmutableState, error) {
 	is, err := abciAPI.NewImmutableState(ctx, state, version)
 	if err != nil {
 		return nil, err
diff --git a/go/consensus/tendermint/apps/scheduler/query.go b/go/consensus/tendermint/apps/scheduler/query.go
index a23d888d22b..ff6d38d59ec 100644
--- a/go/consensus/tendermint/apps/scheduler/query.go
+++ b/go/consensus/tendermint/apps/scheduler/query.go
@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/oasislabs/oasis-core/go/common/crypto/signature"
+	abciAPI "github.com/oasislabs/oasis-core/go/consensus/tendermint/api"
 	registryState "github.com/oasislabs/oasis-core/go/consensus/tendermint/apps/registry/state"
 	schedulerState "github.com/oasislabs/oasis-core/go/consensus/tendermint/apps/scheduler/state"
 	scheduler "github.com/oasislabs/oasis-core/go/scheduler/api"
@@ -19,18 +20,18 @@ type Query interface {
 
 // QueryFactory is the scheduler query factory.
 type QueryFactory struct {
-	app *schedulerApplication
+	state abciAPI.ApplicationQueryState
 }
 
 // QueryAt returns the scheduler query interface for a specific height.
 func (sf *QueryFactory) QueryAt(ctx context.Context, height int64) (Query, error) {
-	state, err := schedulerState.NewImmutableState(ctx, sf.app.state, height)
+	state, err := schedulerState.NewImmutableState(ctx, sf.state, height)
 	if err != nil {
 		return nil, err
 	}
 
 	// Some queries need access to the registry to give useful responses.
-	regState, err := registryState.NewImmutableState(ctx, sf.app.state, height)
+	regState, err := registryState.NewImmutableState(ctx, sf.state, height)
 	if err != nil {
 		return nil, err
 	}
@@ -98,5 +99,11 @@ func (sq *schedulerQuerier) KindsCommittees(ctx context.Context, kinds []schedul
 }
 
 func (app *schedulerApplication) QueryFactory() interface{} {
-	return &QueryFactory{app}
+	return &QueryFactory{app.state}
+}
+
+// NewQueryFactory returns a new QueryFactory backed by the given state
+// instance.
+func NewQueryFactory(state abciAPI.ApplicationQueryState) *QueryFactory {
+	return &QueryFactory{state}
 }
diff --git a/go/consensus/tendermint/apps/scheduler/state/state.go b/go/consensus/tendermint/apps/scheduler/state/state.go
index fa2723cb0f7..74e58ac9a01 100644
--- a/go/consensus/tendermint/apps/scheduler/state/state.go
+++ b/go/consensus/tendermint/apps/scheduler/state/state.go
@@ -163,7 +163,7 @@ func (s *ImmutableState) ConsensusParameters(ctx context.Context) (*api.Consensu
 	return &params, nil
 }
 
-func NewImmutableState(ctx context.Context, state abciAPI.ApplicationState, version int64) (*ImmutableState, error) {
+func NewImmutableState(ctx context.Context, state abciAPI.ApplicationQueryState, version int64) (*ImmutableState, error) {
 	is, err := abciAPI.NewImmutableState(ctx, state, version)
 	if err != nil {
 		return nil, err
diff --git a/go/consensus/tendermint/apps/staking/query.go b/go/consensus/tendermint/apps/staking/query.go
index a7f40aa728d..24a4809e95f 100644
--- a/go/consensus/tendermint/apps/staking/query.go
+++ b/go/consensus/tendermint/apps/staking/query.go
@@ -5,6 +5,7 @@ import (
 
 	"github.com/oasislabs/oasis-core/go/common/crypto/signature"
 	"github.com/oasislabs/oasis-core/go/common/quantity"
+	abciAPI "github.com/oasislabs/oasis-core/go/consensus/tendermint/api"
 	stakingState "github.com/oasislabs/oasis-core/go/consensus/tendermint/apps/staking/state"
 	epochtime "github.com/oasislabs/oasis-core/go/epochtime/api"
 	staking "github.com/oasislabs/oasis-core/go/staking/api"
@@ -27,12 +28,12 @@ type Query interface {
 
 // QueryFactory is the staking query factory.
 type QueryFactory struct {
-	app *stakingApplication
+	state abciAPI.ApplicationQueryState
 }
 
 // QueryAt returns the staking query interface for a specific height.
 func (sf *QueryFactory) QueryAt(ctx context.Context, height int64) (Query, error) {
-	state, err := stakingState.NewImmutableState(ctx, sf.app.state, height)
+	state, err := stakingState.NewImmutableState(ctx, sf.state, height)
 	if err != nil {
 		return nil, err
 	}
@@ -93,5 +94,11 @@ func (sq *stakingQuerier) ConsensusParameters(ctx context.Context) (*staking.Con
 }
 
 func (app *stakingApplication) QueryFactory() interface{} {
-	return &QueryFactory{app}
+	return &QueryFactory{app.state}
+}
+
+// NewQueryFactory returns a new QueryFactory backed by the given state
+// instance.
+func NewQueryFactory(state abciAPI.ApplicationQueryState) *QueryFactory {
+	return &QueryFactory{state}
 }
diff --git a/go/consensus/tendermint/apps/staking/state/state.go b/go/consensus/tendermint/apps/staking/state/state.go
index c1f055f8a1b..f1e8260dfbb 100644
--- a/go/consensus/tendermint/apps/staking/state/state.go
+++ b/go/consensus/tendermint/apps/staking/state/state.go
@@ -489,7 +489,7 @@ func (s *ImmutableState) EpochSigning(ctx context.Context) (*EpochSigning, error
 	return &es, nil
 }
 
-func NewImmutableState(ctx context.Context, state abciAPI.ApplicationState, version int64) (*ImmutableState, error) {
+func NewImmutableState(ctx context.Context, state abciAPI.ApplicationQueryState, version int64) (*ImmutableState, error) {
 	is, err := abciAPI.NewImmutableState(ctx, state, version)
 	if err != nil {
 		return nil, err

From 00c0ed319392caa19227cf3808be50a9571cb26b Mon Sep 17 00:00:00 2001
From: Yawning Angel <yawning@schwanenlied.me>
Date: Tue, 19 May 2020 14:44:08 +0000
Subject: [PATCH 4/4] go/oasis-node/cmd/debug: Add the `dumpdb` command

This command will attempt to extract the ABCI state from a combination
of a shutdown node's on-disk database and the genesis document currently
being used by the network, and will write the output as a JSON formatted
genesis document.

Some caveats:

- It is not guaranteed that the dumped output will be usable as an
  actual genesis document without manual intervention.

- Only the state that would be exported via a normal dump from a running
  node will be present in the dump.

- The epochtime base will be that of the original genesis document, and
  not the most recent epoch (different from a genesis dump).
---
 .changelog/2359.feature.md                    |  17 +
 go/oasis-node/cmd/debug/debug.go              |   2 +
 go/oasis-node/cmd/debug/dumpdb/dumpdb.go      | 387 ++++++++++++++++++
 go/oasis-test-runner/cmd/root.go              |   1 +
 .../scenario/e2e/dump_restore.go              |   2 +-
 .../scenario/e2e/gas_fees_staking.go          |   2 +-
 go/oasis-test-runner/scenario/e2e/runtime.go  |  68 ++-
 7 files changed, 476 insertions(+), 3 deletions(-)
 create mode 100644 .changelog/2359.feature.md
 create mode 100644 go/oasis-node/cmd/debug/dumpdb/dumpdb.go

diff --git a/.changelog/2359.feature.md b/.changelog/2359.feature.md
new file mode 100644
index 00000000000..77468fa9e7f
--- /dev/null
+++ b/.changelog/2359.feature.md
@@ -0,0 +1,17 @@
+go/oasis-node/cmd/debug: Add the `dumpdb` command
+
+This command will attempt to extract the ABCI state from a combination
+of a shutdown node's on-disk database and the genesis document currently
+being used by the network, and will write the output as a JSON formatted
+genesis document.
+
+Some caveats:
+
+- It is not guaranteed that the dumped output will be usable as an
+  actual genesis document without manual intervention.
+
+- Only the state that would be exported via a normal dump from a running
+  node will be present in the dump.
+
+- The epochtime base will be that of the original genesis document, and
+  not the most recent epoch (different from a genesis dump).
diff --git a/go/oasis-node/cmd/debug/debug.go b/go/oasis-node/cmd/debug/debug.go
index 4adfd41a994..a94dec80183 100644
--- a/go/oasis-node/cmd/debug/debug.go
+++ b/go/oasis-node/cmd/debug/debug.go
@@ -7,6 +7,7 @@ import (
 	"github.com/oasislabs/oasis-core/go/oasis-node/cmd/debug/byzantine"
 	"github.com/oasislabs/oasis-core/go/oasis-node/cmd/debug/consim"
 	"github.com/oasislabs/oasis-core/go/oasis-node/cmd/debug/control"
+	"github.com/oasislabs/oasis-core/go/oasis-node/cmd/debug/dumpdb"
 	"github.com/oasislabs/oasis-core/go/oasis-node/cmd/debug/fixgenesis"
 	"github.com/oasislabs/oasis-core/go/oasis-node/cmd/debug/storage"
 	"github.com/oasislabs/oasis-core/go/oasis-node/cmd/debug/txsource"
@@ -25,6 +26,7 @@ func Register(parentCmd *cobra.Command) {
 	fixgenesis.Register(debugCmd)
 	control.Register(debugCmd)
 	consim.Register(debugCmd)
+	dumpdb.Register(debugCmd)
 
 	parentCmd.AddCommand(debugCmd)
 }
diff --git a/go/oasis-node/cmd/debug/dumpdb/dumpdb.go b/go/oasis-node/cmd/debug/dumpdb/dumpdb.go
new file mode 100644
index 00000000000..6165ec92944
--- /dev/null
+++ b/go/oasis-node/cmd/debug/dumpdb/dumpdb.go
@@ -0,0 +1,387 @@
+// Package dumpdb implements the dumpdb sub-command.
+package dumpdb
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/spf13/cobra"
+	flag "github.com/spf13/pflag"
+	"github.com/spf13/viper"
+
+	beacon "github.com/oasislabs/oasis-core/go/beacon/api"
+	"github.com/oasislabs/oasis-core/go/common/logging"
+	consensus "github.com/oasislabs/oasis-core/go/consensus/genesis"
+	tendermint "github.com/oasislabs/oasis-core/go/consensus/tendermint"
+	"github.com/oasislabs/oasis-core/go/consensus/tendermint/abci"
+	abciState "github.com/oasislabs/oasis-core/go/consensus/tendermint/abci/state"
+	tendermintAPI "github.com/oasislabs/oasis-core/go/consensus/tendermint/api"
+	beaconApp "github.com/oasislabs/oasis-core/go/consensus/tendermint/apps/beacon"
+	keymanagerApp "github.com/oasislabs/oasis-core/go/consensus/tendermint/apps/keymanager"
+	registryApp "github.com/oasislabs/oasis-core/go/consensus/tendermint/apps/registry"
+	roothashApp "github.com/oasislabs/oasis-core/go/consensus/tendermint/apps/roothash"
+	schedulerApp "github.com/oasislabs/oasis-core/go/consensus/tendermint/apps/scheduler"
+	stakingApp "github.com/oasislabs/oasis-core/go/consensus/tendermint/apps/staking"
+	epochtime "github.com/oasislabs/oasis-core/go/epochtime/api"
+	genesis "github.com/oasislabs/oasis-core/go/genesis/api"
+	genesisFile "github.com/oasislabs/oasis-core/go/genesis/file"
+	keymanager "github.com/oasislabs/oasis-core/go/keymanager/api"
+	cmdCommon "github.com/oasislabs/oasis-core/go/oasis-node/cmd/common"
+	"github.com/oasislabs/oasis-core/go/oasis-node/cmd/common/flags"
+	registry "github.com/oasislabs/oasis-core/go/registry/api"
+	roothash "github.com/oasislabs/oasis-core/go/roothash/api"
+	scheduler "github.com/oasislabs/oasis-core/go/scheduler/api"
+	staking "github.com/oasislabs/oasis-core/go/staking/api"
+	storage "github.com/oasislabs/oasis-core/go/storage/api"
+	storageDB "github.com/oasislabs/oasis-core/go/storage/database"
+
+	"encoding/json"
+)
+
+const (
+	cfgDumpOutput     = "dump.output"
+	cfgDumpReadOnlyDB = "dump.read_only_db"
+	cfgDumpVersion    = "dump.version"
+)
+
+var (
+	dumpDBCmd = &cobra.Command{
+		Use:   "dumpdb",
+		Short: "dump the on-disk consensus DB to a JSON document",
+		Run:   doDumpDB,
+	}
+
+	dumpDBFlags = flag.NewFlagSet("", flag.ContinueOnError)
+
+	logger = logging.GetLogger("cmd/debug/dumpdb")
+)
+
+func doDumpDB(cmd *cobra.Command, args []string) {
+	var ok bool
+	defer func() {
+		if !ok {
+			os.Exit(1)
+		}
+	}()
+
+	if err := cmdCommon.Init(); err != nil {
+		cmdCommon.EarlyLogAndExit(err)
+	}
+
+	dataDir := cmdCommon.DataDir()
+	if dataDir == "" {
+		logger.Error("data directory must be set")
+		return
+	}
+
+	// Load the old genesis document, required for filling in parameters
+	// that are not persisted to ABCI state.
+	fp, err := genesisFile.NewFileProvider(flags.GenesisFile())
+	if err != nil {
+		logger.Error("failed to load existing genesis document",
+			"err", err,
+		)
+		return
+	}
+	oldDoc, err := fp.GetGenesisDocument()
+	if err != nil {
+		logger.Error("failed to get existing genesis document",
+			"err", err,
+		)
+		return
+	}
+
+	// Initialize the ABCI state storage for access.
+	//
+	// Note: While it would be great to always use read-only DB access,
+	// badger will refuse to open a DB that isn't closed properly in
+	// read-only mode because it needs to truncate the value log.
+	//
+	// Hope you have backups if you ever run into this.
+	ctx := context.Background()
+	ldb, _, stateRoot, err := abci.InitStateStorage(
+		ctx,
+		&abci.ApplicationConfig{
+			DataDir:           filepath.Join(dataDir, tendermint.StateDir),
+			StorageBackend:    storageDB.BackendNameBadgerDB, // No other backend for now.
+			MemoryOnlyStorage: false,
+			ReadOnlyStorage:   viper.GetBool(cfgDumpReadOnlyDB),
+		},
+	)
+	if err != nil {
+		logger.Error("failed to initialize ABCI storage backend",
+			"err", err,
+		)
+	}
+	defer ldb.Cleanup()
+
+	latestVersion := int64(stateRoot.Version)
+	dumpVersion := viper.GetInt64(cfgDumpVersion)
+	if dumpVersion == 0 {
+		dumpVersion = latestVersion
+	}
+	if dumpVersion <= 0 || dumpVersion > latestVersion {
+		logger.Error("dump requested for version that does not exist",
+			"dump_version", dumpVersion,
+			"latest_version", latestVersion,
+		)
+		return
+	}
+
+	// Generate the dump by querying all of the relevant backends, and
+	// extracting the immutable parameters from the current genesis
+	// document.
+	//
+	// WARNING: The state is not guaranteed to be usable as a genesis
+	// document without manual intervention, and only the state that
+	// would be exported by the normal dump process will be present
+	// in the dump.
+	qs := &dumpQueryState{
+		ldb:    ldb,
+		height: dumpVersion,
+	}
+	doc := &genesis.Document{
+		Height:    qs.BlockHeight(),
+		Time:      time.Now(), // XXX: Make this deterministic?
+		ChainID:   oldDoc.ChainID,
+		EpochTime: oldDoc.EpochTime,
+		HaltEpoch: oldDoc.HaltEpoch,
+		ExtraData: oldDoc.ExtraData,
+	}
+
+	// BUG(?): EpochTime.Base in a exported dump will be set to the
+	// current epoch, this uses the original genesis doc.  I'm not
+	// sure if there is a right answer here.
+
+	// Registry
+	registrySt, err := dumpRegistry(ctx, qs)
+	if err != nil {
+		logger.Error("failed to dump registry state",
+			"err", err,
+		)
+	}
+	doc.Registry = *registrySt
+
+	// RootHash
+	rootHashSt, err := dumpRootHash(ctx, qs)
+	if err != nil {
+		logger.Error("failed to dump root hash state",
+			"err", err,
+		)
+		return
+	}
+	doc.RootHash = *rootHashSt
+
+	// Staking
+	stakingSt, err := dumpStaking(ctx, qs)
+	if err != nil {
+		logger.Error("failed to dump staking state",
+			"err", err,
+		)
+		return
+	}
+	doc.Staking = *stakingSt
+
+	// KeyManager
+	keyManagerSt, err := dumpKeyManager(ctx, qs)
+	if err != nil {
+		logger.Error("failed to dump key manager state",
+			"err", err,
+		)
+		return
+	}
+	doc.KeyManager = *keyManagerSt
+
+	// Scheduler
+	schedulerSt, err := dumpScheduler(ctx, qs)
+	if err != nil {
+		logger.Error("failed to dump scheduler state",
+			"err", err,
+		)
+		return
+	}
+	doc.Scheduler = *schedulerSt
+
+	// Beacon
+	beaconSt, err := dumpBeacon(ctx, qs)
+	if err != nil {
+		logger.Error("failed to dump beacon state",
+			"err", err,
+		)
+		return
+	}
+	doc.Beacon = *beaconSt
+
+	// Consensus
+	consensusSt, err := dumpConsensus(ctx, qs)
+	if err != nil {
+		logger.Error("failed to dump consensus state",
+			"err", err,
+		)
+		return
+	}
+	doc.Consensus = *consensusSt
+
+	logger.Info("writing state dump",
+		"output", viper.GetString(cfgDumpOutput),
+	)
+
+	// Write out the document.
+	w, shouldClose, err := cmdCommon.GetOutputWriter(cmd, cfgDumpOutput)
+	if err != nil {
+		logger.Error("failed to get output writer for state dump",
+			"err", err,
+		)
+		return
+	}
+	if shouldClose {
+		defer w.Close()
+	}
+	raw, err := json.Marshal(doc)
+	if err != nil {
+		logger.Error("failed to marshal state dump into JSON",
+			"err", err,
+		)
+		return
+	}
+	if _, err := w.Write(raw); err != nil {
+		logger.Error("failed to write state dump file",
+			"err", err,
+		)
+		return
+	}
+
+	ok = true
+}
+
+func dumpRegistry(ctx context.Context, qs *dumpQueryState) (*registry.Genesis, error) {
+	qf := registryApp.NewQueryFactory(qs)
+	q, err := qf.QueryAt(ctx, qs.BlockHeight())
+	if err != nil {
+		return nil, fmt.Errorf("dumpdb: failed to create registry query: %w", err)
+	}
+	st, err := q.Genesis(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("dumpdb: failed to dump registry state: %w", err)
+	}
+	return st, nil
+}
+
+func dumpRootHash(ctx context.Context, qs *dumpQueryState) (*roothash.Genesis, error) {
+	qf := roothashApp.NewQueryFactory(qs)
+	q, err := qf.QueryAt(ctx, qs.BlockHeight())
+	if err != nil {
+		return nil, fmt.Errorf("dumpdb: failed to create root hash query: %w", err)
+	}
+	st, err := q.Genesis(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("dumpdb: failed to dump root hash state: %w", err)
+	}
+	return st, nil
+}
+
+func dumpStaking(ctx context.Context, qs *dumpQueryState) (*staking.Genesis, error) {
+	qf := stakingApp.NewQueryFactory(qs)
+	q, err := qf.QueryAt(ctx, qs.BlockHeight())
+	if err != nil {
+		return nil, fmt.Errorf("dumpdb: failed to create staking query: %w", err)
+	}
+	st, err := q.Genesis(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("dumpdb: failed to dump staking state: %w", err)
+	}
+	return st, nil
+}
+
+func dumpKeyManager(ctx context.Context, qs *dumpQueryState) (*keymanager.Genesis, error) {
+	qf := keymanagerApp.NewQueryFactory(qs)
+	q, err := qf.QueryAt(ctx, qs.BlockHeight())
+	if err != nil {
+		return nil, fmt.Errorf("dumpdb: failed to create key manager query: %w", err)
+	}
+	st, err := q.Genesis(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("dumpdb: failed to dump key manager state: %w", err)
+	}
+	return st, nil
+}
+
+func dumpScheduler(ctx context.Context, qs *dumpQueryState) (*scheduler.Genesis, error) {
+	qf := schedulerApp.NewQueryFactory(qs)
+	q, err := qf.QueryAt(ctx, qs.BlockHeight())
+	if err != nil {
+		return nil, fmt.Errorf("dumpdb: failed to create scheduler query: %w", err)
+	}
+	st, err := q.Genesis(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("dumpdb: failed to dump scheduler state: %w", err)
+	}
+	return st, nil
+}
+
+func dumpBeacon(ctx context.Context, qs *dumpQueryState) (*beacon.Genesis, error) {
+	qf := beaconApp.NewQueryFactory(qs)
+	q, err := qf.QueryAt(ctx, qs.BlockHeight())
+	if err != nil {
+		return nil, fmt.Errorf("dumpdb: failed to create beacon query: %w", err)
+	}
+	st, err := q.Genesis(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("dumpdb: failed to dump beacon state: %w", err)
+	}
+	return st, nil
+}
+
+func dumpConsensus(ctx context.Context, qs *dumpQueryState) (*consensus.Genesis, error) {
+	is, err := abciState.NewImmutableState(ctx, qs, qs.BlockHeight())
+	if err != nil {
+		return nil, fmt.Errorf("dumpdb: failed to get consensus state: %w", err)
+	}
+	params, err := is.ConsensusParameters(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("dumpdb: failed to get consensus params: %w", err)
+	}
+	_ = params
+	return &consensus.Genesis{
+		Backend:    tendermintAPI.BackendName,
+		Parameters: *params,
+	}, nil
+}
+
+type dumpQueryState struct {
+	ldb    storage.LocalBackend
+	height int64
+}
+
+func (qs *dumpQueryState) Storage() storage.LocalBackend {
+	return qs.ldb
+}
+
+func (qs *dumpQueryState) BlockHeight() int64 {
+	return qs.height
+}
+
+func (qs *dumpQueryState) GetEpoch(ctx context.Context, blockHeight int64) (epochtime.EpochTime, error) {
+	// This is only required because certain registry backend queries
+	// need the epoch to filter out expired nodes.  It is not
+	// implemented because acquiring a full state dump does not
+	// involve any of the relevant queries.
+	return epochtime.EpochTime(0), fmt.Errorf("dumpdb/dumpQueryState: GetEpoch not supported")
+}
+
+// Register registers the dumpdb sub-commands.
+func Register(parentCmd *cobra.Command) {
+	dumpDBCmd.Flags().AddFlagSet(flags.GenesisFileFlags)
+	dumpDBCmd.Flags().AddFlagSet(dumpDBFlags)
+	parentCmd.AddCommand(dumpDBCmd)
+}
+
+func init() {
+	dumpDBFlags.String(cfgDumpOutput, "dump.json", "path to dumped ABCI state")
+	dumpDBFlags.Bool(cfgDumpReadOnlyDB, false, "read-only DB access")
+	dumpDBFlags.Int64(cfgDumpVersion, 0, "ABCI state version to dump (0 = most recent)")
+	_ = viper.BindPFlags(dumpDBFlags)
+}
diff --git a/go/oasis-test-runner/cmd/root.go b/go/oasis-test-runner/cmd/root.go
index d5a0cc3d123..75c06b15ecf 100644
--- a/go/oasis-test-runner/cmd/root.go
+++ b/go/oasis-test-runner/cmd/root.go
@@ -550,5 +550,6 @@ func init() {
 		}
 
 		viper.Set(nodeFlags.CfgDebugDontBlameOasis, true)
+		viper.Set(nodeCommon.CfgDebugAllowTestKeys, true)
 	})
 }
diff --git a/go/oasis-test-runner/scenario/e2e/dump_restore.go b/go/oasis-test-runner/scenario/e2e/dump_restore.go
index 4269b7dfdda..d0928e94eb5 100644
--- a/go/oasis-test-runner/scenario/e2e/dump_restore.go
+++ b/go/oasis-test-runner/scenario/e2e/dump_restore.go
@@ -52,7 +52,7 @@ func (sc *dumpRestoreImpl) Run(childEnv *env.Env) error {
 	if err != nil {
 		return err
 	}
-	if err = sc.dumpRestoreNetwork(childEnv, fixture); err != nil {
+	if err = sc.dumpRestoreNetwork(childEnv, fixture, true); err != nil {
 		return err
 	}
 
diff --git a/go/oasis-test-runner/scenario/e2e/gas_fees_staking.go b/go/oasis-test-runner/scenario/e2e/gas_fees_staking.go
index deff9817f7f..8fc1bcc0002 100644
--- a/go/oasis-test-runner/scenario/e2e/gas_fees_staking.go
+++ b/go/oasis-test-runner/scenario/e2e/gas_fees_staking.go
@@ -121,7 +121,7 @@ func (sc *gasFeesImpl) Run(childEnv *env.Env) error {
 		if err != nil {
 			return err
 		}
-		if err := sc.dumpRestoreNetwork(childEnv, fixture); err != nil {
+		if err := sc.dumpRestoreNetwork(childEnv, fixture, false); err != nil {
 			return err
 		}
 		if err := sc.runTests(ctx); err != nil {
diff --git a/go/oasis-test-runner/scenario/e2e/runtime.go b/go/oasis-test-runner/scenario/e2e/runtime.go
index eeece0e65ac..a9197aa56d2 100644
--- a/go/oasis-test-runner/scenario/e2e/runtime.go
+++ b/go/oasis-test-runner/scenario/e2e/runtime.go
@@ -1,7 +1,9 @@
 package e2e
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"os/exec"
 	"path/filepath"
@@ -14,6 +16,7 @@ import (
 	"github.com/oasislabs/oasis-core/go/common/cbor"
 	"github.com/oasislabs/oasis-core/go/common/node"
 	"github.com/oasislabs/oasis-core/go/common/sgx"
+	genesisFile "github.com/oasislabs/oasis-core/go/genesis/file"
 	cmdCommon "github.com/oasislabs/oasis-core/go/oasis-node/cmd/common"
 	cmdNode "github.com/oasislabs/oasis-core/go/oasis-node/cmd/node"
 	"github.com/oasislabs/oasis-core/go/oasis-test-runner/env"
@@ -362,7 +365,7 @@ func (sc *runtimeImpl) cleanTendermintStorage(childEnv *env.Env) error {
 	return nil
 }
 
-func (sc *runtimeImpl) dumpRestoreNetwork(childEnv *env.Env, fixture *oasis.NetworkFixture) error {
+func (sc *runtimeImpl) dumpRestoreNetwork(childEnv *env.Env, fixture *oasis.NetworkFixture, doDbDump bool) error {
 	// Dump-restore network.
 	sc.logger.Info("dumping network state",
 		"child", childEnv,
@@ -384,6 +387,12 @@ func (sc *runtimeImpl) dumpRestoreNetwork(childEnv *env.Env, fixture *oasis.Netw
 	logger.Info("stopping the network")
 	sc.net.Stop()
 
+	if doDbDump {
+		if err := sc.dumpDatabase(childEnv, fixture, dumpPath); err != nil {
+			return fmt.Errorf("scenario/e2e/dump_restore: failed to dump database: %w", err)
+		}
+	}
+
 	if len(sc.net.StorageWorkers()) > 0 {
 		// Dump storage.
 		args = []string{
@@ -427,6 +436,63 @@ func (sc *runtimeImpl) dumpRestoreNetwork(childEnv *env.Env, fixture *oasis.Netw
 	return nil
 }
 
+func (sc *runtimeImpl) dumpDatabase(childEnv *env.Env, fixture *oasis.NetworkFixture, exportPath string) error {
+	// Load the existing export.
+	eFp, err := genesisFile.NewFileProvider(exportPath)
+	if err != nil {
+		return fmt.Errorf("failed to instantiate file provider (export): %w", err)
+	}
+	exportedDoc, err := eFp.GetGenesisDocument()
+	if err != nil {
+		return fmt.Errorf("failed to get genesis doc (export): %w", err)
+	}
+
+	logger.Info("dumping via debug dumpdb")
+
+	// Dump the state with the debug command off one of the validators.
+	dbDumpPath := filepath.Join(childEnv.Dir(), "debug_dump.json")
+	args := []string{
+		"debug", "dumpdb",
+		"--datadir", sc.net.Validators()[0].DataDir(),
+		"-g", sc.net.GenesisPath(),
+		"--dump.version", fmt.Sprintf("%d", exportedDoc.Height),
+		"--dump.output", dbDumpPath,
+		"--debug.dont_blame_oasis",
+		"--debug.allow_test_keys",
+	}
+	if err = cli.RunSubCommand(childEnv, logger, "debug-dump", sc.net.Config().NodeBinary, args); err != nil {
+		return fmt.Errorf("failed to dump database: %w", err)
+	}
+
+	// Load the dumped state.
+	fp, err := genesisFile.NewFileProvider(dbDumpPath)
+	if err != nil {
+		return fmt.Errorf("failed to instantiate file provider (db): %w", err)
+	}
+	dbDoc, err := fp.GetGenesisDocument()
+	if err != nil {
+		return fmt.Errorf("failed to get genesis doc (dump): %w", err)
+	}
+
+	// Compare the two documents for approximate equality.  Note: Certain
+	// fields will be different, so those are fixed up before the comparison.
+	dbDoc.EpochTime.Base = exportedDoc.EpochTime.Base
+	dbDoc.Time = exportedDoc.Time
+	dbRaw, err := json.Marshal(dbDoc)
+	if err != nil {
+		return fmt.Errorf("failed to marshal fixed up dump: %w", err)
+	}
+	expRaw, err := json.Marshal(exportedDoc)
+	if err != nil {
+		return fmt.Errorf("failed to re-marshal export doc: %w", err)
+	}
+	if !bytes.Equal(expRaw, dbRaw) {
+		return fmt.Errorf("dump does not match state export")
+	}
+
+	return nil
+}
+
 func (sc *runtimeImpl) finishWithoutChild() error {
 	var err error
 	select {