From 139a29e7e204d6848c5d4a5ed2472597aa939661 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Wed, 25 Oct 2023 12:09:03 +0200 Subject: [PATCH] refactor(network): call `app.Close()` on network cleanup (#18249) --- simapp/internal/testnet/cometrpc_test.go | 178 ------ simapp/internal/testnet/cometstarter_test.go | 141 ----- simapp/internal/testnet/doc.go | 10 - simapp/internal/testnet/example_basic_test.go | 119 ---- testutil/network/network.go | 10 +- testutil/network/util.go | 8 +- testutil/testnet/cometrpclock.go | 100 ---- testutil/testnet/cometstarter.go | 248 -------- testutil/testnet/delegator.go | 59 -- testutil/testnet/doc.go | 4 - testutil/testnet/genesis.go | 528 ------------------ testutil/testnet/genesis_test.go | 245 -------- testutil/testnet/network.go | 116 ---- testutil/testnet/nodehelpers.go | 43 -- testutil/testnet/nodes.go | 70 --- testutil/testnet/nodes_test.go | 30 - testutil/testnet/validator.go | 189 ------- 17 files changed, 14 insertions(+), 2084 deletions(-) delete mode 100644 simapp/internal/testnet/cometrpc_test.go delete mode 100644 simapp/internal/testnet/cometstarter_test.go delete mode 100644 simapp/internal/testnet/doc.go delete mode 100644 simapp/internal/testnet/example_basic_test.go delete mode 100644 testutil/testnet/cometrpclock.go delete mode 100644 testutil/testnet/cometstarter.go delete mode 100644 testutil/testnet/delegator.go delete mode 100644 testutil/testnet/doc.go delete mode 100644 testutil/testnet/genesis.go delete mode 100644 testutil/testnet/genesis_test.go delete mode 100644 testutil/testnet/network.go delete mode 100644 testutil/testnet/nodehelpers.go delete mode 100644 testutil/testnet/nodes.go delete mode 100644 testutil/testnet/nodes_test.go delete mode 100644 testutil/testnet/validator.go diff --git a/simapp/internal/testnet/cometrpc_test.go b/simapp/internal/testnet/cometrpc_test.go deleted file mode 100644 index 6e7d1a17ed35..000000000000 --- a/simapp/internal/testnet/cometrpc_test.go +++ /dev/null @@ -1,178 +0,0 @@ -package testnet_test - -import ( - "context" - "testing" - "time" - - cmtcfg "github.com/cometbft/cometbft/config" - "github.com/cometbft/cometbft/rpc/client/http" - dbm "github.com/cosmos/cosmos-db" - "github.com/stretchr/testify/require" - - "cosmossdk.io/log" - "cosmossdk.io/simapp" - - "github.com/cosmos/cosmos-sdk/baseapp" - "github.com/cosmos/cosmos-sdk/codec" - codectypes "github.com/cosmos/cosmos-sdk/codec/types" - simtestutil "github.com/cosmos/cosmos-sdk/testutil/sims" - "github.com/cosmos/cosmos-sdk/testutil/testnet" - sdk "github.com/cosmos/cosmos-sdk/types" - banktypes "github.com/cosmos/cosmos-sdk/x/bank/types" -) - -const memdb = "memdb" - -// A single comet server in a network runs an RPC server successfully. -func TestCometRPC_SingleRPCServer(t *testing.T) { - const nVals = 2 - - valPKs := testnet.NewValidatorPrivKeys(nVals) - cmtVals := valPKs.CometGenesisValidators() - stakingVals := cmtVals.StakingValidators() - - const chainID = "comet-rpc-singleton" - - b := testnet.DefaultGenesisBuilderOnlyValidators( - chainID, - stakingVals, - sdk.NewCoin(sdk.DefaultBondDenom, sdk.DefaultPowerReduction), - ) - - jGenesis := b.Encode() - - // Logs shouldn't be necessary here because we are exercising CometStarter, - // and only doing a very basic check that the RPC talks to the app. - logger := log.NewNopLogger() - - nodes, err := testnet.NewNetwork(nVals, func(idx int) *testnet.CometStarter { - rootDir := t.TempDir() - - app := simapp.NewSimApp( - logger, - dbm.NewMemDB(), - nil, - true, - simtestutil.NewAppOptionsWithFlagHome(rootDir), - baseapp.SetChainID(chainID), - ) - - cfg := cmtcfg.DefaultConfig() - cfg.BaseConfig.DBBackend = memdb - - cs := testnet.NewCometStarter( - app, - cfg, - valPKs[idx].Val, - jGenesis, - rootDir, - ) - - // Only enable the RPC on the first service. - if idx == 0 { - cs = cs.RPCListen() - } - - return cs - }) - defer func() { - err := nodes.StopAndWait() - if err != nil { - panic(err) - } - }() - require.NoError(t, err) - - // Once HTTP client to be shared across the following subtests. - c, err := http.New(nodes[0].Config().RPC.ListenAddress, "/websocket") - require.NoError(t, err) - - t.Run("status query", func(t *testing.T) { - ctx := context.Background() - st, err := c.Status(ctx) - require.NoError(t, err) - - // Simple assertion to ensure we have a functioning RPC. - require.Equal(t, chainID, st.NodeInfo.Network) - }) - - // Block until reported height is at least 1, - // otherwise we can't make transactions. - require.NoError(t, testnet.WaitForNodeHeight(nodes[0], 1, 10*time.Second)) - - t.Run("simple abci query", func(t *testing.T) { - res, err := c.ABCIQuery( - context.Background(), - "/cosmos.bank.v1beta1.Query/TotalSupply", - nil, - ) - require.NoError(t, err) - - registry := codectypes.NewInterfaceRegistry() - cdc := codec.NewProtoCodec(registry) - - var tsResp banktypes.QueryTotalSupplyResponse - require.NoError(t, cdc.Unmarshal(res.Response.Value, &tsResp)) - - // Just check that something is reported in the supply. - require.NotEmpty(t, tsResp.Supply) - }) -} - -// Starting two comet instances with an RPC server, -// fails with a predictable error. -func TestCometRPC_MultipleRPCError(t *testing.T) { - const nVals = 2 - - valPKs := testnet.NewValidatorPrivKeys(nVals) - cmtVals := valPKs.CometGenesisValidators() - stakingVals := cmtVals.StakingValidators() - - const chainID = "comet-rpc-multiple" - - b := testnet.DefaultGenesisBuilderOnlyValidators( - chainID, - stakingVals, - sdk.NewCoin(sdk.DefaultBondDenom, sdk.DefaultPowerReduction), - ) - - jGenesis := b.Encode() - - // Logs shouldn't be necessary here because we are exercising CometStarter. - logger := log.NewNopLogger() - - nodes, err := testnet.NewNetwork(nVals, func(idx int) *testnet.CometStarter { - rootDir := t.TempDir() - - app := simapp.NewSimApp( - logger, - dbm.NewMemDB(), - nil, - true, - simtestutil.NewAppOptionsWithFlagHome(rootDir), - baseapp.SetChainID(chainID), - ) - - cfg := cmtcfg.DefaultConfig() - cfg.BaseConfig.DBBackend = memdb - - return testnet.NewCometStarter( - app, - cfg, - valPKs[idx].Val, - jGenesis, - rootDir, - ).RPCListen() // Every node has RPCListen enabled, which will cause a failure. - }) - defer func() { - err := nodes.StopAndWait() - if err != nil { - panic(err) - } - }() - // Returned error is convertible to CometRPCInUseError. - // We can't test the exact value because it includes a stack trace. - require.Error(t, err) - require.ErrorAs(t, err, new(testnet.CometRPCInUseError)) -} diff --git a/simapp/internal/testnet/cometstarter_test.go b/simapp/internal/testnet/cometstarter_test.go deleted file mode 100644 index 21e1695b3571..000000000000 --- a/simapp/internal/testnet/cometstarter_test.go +++ /dev/null @@ -1,141 +0,0 @@ -package testnet_test - -import ( - "fmt" - "math/rand" - "net" - "testing" - "time" - - cmtcfg "github.com/cometbft/cometbft/config" - dbm "github.com/cosmos/cosmos-db" - "github.com/stretchr/testify/require" - - "cosmossdk.io/log" - "cosmossdk.io/simapp" - - "github.com/cosmos/cosmos-sdk/baseapp" - simtestutil "github.com/cosmos/cosmos-sdk/testutil/sims" - "github.com/cosmos/cosmos-sdk/testutil/testnet" - sdk "github.com/cosmos/cosmos-sdk/types" -) - -// Use a limited set of available ports to ensure that -// retries eventually land on a free port. -func TestCometStarter_PortContention(t *testing.T) { - if testing.Short() { - t.Skip("skipping long test in short mode") - } - - const nVals = 4 - - // Find n+1 addresses that should be free. - // Ephemeral port range should start at about 49k+ - // according to `sysctl net.inet.ip.portrange` on macOS, - // and at about 32k+ on Linux - // according to `sysctl net.ipv4.ip_local_port_range`. - // - // Because we attempt to find free addresses outside that range, - // it is unlikely that another process will claim a port - // we discover to be free, during the time this test runs. - const portSeekStart = 19000 - reuseAddrs := make([]string, 0, nVals+1) - for i := portSeekStart; i < portSeekStart+1000; i++ { - addr := fmt.Sprintf("127.0.0.1:%d", i) - ln, err := net.Listen("tcp", addr) - if err != nil { - // No need to log the failure. - continue - } - - // If the port was free, append it to our reusable addresses. - reuseAddrs = append(reuseAddrs, "tcp://"+addr) - _ = ln.Close() - - if len(reuseAddrs) == nVals+1 { - break - } - } - - if len(reuseAddrs) != nVals+1 { - t.Fatalf("needed %d free ports but only found %d", nVals+1, len(reuseAddrs)) - } - - // Now that we have one more port than the number of validators, - // there is a good chance that picking a random port will conflict with a previously chosen one. - // But since CometStarter retries several times, - // it should eventually land on a free port. - - valPKs := testnet.NewValidatorPrivKeys(nVals) - cmtVals := valPKs.CometGenesisValidators() - stakingVals := cmtVals.StakingValidators() - - const chainID = "simapp-cometstarter" - - b := testnet.DefaultGenesisBuilderOnlyValidators( - chainID, - stakingVals, - sdk.NewCoin(sdk.DefaultBondDenom, sdk.DefaultPowerReduction), - ) - - jGenesis := b.Encode() - - // Use an info-level logger, because the debug logs in comet are noisy - // and there is a data race in comet debug logs, - // due to be fixed in v0.37.1 which is not yet released: - // https://github.com/cometbft/cometbft/pull/532 - logger := log.NewTestLoggerInfo(t) - - const nRuns = 4 - for i := 0; i < nRuns; i++ { - t.Run(fmt.Sprintf("attempt %d", i), func(t *testing.T) { - nodes, err := testnet.NewNetwork(nVals, func(idx int) *testnet.CometStarter { - rootDir := t.TempDir() - - app := simapp.NewSimApp( - logger.With("instance", idx), - dbm.NewMemDB(), - nil, - true, - simtestutil.NewAppOptionsWithFlagHome(rootDir), - baseapp.SetChainID(chainID), - ) - - cfg := cmtcfg.DefaultConfig() - - // memdb is sufficient for this test. - cfg.BaseConfig.DBBackend = "memdb" - - return testnet.NewCometStarter( - app, - cfg, - valPKs[idx].Val, - jGenesis, - rootDir, - ). - Logger(logger.With("rootmodule", fmt.Sprintf("comet_node-%d", idx))). - TCPAddrChooser(func() string { - // This chooser function is the key of this test, - // where there is only one more available address than there are nodes. - // Therefore it is likely that an address will already be in use, - // thereby exercising the address-in-use retry. - return reuseAddrs[rand.Intn(len(reuseAddrs))] - }) - }) - - // Ensure nodes are stopped completely, - // so that we don't get t.Cleanup errors around directories not being empty. - defer func() { - err := nodes.StopAndWait() - if err != nil { - panic(err) - } - }() - require.NoError(t, err) - - // Ensure that the height advances. - // Looking for height 2 seems more meaningful than 1. - require.NoError(t, testnet.WaitForNodeHeight(nodes[0], 2, 10*time.Second)) - }) - } -} diff --git a/simapp/internal/testnet/doc.go b/simapp/internal/testnet/doc.go deleted file mode 100644 index 19064ddd0f45..000000000000 --- a/simapp/internal/testnet/doc.go +++ /dev/null @@ -1,10 +0,0 @@ -// Package testnet contains tests for -// [github.com/cosmos/cosmos-sdk/testutil/testnet]. -// -// Eventually all of these tests will move into that package, -// but that is currently blocked on having a minimal app defined -// in the root cosmos-sdk Go module. -// Once that app is available, the contents of this package -// will be moved to testutil/testnet, -// and references to SimApp will be replaced by the minimal app. -package testnet diff --git a/simapp/internal/testnet/example_basic_test.go b/simapp/internal/testnet/example_basic_test.go deleted file mode 100644 index f1a094c0815b..000000000000 --- a/simapp/internal/testnet/example_basic_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package testnet_test - -import ( - "fmt" - "os" - "path/filepath" - - cmtcfg "github.com/cometbft/cometbft/config" - dbm "github.com/cosmos/cosmos-db" - - "cosmossdk.io/log" - "cosmossdk.io/simapp" - - "github.com/cosmos/cosmos-sdk/baseapp" - simtestutil "github.com/cosmos/cosmos-sdk/testutil/sims" - "github.com/cosmos/cosmos-sdk/testutil/testnet" - sdk "github.com/cosmos/cosmos-sdk/types" -) - -func Example_basicUsage() { - const nVals = 2 - - // Set up new private keys for the set of validators. - valPKs := testnet.NewValidatorPrivKeys(nVals) - - // Comet-style validators. - cmtVals := valPKs.CometGenesisValidators() - - // Cosmos SDK staking validators for genesis. - stakingVals := cmtVals.StakingValidators() - - const chainID = "example-basic" - - // Create a genesis builder that only requires validators, - // without any separate delegator accounts. - // - // If you need further customization, start with testnet.NewGenesisBuilder(). - b := testnet.DefaultGenesisBuilderOnlyValidators( - chainID, - stakingVals, - // The amount to use in each validator's account during gentx. - sdk.NewCoin(sdk.DefaultBondDenom, sdk.DefaultPowerReduction), - ) - - // JSON-formatted genesis. - jGenesis := b.Encode() - - // In this example, we have an outer root directory for the validators. - // Use t.TempDir() in tests. - rootDir, err := os.MkdirTemp("", "testnet-example-") - if err != nil { - panic(err) - } - defer os.RemoveAll(rootDir) - - // In tests, you probably want to use log.NewTestLoggerInfo(t). - logger := log.NewNopLogger() - - // The NewNetwork function creates a network of validators. - // We have to provide a callback to return CometStarter instances. - // NewNetwork will start all the comet instances concurrently - // and join the nodes together. - nodes, err := testnet.NewNetwork(nVals, func(idx int) *testnet.CometStarter { - // Make a new directory for the validator being created. - // In tests, this would be a simpler call to t.TempDir(). - dir := filepath.Join(rootDir, fmt.Sprintf("val-%d", idx)) - if err := os.Mkdir(dir, 0o755); err != nil { - panic(err) - } - - // TODO: use a different minimal app for this. - app := simapp.NewSimApp( - logger.With("instance", idx), - dbm.NewMemDB(), - nil, - true, - simtestutil.NewAppOptionsWithFlagHome(rootDir), - baseapp.SetChainID(chainID), - ) - - // Each CometStarter instance must be associated with - // a distinct comet Config object, - // as the CometStarter will automatically modify some fields, - // including P2P.ListenAddress. - cfg := cmtcfg.DefaultConfig() - - // No need to persist comet's DB to disk in this example. - cfg.BaseConfig.DBBackend = "memdb" - - return testnet.NewCometStarter( - app, - cfg, - valPKs[idx].Val, // Validator private key for this comet instance. - jGenesis, // Raw bytes of genesis file. - dir, // Where to put files on disk. - ).Logger(logger.With("root_module", fmt.Sprintf("comet_%d", idx))) - }) - if err != nil { - panic(err) - } - - // StopAndWait must be deferred before the error check,"os" - // as the nodes value may contain some successfully started instances. - defer func() { - err := nodes.StopAndWait() - if err != nil { - panic(err) - } - }() - // Now you can begin interacting with the nodes. - // For the sake of this example, we'll just check - // a couple simple properties of one node. - fmt.Println(nodes[0].IsListening()) - fmt.Println(nodes[0].GenesisDoc().ChainID) - - // Output: - // true - // example-basic -} diff --git a/testutil/network/network.go b/testutil/network/network.go index 58a35e1e9f5d..5003c83e259c 100644 --- a/testutil/network/network.go +++ b/testutil/network/network.go @@ -292,6 +292,7 @@ type ( ValAddress sdk.ValAddress RPCClient cmtclient.Client + app servertypes.Application tmNode *node.Node api *api.Server grpc *grpc.Server @@ -636,8 +637,7 @@ func New(l Logger, baseDir string, cfg Config) (*Network, error) { l.Log("starting test network...") for idx, v := range network.Validators { - err := startInProcess(cfg, v) - if err != nil { + if err := startInProcess(cfg, v); err != nil { return nil, err } l.Log("started validator", idx) @@ -828,6 +828,12 @@ func (n *Network) Cleanup() { if v.grpcWeb != nil { _ = v.grpcWeb.Close() } + + if v.app != nil { + if err := v.app.Close(); err != nil { + n.Logger.Log("failed to stop validator ABCI application", "err", err) + } + } } time.Sleep(100 * time.Millisecond) diff --git a/testutil/network/util.go b/testutil/network/util.go index ecc5c54f73f5..d94ee27af2c0 100644 --- a/testutil/network/util.go +++ b/testutil/network/util.go @@ -18,6 +18,8 @@ import ( cmttime "github.com/cometbft/cometbft/types/time" "golang.org/x/sync/errgroup" + "cosmossdk.io/log" + "github.com/cosmos/cosmos-sdk/server" "github.com/cosmos/cosmos-sdk/server/api" servergrpc "github.com/cosmos/cosmos-sdk/server/grpc" @@ -43,6 +45,8 @@ func startInProcess(cfg Config, val *Validator) error { } app := cfg.AppConstructor(*val) + val.app = app + appGenesisProvider := func() (*cmttypes.GenesisDoc, error) { appGenesis, err := genutiltypes.AppGenesisFromFile(cmtCfg.GenesisFile()) if err != nil { @@ -101,14 +105,14 @@ func startInProcess(cfg Config, val *Validator) error { // Start the gRPC server in a goroutine. Note, the provided ctx will ensure // that the server is gracefully shut down. val.errGroup.Go(func() error { - return servergrpc.StartGRPCServer(ctx, logger.With("module", "grpc-server"), grpcCfg, grpcSrv) + return servergrpc.StartGRPCServer(ctx, logger.With(log.ModuleKey, "grpc-server"), grpcCfg, grpcSrv) }) val.grpc = grpcSrv } if val.APIAddress != "" { - apiSrv := api.New(val.ClientCtx, logger.With("module", "api-server"), val.grpc) + apiSrv := api.New(val.ClientCtx, logger.With(log.ModuleKey, "api-server"), val.grpc) app.RegisterAPIRoutes(apiSrv, val.AppConfig.API) val.errGroup.Go(func() error { diff --git a/testutil/testnet/cometrpclock.go b/testutil/testnet/cometrpclock.go deleted file mode 100644 index 17c05440d3e4..000000000000 --- a/testutil/testnet/cometrpclock.go +++ /dev/null @@ -1,100 +0,0 @@ -package testnet - -import ( - "fmt" - "runtime/debug" - "sync" - - "github.com/cometbft/cometbft/node" -) - -// CometBFT v0.37 uses a singleton to manage the RPC "environment". -// v0.38 will not have that restriction, which was removed in commit: -// https://github.com/cometbft/cometbft/commit/3324f49fb7e7b40189726746493e83b82a61b558 -// -// We manage a corresponding global lock to ensure -// we don't attempt to use multiple active RPC servers in one process, -// which would result in unpredictable or incorrect behavior. -// Once the SDK adopts Comet v0.38+, we can remove this global lock mechanism. - -// Our singleton complementing Comet's global RPC state. -var globalCometMu = new(cometRPCMutex) - -type cometRPCMutex struct { - mu sync.Mutex - - prevLockStack []byte -} - -// CometRPCInUseError is returned on a failure to acquire -// the global comet RPC lock. -// -// This type will be removed once the Cosmos SDK adopts CometBFT v0.38 or newer. -type CometRPCInUseError struct { - prevStack []byte -} - -func (e CometRPCInUseError) Error() string { - return fmt.Sprintf(`Failed to acquire global lock for Comet RPC servers. - -If this in a test using t.Parallel(), remove the call to t.Parallel(). - -If this is in a test NOT using t.Parallel, -ensure that other callers call both Stop() and Wait() on the nodes. - -If there are multiple comet instances in one test using RPC servers, -ensure that only one instance has the RPC listener enabled. - -These restrictions will be loosened once cosmos-sdk adopts comet-bft v0.38 or newer. - -Stack where lock was previously acquired: -%s -`, e.prevStack) -} - -// Acquire attempts to acquire the underlying mutex. -// If it cannot be acquired on the first attempt, -// Acquire returns a [CometRPCInUseError] value. -func (m *cometRPCMutex) Acquire() error { - if !m.mu.TryLock() { - // If we can't acquire the lock, - // there is another active comet node using RPC. - // - // This was initially going to be a panic, - // but we can't easily write tests against that since - // the panic occurs in a separate goroutine - // when called through NewNetwork. - // - // Note, reading m.prevLockStack without holding m.mu - // is technically a data race, - // since it is possible that the previous caller was about to unlock. - // Nonetheless, the programmer is responsible for avoiding that situation, - // and a data race during a failure isn't particularly concerning. - return CometRPCInUseError{prevStack: m.prevLockStack} - } - - // Now we hold the lock, so first record the stack when the lock was taken. - m.prevLockStack = debug.Stack() - - return nil -} - -// Release unlocks m depending on n. -// If n is nil, m is unlocked immediately. -// If n is not nil, a new goroutine is created -// and n is released after the node has finished running. -func (m *cometRPCMutex) Release(n *node.Node) { - if n == nil { - m.prevLockStack = nil - m.mu.Unlock() - return - } - - go m.releaseAfterWait(n) -} - -func (m *cometRPCMutex) releaseAfterWait(n *node.Node) { - n.Wait() - m.prevLockStack = nil - m.mu.Unlock() -} diff --git a/testutil/testnet/cometstarter.go b/testutil/testnet/cometstarter.go deleted file mode 100644 index 088451e8c45d..000000000000 --- a/testutil/testnet/cometstarter.go +++ /dev/null @@ -1,248 +0,0 @@ -package testnet - -import ( - "errors" - "fmt" - "net" - "os" - "path/filepath" - "syscall" - - cmtcfg "github.com/cometbft/cometbft/config" - cmted25519 "github.com/cometbft/cometbft/crypto/ed25519" - "github.com/cometbft/cometbft/node" - "github.com/cometbft/cometbft/p2p" - "github.com/cometbft/cometbft/privval" - "github.com/cometbft/cometbft/proxy" - cmttypes "github.com/cometbft/cometbft/types" - - "cosmossdk.io/log" - - "github.com/cosmos/cosmos-sdk/server" - servercmtlog "github.com/cosmos/cosmos-sdk/server/log" - servertypes "github.com/cosmos/cosmos-sdk/server/types" - genutiltypes "github.com/cosmos/cosmos-sdk/x/genutil/types" -) - -// CometStarter offers a builder-pattern interface to -// starting a Comet instance with an ABCI application running alongside. -// -// As CometStart is more broadly used in the codebase, -// the number of available methods on CometStarter will grow. -type CometStarter struct { - logger log.Logger - app servertypes.ABCI - cfg *cmtcfg.Config - valPrivKey cmted25519.PrivKey - genesis []byte - rootDir string - rpcListen bool - tcpAddrChooser func() string - startTries int -} - -// NewCometStarter accepts a minimal set of arguments to start comet with an ABCI app. -// For further configuration, chain other CometStarter methods before calling Start: -// -// NewCometStarter(...).Logger(...).Start() -func NewCometStarter( - app servertypes.ABCI, - cfg *cmtcfg.Config, - valPrivKey cmted25519.PrivKey, - genesis []byte, - rootDir string, -) *CometStarter { - cfg.SetRoot(rootDir) - - // CometStarter won't work without these settings, - // so set them unconditionally. - cfg.P2P.AllowDuplicateIP = true - cfg.P2P.AddrBookStrict = false - - // For now, we disallow RPC listening. - // Comet v0.37 uses a global value such that multiple comet nodes in one process - // end up contending over one "rpc environment" and only the last-started validator - // will control the RPC service. - // - // The "rpc environment" was removed as a global in - // https://github.com/cometbft/cometbft/commit/3324f49fb7e7b40189726746493e83b82a61b558 - // which is due to land in v0.38. - // - // At that point, we should keep the default as RPC off, - // but we should add a RPCListen method to opt in to enabling it. - - // If RPC.ListenAddress is the default value, clear it. - const defaultRPCListenAddr = "tcp://127.0.0.1:26657" - if cfg.RPC.ListenAddress == defaultRPCListenAddr { - cfg.RPC.ListenAddress = "" - } - - // Then if it was set to anything other than empty or the default value, - // fail with a clear explanation on how to enable RPC. - // The RPCListen method must be used in order to correctly pick an available listen address. - if cfg.RPC.ListenAddress != "" { - panic(fmt.Errorf("NewCometStarter: cfg.RPC.ListenAddress must be empty (but was %q); use (*CometStarter).RPCListen() instead", cfg.RPC.ListenAddress)) - } - - // defaultStartTries is somewhat arbitrary. - // Occasionally TestCometStarter_PortContention would fail with 10 tries, - // and bumping it up to 12 makes it almost never fail. - const defaultStartTries = 12 - return &CometStarter{ - logger: log.NewNopLogger(), - app: app, - cfg: cfg, - genesis: genesis, - valPrivKey: valPrivKey, - rootDir: rootDir, - startTries: defaultStartTries, - } -} - -// Logger sets the logger for s and for the eventual started comet instance. -func (s *CometStarter) Logger(logger log.Logger) *CometStarter { - s.logger = logger - return s -} - -// RPCListen enables the RPC listener service on the underlying Comet node. -// The RPC service must be enabled this way so that s can choose a dynamic port, -// retrying if necessary. -// -// Note that there is a limitation in CometBFT v0.37 that -// prevents more than one RPC server running at a time. -// Once the Cosmos SDK has adopted CometBFT v0.38 or newer, -// that limitation will be removed. -func (s *CometStarter) RPCListen() *CometStarter { - s.rpcListen = true - return s -} - -// Start returns a started Comet node. -func (s *CometStarter) Start() (n *node.Node, err error) { - if s.rpcListen { - if err := globalCometMu.Acquire(); err != nil { - return nil, err - } - - // Wrap this defer in an anonymous function so we don't immediately evaluate - // n, which would always be nil at this point. - defer func() { - globalCometMu.Release(n) - }() - } - - fpv, nodeKey, err := s.initDisk() - if err != nil { - return nil, err - } - - appGenesisProvider := func() (*cmttypes.GenesisDoc, error) { - appGenesis, err := genutiltypes.AppGenesisFromFile(s.cfg.GenesisFile()) - if err != nil { - return nil, err - } - - return appGenesis.ToGenesisDoc() - } - - cmtApp := server.NewCometABCIWrapper(s.app) - for i := 0; i < s.startTries; i++ { - s.cfg.P2P.ListenAddress = s.likelyAvailableAddress() - if s.rpcListen { - s.cfg.RPC.ListenAddress = s.likelyAvailableAddress() - } - - n, err := node.NewNode( - s.cfg, - fpv, - nodeKey, - proxy.NewLocalClientCreator(cmtApp), - appGenesisProvider, - cmtcfg.DefaultDBProvider, - node.DefaultMetricsProvider(s.cfg.Instrumentation), - servercmtlog.CometLoggerWrapper{Logger: s.logger}, - ) - if err != nil { - return nil, fmt.Errorf("failed to create comet node: %w", err) - } - - err = n.Start() - if err == nil { - return n, nil - } - - // Error isn't nil -- if it is EADDRINUSE then we can try again. - if errors.Is(err, syscall.EADDRINUSE) { - continue - } - - // Non-nil error that isn't EADDRINUSE, just return the error. - return nil, err - } - - // If we didn't return a node from inside the loop, - // then we must have exhausted our try limit. - return nil, fmt.Errorf("failed to start a comet node within %d tries", s.startTries) -} - -// initDisk creates the config and data directories on disk, -// and other required files, so that comet and the validator work correctly. -// It also generates a node key for validators. -func (s *CometStarter) initDisk() (cmttypes.PrivValidator, *p2p.NodeKey, error) { - if err := os.MkdirAll(filepath.Join(s.rootDir, "config"), 0o750); err != nil { - return nil, nil, fmt.Errorf("failed to make config directory: %w", err) - } - if err := os.MkdirAll(filepath.Join(s.rootDir, "data"), 0o750); err != nil { - return nil, nil, fmt.Errorf("failed to make data directory: %w", err) - } - - fpv := privval.NewFilePV(s.valPrivKey, s.cfg.PrivValidatorKeyFile(), s.cfg.PrivValidatorStateFile()) - fpv.Save() - - if err := os.WriteFile(s.cfg.GenesisFile(), s.genesis, 0o600); err != nil { - return nil, nil, fmt.Errorf("failed to write genesis file: %w", err) - } - - nodeKey, err := p2p.LoadOrGenNodeKey(s.cfg.NodeKeyFile()) - if err != nil { - return nil, nil, err - } - - return fpv, nodeKey, nil -} - -// TCPAddrChooser sets the function to use when selecting a (likely to be free) -// TCP address for comet's P2P port. -// -// This should only be used when testing CometStarter. -// -// It must return a string in format "tcp://IP:PORT". -func (s *CometStarter) TCPAddrChooser(fn func() string) *CometStarter { - s.tcpAddrChooser = fn - return s -} - -// likelyAvailableAddress provides a TCP address that is likely to be available -// for comet or other processes to listen on. -// -// Generally, it is better to directly provide a net.Listener that is already bound to an address, -// but unfortunately comet does not offer that as part of its API. -// Instead, we locally bind to :0 and then report that as a "likely available" port. -// If another process steals that port before our comet instance can bind to it, -// the Start method handles retries. -func (s *CometStarter) likelyAvailableAddress() string { - // If s.TCPAddrChooser was called, use that implementation. - if s.tcpAddrChooser != nil { - return s.tcpAddrChooser() - } - - // Fall back to attempting a random port. - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - panic(fmt.Errorf("failed to bind to random port: %w", err)) - } - - defer ln.Close() - return "tcp://" + ln.Addr().String() -} diff --git a/testutil/testnet/delegator.go b/testutil/testnet/delegator.go deleted file mode 100644 index 5ed8ef0ed6d5..000000000000 --- a/testutil/testnet/delegator.go +++ /dev/null @@ -1,59 +0,0 @@ -package testnet - -import ( - "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" - sdk "github.com/cosmos/cosmos-sdk/types" - authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" - banktypes "github.com/cosmos/cosmos-sdk/x/bank/types" -) - -// DelegatorPrivKeys is a slice of secp256k1.PrivKey. -type DelegatorPrivKeys []*secp256k1.PrivKey - -// NewDelegatorPrivKeysreturns a DelegatorPrivKeys of length n, -// where each set of keys is dynamically generated. -func NewDelegatorPrivKeys(n int) DelegatorPrivKeys { - dpk := make(DelegatorPrivKeys, n) - - for i := range dpk { - dpk[i] = secp256k1.GenPrivKey() - } - - return dpk -} - -// BaseAccounts returns the base accounts corresponding to the delegators' public keys. -func (dpk DelegatorPrivKeys) BaseAccounts() BaseAccounts { - ba := make(BaseAccounts, len(dpk)) - - for i, pk := range dpk { - pubKey := pk.PubKey() - - const accountNumber = 0 - const sequenceNumber = 0 - - ba[i] = authtypes.NewBaseAccount( - pubKey.Address().Bytes(), pubKey, accountNumber, sequenceNumber, - ) - } - - return ba -} - -// BaseAccounts is a slice of [*authtypes.BaseAccount]. -type BaseAccounts []*authtypes.BaseAccount - -// Balances creates a slice of [banktypes.Balance] for each account in ba, -// where each balance has an identical Coins value of the singleBalance argument. -func (ba BaseAccounts) Balances(singleBalance sdk.Coins) []banktypes.Balance { - balances := make([]banktypes.Balance, len(ba)) - - for i, b := range ba { - balances[i] = banktypes.Balance{ - Address: b.GetAddress().String(), - Coins: singleBalance, - } - } - - return balances -} diff --git a/testutil/testnet/doc.go b/testutil/testnet/doc.go deleted file mode 100644 index 55d4dbe441d4..000000000000 --- a/testutil/testnet/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -// Package testnet provides APIs for easily create and configure -// validators, genesis files, and comet instances, -// to support testing app chain instances in-process. -package testnet diff --git a/testutil/testnet/genesis.go b/testutil/testnet/genesis.go deleted file mode 100644 index 6e2590f799a6..000000000000 --- a/testutil/testnet/genesis.go +++ /dev/null @@ -1,528 +0,0 @@ -package testnet - -import ( - "context" - "encoding/json" - "fmt" - "strconv" - "time" - - cmttypes "github.com/cometbft/cometbft/types" - - "cosmossdk.io/math" - distributiontypes "cosmossdk.io/x/distribution/types" - - "github.com/cosmos/cosmos-sdk/codec" - "github.com/cosmos/cosmos-sdk/codec/address" - codectypes "github.com/cosmos/cosmos-sdk/codec/types" - cryptocodec "github.com/cosmos/cosmos-sdk/crypto/codec" - "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" - sdk "github.com/cosmos/cosmos-sdk/types" - "github.com/cosmos/cosmos-sdk/types/tx/signing" - authsigning "github.com/cosmos/cosmos-sdk/x/auth/signing" - authtx "github.com/cosmos/cosmos-sdk/x/auth/tx" - authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" - banktypes "github.com/cosmos/cosmos-sdk/x/bank/types" - consensusparamtypes "github.com/cosmos/cosmos-sdk/x/consensus/types" - genutiltypes "github.com/cosmos/cosmos-sdk/x/genutil/types" - minttypes "github.com/cosmos/cosmos-sdk/x/mint/types" - slashingtypes "github.com/cosmos/cosmos-sdk/x/slashing/types" - stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types" -) - -// GenesisBuilder enables constructing a genesis file, -// following a builder pattern. -// -// None of the methods on GenesisBuilder return an error, -// choosing instead to panic. -// GenesisBuilder is only intended for use in tests, -// where inputs are predetermined and expected to succeed. -type GenesisBuilder struct { - amino *codec.LegacyAmino - codec *codec.ProtoCodec - - // The value used in ChainID. - // Some other require this value, - // so store it as a field instead of re-parsing it from JSON. - chainID string - - // The outer JSON object. - // Most data goes into app_state, but there are some top-level fields. - outer map[string]json.RawMessage - - // Many of GenesisBuilder's methods operate on the app_state JSON object, - // so we track that separately and nest it inside outer upon a call to JSON(). - appState map[string]json.RawMessage - - gentxs []sdk.Tx -} - -// NewGenesisBuilder returns an initialized GenesisBuilder. -// -// The returned GenesisBuilder has an initial height of 1 -// and a genesis_time of the current time when the function was called. -func NewGenesisBuilder() *GenesisBuilder { - ir := codectypes.NewInterfaceRegistry() - cryptocodec.RegisterInterfaces(ir) - stakingtypes.RegisterInterfaces(ir) - banktypes.RegisterInterfaces(ir) - authtypes.RegisterInterfaces(ir) - pCodec := codec.NewProtoCodec(ir) - - return &GenesisBuilder{ - amino: codec.NewLegacyAmino(), - codec: pCodec, - - outer: map[string]json.RawMessage{ - "initial_height": json.RawMessage(`"1"`), - "genesis_time": json.RawMessage( - strconv.AppendQuote(nil, time.Now().UTC().Format(time.RFC3339Nano)), - ), - }, - appState: map[string]json.RawMessage{}, - } -} - -// GenTx emulates the gentx CLI, creating a message to create a validator -// represented by val, with "amount" self delegation, -// and signed by privVal. -func (b *GenesisBuilder) GenTx(privVal secp256k1.PrivKey, val cmttypes.GenesisValidator, amount sdk.Coin) *GenesisBuilder { - if b.chainID == "" { - panic(fmt.Errorf("(*GenesisBuilder).GenTx must not be called before (*GenesisBuilder).ChainID")) - } - - pubKey, err := cryptocodec.FromCmtPubKeyInterface(val.PubKey) - if err != nil { - panic(err) - } - - // Produce the create validator message. - msg, err := stakingtypes.NewMsgCreateValidator( - sdk.ValAddress(privVal.PubKey().Address()).String(), - pubKey, - amount, - stakingtypes.Description{ - Moniker: "TODO", - }, - stakingtypes.CommissionRates{ - Rate: math.LegacyMustNewDecFromStr("0.1"), - MaxRate: math.LegacyMustNewDecFromStr("0.2"), - MaxChangeRate: math.LegacyMustNewDecFromStr("0.01"), - }, - math.OneInt(), - ) - if err != nil { - panic(err) - } - - if err := msg.Validate(address.NewBech32Codec("cosmosvaloper")); err != nil { - panic(err) - } - - txConf := authtx.NewTxConfig(b.codec, authtx.DefaultSignModes) - - txb := txConf.NewTxBuilder() - if err := txb.SetMsgs(msg); err != nil { - panic(err) - } - - const signMode = signing.SignMode_SIGN_MODE_DIRECT - - // Need to set the signature object on the tx builder first, - // otherwise we end up signing a different total message - // compared to what gets eventually verified. - if err := txb.SetSignatures( - signing.SignatureV2{ - PubKey: privVal.PubKey(), - Data: &signing.SingleSignatureData{ - SignMode: signMode, - }, - }, - ); err != nil { - panic(err) - } - - // Generate bytes to be signed. - bytesToSign, err := authsigning.GetSignBytesAdapter( - context.Background(), - txConf.SignModeHandler(), - signing.SignMode_SIGN_MODE_DIRECT, - authsigning.SignerData{ - ChainID: b.chainID, - PubKey: privVal.PubKey(), - Address: sdk.MustBech32ifyAddressBytes("cosmos", privVal.PubKey().Address()), // TODO: don't hardcode cosmos1! - - // No account or sequence number for gentx. - }, - txb.GetTx(), - ) - if err != nil { - panic(err) - } - - // Produce the signature. - signed, err := privVal.Sign(bytesToSign) - if err != nil { - panic(err) - } - - // Set the signature on the builder. - if err := txb.SetSignatures( - signing.SignatureV2{ - PubKey: privVal.PubKey(), - Data: &signing.SingleSignatureData{ - SignMode: signMode, - Signature: signed, - }, - }, - ); err != nil { - panic(err) - } - - b.gentxs = append(b.gentxs, txb.GetTx()) - - return b -} - -// ChainID sets the genesis's "chain_id" field. -func (b *GenesisBuilder) ChainID(id string) *GenesisBuilder { - b.chainID = id - - var err error - b.outer["chain_id"], err = json.Marshal(id) - if err != nil { - panic(err) - } - - return b -} - -// GenesisTime sets the genesis's "genesis_time" field. -// Note that [NewGenesisBuilder] sets the genesis time to the current time by default. -func (b *GenesisBuilder) GenesisTime(t time.Time) *GenesisBuilder { - var err error - b.outer["genesis_time"], err = json.Marshal(t.Format(time.RFC3339Nano)) - if err != nil { - panic(err) - } - return b -} - -// InitialHeight sets the genesis's "initial_height" field to h. -// Note that [NewGenesisBuilder] sets the initial height to 1 by default. -func (b *GenesisBuilder) InitialHeight(h int64) *GenesisBuilder { - var err error - b.outer["initial_height"], err = json.Marshal(strconv.FormatInt(h, 10)) - if err != nil { - panic(err) - } - return b -} - -// AuthParams sets the auth params on the genesis. -func (b *GenesisBuilder) AuthParams(params authtypes.Params) *GenesisBuilder { - var err error - b.appState[authtypes.ModuleName], err = json.Marshal(map[string]any{ - "params": params, - }) - if err != nil { - panic(err) - } - - return b -} - -// DefaultAuthParams calls b.AuthParams with [authtypes.DefaultParams], -// as a convenience so that callers do not have to import the authtypes package. -func (b *GenesisBuilder) DefaultAuthParams() *GenesisBuilder { - return b.AuthParams(authtypes.DefaultParams()) -} - -// Consensus sets the consensus parameters and initial validators. -// -// If params is nil, [cmttypes.DefaultConsensusParams] is used. -func (b *GenesisBuilder) Consensus(params *cmttypes.ConsensusParams, vals CometGenesisValidators) *GenesisBuilder { - if params == nil { - params = cmttypes.DefaultConsensusParams() - } - - var err error - b.outer[consensusparamtypes.ModuleName], err = (&genutiltypes.ConsensusGenesis{ - Params: params, - Validators: vals.ToComet(), - }).MarshalJSON() - if err != nil { - panic(err) - } - - return b -} - -// Staking sets the staking parameters, validators, and delegations on the genesis. -// -// This also modifies the bank state's balances to include the bonded pool balance. -func (b *GenesisBuilder) Staking( - params stakingtypes.Params, - vals StakingValidators, - delegations []stakingtypes.Delegation, -) *GenesisBuilder { - var err error - b.appState[stakingtypes.ModuleName], err = b.codec.MarshalJSON( - stakingtypes.NewGenesisState(params, vals.ToStakingType(), delegations), - ) - if err != nil { - panic(err) - } - - // Modify bank state for bonded pool. - - var coins sdk.Coins - for _, v := range vals { - coins = coins.Add(sdk.NewCoin(sdk.DefaultBondDenom, v.V.Tokens)) - } - - bondedPoolBalance := banktypes.Balance{ - Address: authtypes.NewModuleAddress(stakingtypes.BondedPoolName).String(), - Coins: coins, - } - - // get bank types genesis, add account - - bankGenesis := banktypes.GetGenesisStateFromAppState(b.codec, b.appState) - bankGenesis.Balances = append(bankGenesis.Balances, bondedPoolBalance) - - b.appState[banktypes.ModuleName], err = b.codec.MarshalJSON(bankGenesis) - if err != nil { - panic(err) - } - - return b -} - -// StakingWithDefaultParams calls b.Staking, providing [stakingtypes.DefaultParams] -// so that callers don't necessarily have to import [stakingtypes]. -func (b *GenesisBuilder) StakingWithDefaultParams(vals StakingValidators, delegations []stakingtypes.Delegation) *GenesisBuilder { - return b.Staking(stakingtypes.DefaultParams(), vals, delegations) -} - -// DefaultStaking is shorthand for b.StakingWithDefaultParams with nil validators and delegations. -func (b *GenesisBuilder) DefaultStaking() *GenesisBuilder { - return b.StakingWithDefaultParams(nil, nil) -} - -// Banking sets the banking genesis state. -func (b *GenesisBuilder) Banking( - params banktypes.Params, - balances []banktypes.Balance, - totalSupply sdk.Coins, - denomMetadata []banktypes.Metadata, - sendEnabled []banktypes.SendEnabled, -) *GenesisBuilder { - var err error - b.appState[banktypes.ModuleName], err = b.codec.MarshalJSON( - banktypes.NewGenesisState( - params, - balances, - totalSupply, - denomMetadata, - sendEnabled, - ), - ) - if err != nil { - panic(err) - } - return b -} - -// BankingWithDefaultParams calls b.Banking with [banktypes.DefaultParams], -// so that callers don't necessarily have to import [banktypes]. -func (b *GenesisBuilder) BankingWithDefaultParams( - balances []banktypes.Balance, - totalSupply sdk.Coins, - denomMetadata []banktypes.Metadata, - sendEnabled []banktypes.SendEnabled, -) *GenesisBuilder { - return b.Banking( - banktypes.DefaultParams(), - balances, - totalSupply, - denomMetadata, - sendEnabled, - ) -} - -// Mint sets the mint genesis state. -func (b *GenesisBuilder) Mint(m minttypes.Minter, p minttypes.Params) *GenesisBuilder { - var err error - b.appState[minttypes.ModuleName], err = b.codec.MarshalJSON( - minttypes.NewGenesisState(m, p), - ) - if err != nil { - panic(err) - } - return b -} - -// DefaultMint calls b.Mint with [minttypes.DefaultInitialMinter] and [minttypes.DefaultParams]. -func (b *GenesisBuilder) DefaultMint() *GenesisBuilder { - return b.Mint(minttypes.DefaultInitialMinter(), minttypes.DefaultParams()) -} - -// Slashing sets the slashing genesis state. -func (b *GenesisBuilder) Slashing( - params slashingtypes.Params, - si []slashingtypes.SigningInfo, - mb []slashingtypes.ValidatorMissedBlocks, -) *GenesisBuilder { - var err error - b.appState[slashingtypes.ModuleName], err = b.codec.MarshalJSON( - slashingtypes.NewGenesisState(params, si, mb), - ) - if err != nil { - panic(err) - } - return b -} - -// SlashingWithDefaultParams calls b.Slashing with [slashingtypes.DefaultParams], -// so that callers don't necessarily have to import [slashingtypes]. -func (b *GenesisBuilder) SlashingWithDefaultParams( - si []slashingtypes.SigningInfo, - mb []slashingtypes.ValidatorMissedBlocks, -) *GenesisBuilder { - return b.Slashing(slashingtypes.DefaultParams(), si, mb) -} - -// DefaultSlashing is shorthand for b.SlashingWithDefaultParams -// with nil signing info and validator missed blocks. -func (b *GenesisBuilder) DefaultSlashing() *GenesisBuilder { - return b.SlashingWithDefaultParams(nil, nil) -} - -// BaseAccounts sets the initial base accounts and balances. -func (b *GenesisBuilder) BaseAccounts(ba BaseAccounts, balances []banktypes.Balance) *GenesisBuilder { - // Logic mostly copied from AddGenesisAccount. - - authGenState := authtypes.GetGenesisStateFromAppState(b.codec, b.appState) - bankGenState := banktypes.GetGenesisStateFromAppState(b.codec, b.appState) - - accs, err := authtypes.UnpackAccounts(authGenState.Accounts) - if err != nil { - panic(err) - } - - for _, a := range ba { - accs = append(accs, a) - } - accs = authtypes.SanitizeGenesisAccounts(accs) - - genAccs, err := authtypes.PackAccounts(accs) - if err != nil { - panic(err) - } - - authGenState.Accounts = genAccs - jAuthGenState, err := b.codec.MarshalJSON(&authGenState) - if err != nil { - panic(err) - } - b.appState[authtypes.ModuleName] = jAuthGenState - - bankGenState.Balances = append(bankGenState.Balances, balances...) - bankGenState.Balances = banktypes.SanitizeGenesisBalances(bankGenState.Balances) - - jBankState, err := b.codec.MarshalJSON(bankGenState) - if err != nil { - panic(err) - } - b.appState[banktypes.ModuleName] = jBankState - return b -} - -func (b *GenesisBuilder) Distribution(g *distributiontypes.GenesisState) *GenesisBuilder { - j, err := b.codec.MarshalJSON(g) - if err != nil { - panic(err) - } - - b.appState[distributiontypes.ModuleName] = j - return b -} - -func (b *GenesisBuilder) DefaultDistribution() *GenesisBuilder { - return b.Distribution(distributiontypes.DefaultGenesisState()) -} - -// JSON returns the map of the genesis after applying some final transformations. -func (b *GenesisBuilder) JSON() map[string]json.RawMessage { - gentxGenesisState := genutiltypes.NewGenesisStateFromTx( - authtx.NewTxConfig(b.codec, authtx.DefaultSignModes).TxJSONEncoder(), - b.gentxs, - ) - - if err := genutiltypes.ValidateGenesis( - gentxGenesisState, - authtx.NewTxConfig(b.codec, authtx.DefaultSignModes).TxJSONDecoder(), - genutiltypes.DefaultMessageValidator, - ); err != nil { - panic(err) - } - - b.appState = genutiltypes.SetGenesisStateInAppState( - b.codec, b.appState, gentxGenesisState, - ) - - appState, err := b.amino.MarshalJSON(b.appState) - if err != nil { - panic(err) - } - - b.outer["app_state"] = appState - - return b.outer -} - -// Encode returns the JSON-encoded, finalized genesis. -func (b *GenesisBuilder) Encode() []byte { - j, err := b.amino.MarshalJSON(b.JSON()) - if err != nil { - panic(err) - } - - return j -} - -// DefaultGenesisBuilderOnlyValidators returns a GenesisBuilder configured only with the given StakingValidators, -// with default parameters everywhere else. -// validatorAmount is the amount to give each validator during gentx. -// -// This is a convenience for the common case of nothing special in the genesis. -// For anything outside of the defaults, -// the longhand form of NewGenesisBuilder().ChainID(chainID)... should be used. -func DefaultGenesisBuilderOnlyValidators( - chainID string, - sv StakingValidators, - validatorAmount sdk.Coin, -) *GenesisBuilder { - cmtVals := make(CometGenesisValidators, len(sv)) - for i := range sv { - cmtVals[i] = sv[i].C - } - - b := NewGenesisBuilder(). - ChainID(chainID). - DefaultAuthParams(). - Consensus(nil, cmtVals). - BaseAccounts(sv.BaseAccounts(), nil). - StakingWithDefaultParams(nil, nil). - BankingWithDefaultParams(sv.Balances(), nil, nil, nil). - DefaultDistribution(). - DefaultMint(). - SlashingWithDefaultParams(nil, nil) - - for _, v := range sv { - b.GenTx(*v.PK.Del, v.C.V, sdk.NewCoin(sdk.DefaultBondDenom, sdk.DefaultPowerReduction)) - } - - return b -} diff --git a/testutil/testnet/genesis_test.go b/testutil/testnet/genesis_test.go deleted file mode 100644 index c64044187889..000000000000 --- a/testutil/testnet/genesis_test.go +++ /dev/null @@ -1,245 +0,0 @@ -package testnet_test - -import ( - "encoding/base64" - "encoding/json" - "testing" - "time" - - cmted25519 "github.com/cometbft/cometbft/crypto/ed25519" - "github.com/stretchr/testify/require" - - "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" - "github.com/cosmos/cosmos-sdk/testutil/testnet" - sdk "github.com/cosmos/cosmos-sdk/types" - "github.com/cosmos/cosmos-sdk/types/bech32" -) - -func TestGenesisBuilder_GenesisTime(t *testing.T) { - t.Run("defaults to current time", func(t *testing.T) { - before := time.Now() - time.Sleep(time.Millisecond) // So that the genesis time will be strictly after "before". - gb := testnet.NewGenesisBuilder() - time.Sleep(time.Millisecond) // So that the genesis time will be strictly before "after". - after := time.Now() - - var gts string - require.NoError(t, json.Unmarshal(gb.JSON()["genesis_time"], >s)) - - gt, err := time.Parse(time.RFC3339Nano, gts) - require.NoError(t, err) - - require.True(t, gt.After(before)) - require.True(t, after.After(gt)) - }) - - t.Run("can be set to arbitrary time", func(t *testing.T) { - want := time.Date(2023, 3, 27, 9, 50, 23, 0, time.UTC) - - gb := testnet.NewGenesisBuilder().GenesisTime(want) - - var gts string - require.NoError(t, json.Unmarshal(gb.JSON()["genesis_time"], >s)) - - gt, err := time.Parse(time.RFC3339Nano, gts) - require.NoError(t, err) - - require.True(t, gt.Equal(want)) - }) -} - -func TestGenesisBuilder_InitialHeight(t *testing.T) { - t.Run("defaults to 1", func(t *testing.T) { - var ih string - require.NoError( - t, - json.Unmarshal( - testnet.NewGenesisBuilder().JSON()["initial_height"], - &ih, - ), - ) - - require.Equal(t, ih, "1") - }) - - t.Run("can be set to arbitrary value", func(t *testing.T) { - var ih string - require.NoError( - t, - json.Unmarshal( - testnet.NewGenesisBuilder().InitialHeight(12345).JSON()["initial_height"], - &ih, - ), - ) - - require.Equal(t, ih, "12345") - }) -} - -func TestGenesisBuilder_ChainID(t *testing.T) { - // No default. - gb := testnet.NewGenesisBuilder() - m := gb.JSON() - _, ok := m["chain_id"] - require.False(t, ok) - - var id string - require.NoError( - t, - json.Unmarshal( - gb.ChainID("my-chain").JSON()["chain_id"], - &id, - ), - ) - require.Equal(t, id, "my-chain") -} - -// Use known keys and addresses to assert that correct validator and delegator keys -// occur in the expected locations (i.e. we didn't mistakenly swap the keys anywhere). -func TestGenesisBuilder_GentxAddresses(t *testing.T) { - const valSecret0 = "val-secret-0" - const valAddr0 = "3F3B076353767F046477A6E0982F808C24D1870A" - const valPubKey0 = "ZhVhrOUHnUwYw/GlBSBrw/0X6A261gchCRYkAxGF2jk=" - valKey0 := cmted25519.GenPrivKeyFromSecret([]byte(valSecret0)) - if addr := valKey0.PubKey().Address().String(); addr != valAddr0 { - t.Fatalf("unexpected address %q for validator key 0 (expected %q)", addr, valAddr0) - } - if pub := base64.StdEncoding.EncodeToString(valKey0.PubKey().Bytes()); pub != valPubKey0 { - t.Fatalf("unexpected public key %q for validator key 0 (expected %q)", pub, valAddr0) - } - - const delSecret0 = "del-secret-0" - const delAddr0 = "30D7E04DA313C31B59A46408494B4272F0A9A256" - const delPubKey0 = "Aol+ZF9xBuZmYJrT1QFLpZBvSfr/zEKifWyg0Xi1tsFV" - const delAccAddr0 = "cosmos1xrt7qndrz0p3kkdyvsyyjj6zwtc2ngjky8dcpe" - delKey0 := secp256k1.GenPrivKeyFromSecret([]byte(delSecret0)) - if addr := delKey0.PubKey().Address().String(); addr != delAddr0 { - t.Fatalf("unexpected address %q for delegator key 0 (expected %q)", addr, delAddr0) - } - if pub := base64.StdEncoding.EncodeToString(delKey0.PubKey().Bytes()); pub != delPubKey0 { - t.Fatalf("unexpected public key %q for delegator key 0 (expected %q)", pub, delAddr0) - } - da, err := bech32.ConvertAndEncode("cosmos", delKey0.PubKey().Address().Bytes()) - require.NoError(t, err) - if da != delAccAddr0 { - t.Fatalf("unexpected account address %q for delegator key 0 (expected %q)", da, delAccAddr0) - } - - valPKs := testnet.ValidatorPrivKeys{ - &testnet.ValidatorPrivKey{ - Val: valKey0, - Del: delKey0, - }, - } - cmtVals := valPKs.CometGenesisValidators() - stakingVals := cmtVals.StakingValidators() - valBaseAccounts := stakingVals.BaseAccounts() - - b := testnet.NewGenesisBuilder(). - ChainID("my-test-chain"). - DefaultAuthParams(). - Consensus(nil, cmtVals). - BaseAccounts(valBaseAccounts, nil). - StakingWithDefaultParams(stakingVals, nil) - - for i, v := range valPKs { - b.GenTx(*v.Del, cmtVals[i].V, sdk.NewCoin(sdk.DefaultBondDenom, sdk.DefaultPowerReduction)) - } - - var g struct { - Consensus struct { - Validators []struct { - Address string `json:"address"` - PubKey struct { - Value string `json:"value"` - } `json:"pub_key"` - } `json:"validators"` - } `json:"consensus"` - - AppState struct { - Genutil struct { - GenTxs []struct { - Body struct { - Messages []struct { - Type string `json:"@type"` - DelegatorAddress string `json:"delegator_address"` - ValidatorAddress string `json:"validator_address"` - PubKey struct { - Key string `json:"key"` - } `json:"pubkey"` - } `json:"messages"` - } `json:"body"` - AuthInfo struct { - SignerInfos []struct { - PublicKey struct { - Key string `json:"key"` - } `json:"public_key"` - } `json:"signer_infos"` - } `json:"auth_info"` - } `json:"gen_txs"` - } `json:"genutil"` - - Auth struct { - Accounts []struct { - Address string `json:"address"` - PubKey struct { - Key string `json:"key"` - } `json:"pub_key"` - } `json:"accounts"` - } `json:"auth"` - } `json:"app_state"` - } - if err := json.Unmarshal(b.Encode(), &g); err != nil { - t.Fatal(err) - } - - // Validator encoded as expected. - vals := g.Consensus.Validators - require.Equal(t, vals[0].Address, valAddr0) - require.Equal(t, vals[0].PubKey.Value, valPubKey0) - - // Public keys on gentx message match correct keys (no ed25519/secp256k1 mismatch). - gentxs := g.AppState.Genutil.GenTxs - require.Equal(t, gentxs[0].Body.Messages[0].PubKey.Key, valPubKey0) - require.Equal(t, gentxs[0].AuthInfo.SignerInfos[0].PublicKey.Key, delPubKey0) - - // The only base account in this genesis, matches the secp256k1 key. - acct := g.AppState.Auth.Accounts[0] - require.Equal(t, acct.Address, delAccAddr0) - require.Equal(t, acct.PubKey.Key, delPubKey0) -} - -func ExampleGenesisBuilder() { - const nVals = 4 - - // Generate random private keys for validators. - valPKs := testnet.NewValidatorPrivKeys(nVals) - - // Produce the comet representation of those validators. - cmtVals := valPKs.CometGenesisValidators() - - stakingVals := cmtVals.StakingValidators() - - // Configure a new genesis builder - // with a fairly thorough set of defaults. - // - // If you only ever need defaults, you can use DefaultGenesisBuilderOnlyValidators(). - b := testnet.NewGenesisBuilder(). - ChainID("my-chain-id"). - DefaultAuthParams(). - Consensus(nil, cmtVals). - BaseAccounts(stakingVals.BaseAccounts(), nil). - DefaultStaking(). - BankingWithDefaultParams(stakingVals.Balances(), nil, nil, nil). - DefaultDistribution(). - DefaultMint(). - DefaultSlashing() - - for i := range stakingVals { - b.GenTx(*valPKs[i].Del, cmtVals[i].V, sdk.NewCoin(sdk.DefaultBondDenom, sdk.DefaultPowerReduction)) - } - - // Now, you can access b.JSON() if you need to make further modifications - // not (yet) supported by the GenesisBuilder API, - // or you can use b.Encode() for the serialzed JSON of the genesis. -} diff --git a/testutil/testnet/network.go b/testutil/testnet/network.go deleted file mode 100644 index ecba45fa0580..000000000000 --- a/testutil/testnet/network.go +++ /dev/null @@ -1,116 +0,0 @@ -package testnet - -import ( - "errors" - "fmt" - "sync" - - "github.com/cometbft/cometbft/p2p" -) - -// NewNetwork concurrently calls createCometStarter, nVals times; -// then it returns a slice of started comet nodes, -// in order corresponding with the number passed to createCometStarter. -// The returned nodes will all be peered together, -// by dialing each node's [github.com/cometbft/cometbft/p2p/pex.Reactor] to each other. -// -// Every node is attempted to be started, -// and any errors collected are joined together and returned. -// -// In the event of errors, a non-nil Nodes slice may still be returned -// and some elements may be nil. -// Callers should call [Nodes.Stop] and [Nodes.Wait] to perform cleanup, -// regardless of the returned error. -func NewNetwork(nVals int, createCometStarter func(int) *CometStarter) (Nodes, error) { - // The ordered slice of nodes; correct indexing is important. - // The creator goroutines will write directly into this slice. - nodes := make(Nodes, nVals) - - // Every node will be started in its own goroutine. - // We collect the switches so that each node can dial every other node. - switchCh := make(chan *p2p.Switch, nVals) - errCh := make(chan error, nVals) - - var wg sync.WaitGroup - // Start goroutines to populate nodes slice and notify as each node is available. - for i := 0; i < nVals; i++ { - wg.Add(1) - go func(i int) { - defer wg.Done() - - n, err := createCometStarter(i).Start() - if err != nil { - errCh <- fmt.Errorf("failed to start node %d: %w", i, err) - return - } - - // Notify that the new node's switch is available, so this node can be - // peered with the other nodes. - switchCh <- n.Switch() - - // And assign the node into its correct index in the ordered slice. - nodes[i] = n - }(i) - } - - // Once all the creation goroutines are complete, close the channels, - // to signal to the collection goroutines. - go func() { - wg.Wait() - close(errCh) - close(switchCh) - }() - - joinPeersDone := make(chan struct{}) - go joinPeers(switchCh, joinPeersDone) - - finalErrCh := make(chan error, 1) - go collectErrors(errCh, finalErrCh) - - // If there were any errors, return them. - // And return any set nodes, so that they can be cleaned up properly. - if err := <-finalErrCh; err != nil { - return nodes, err - } - - // No errors, so wait for peer joining to complete - // before returning the ordered nodes. - <-joinPeersDone - return nodes, nil -} - -// collectErrors collects all errors that arrive on the in channel, -// joins them, then sends the joined final error on the out channel. -func collectErrors(in <-chan error, out chan<- error) { - var errs []error - for err := range in { - errs = append(errs, err) - } - - var res error - if len(errs) > 0 { - res = errors.Join(errs...) - } - out <- res -} - -// joinPeers collects each switch arriving on newSwitches; -// each time a new switch arrives, it dials every previously seen switch. -// -// This allows each node to be started independently and concurrently -// without predetermined p2p ports. -func joinPeers(newSwitches <-chan *p2p.Switch, done chan<- struct{}) { - defer close(done) - - var readySwitches []*p2p.Switch - for newSwitch := range newSwitches { - newNetAddr := newSwitch.NetAddress() - for _, s := range readySwitches { - // For every new switch, connect with all the previously seen switches. - // It might not be necessary to dial in both directions, but it shouldn't hurt. - _ = s.DialPeerWithAddress(newNetAddr) - _ = newSwitch.DialPeerWithAddress(s.NetAddress()) - } - readySwitches = append(readySwitches, newSwitch) - } -} diff --git a/testutil/testnet/nodehelpers.go b/testutil/testnet/nodehelpers.go deleted file mode 100644 index dee10aeb7f30..000000000000 --- a/testutil/testnet/nodehelpers.go +++ /dev/null @@ -1,43 +0,0 @@ -package testnet - -import ( - "fmt" - "time" - - "github.com/cometbft/cometbft/node" -) - -// WaitForNodeHeight blocks until the node's consensus state reports -// a last height equal to or greater than desiredHeight. -// If totalWait has elapsed and the desired height has not been reached, -// an error is returned. -func WaitForNodeHeight(n *node.Node, desiredHeight int64, totalWait time.Duration) error { - const backOff = 100 * time.Millisecond - attempts := int64(totalWait / backOff) - - // In Comet 0.37, the consensus state was exposed directly on the Node. - // As of 0.38, the node no longer exposes consensus state, - // but the consensus state is available as a field on the RPC environment. - // - // Luckily, in 0.38 the RPC environment is no longer a package-level singleton, - // so retrieving the RPC environment for a single node should be safe. - env, err := n.ConfigureRPC() - if err != nil { - return fmt.Errorf("failed to configure RPC to reach into consensus state: %w", err) - } - - curHeight := int64(-1) - for i := int64(0); i < attempts; i++ { - curHeight = env.ConsensusState.GetState().LastBlockHeight - - if curHeight < desiredHeight { - time.Sleep(backOff) - continue - } - - // Met or exceeded target height. - return nil - } - - return fmt.Errorf("node did not reach desired height %d in %s; only reached height %d", desiredHeight, totalWait, curHeight) -} diff --git a/testutil/testnet/nodes.go b/testutil/testnet/nodes.go deleted file mode 100644 index d47699fc6fe2..000000000000 --- a/testutil/testnet/nodes.go +++ /dev/null @@ -1,70 +0,0 @@ -package testnet - -import ( - "errors" - "fmt" - "time" - - "github.com/cometbft/cometbft/node" -) - -// Nodes is a slice of comet nodes, -// with some additional convenience methods. -// -// Nodes may contain nil elements, -// so that a partially failed call to NewNetwork -// can still be properly cleaned up. -type Nodes []*node.Node - -// Stop stops each node sequentially. -// All errors occurring during stop are returned as a joined error. -// -// Nil elements in ns are skipped. -func (ns Nodes) Stop() error { - var errs []error - for i, n := range ns { - if n == nil { - continue - } - if err := n.Stop(); err != nil { - errs = append(errs, fmt.Errorf("failed to stop node %d: %w", i, err)) - } - } - - if len(errs) > 0 { - return errors.Join(errs...) - } - - return nil -} - -// Wait blocks until every node in ns has completely stopped. -// -// Nil elements in ns are skipped. -func (ns Nodes) Wait() { - for _, n := range ns { - if n == nil { - continue - } - n.Wait() - } -} - -// StopAndWait is shorthand for calling both Stop() and Wait(), -// useful as a deferred call in tests. -func (ns Nodes) StopAndWait() error { - err := ns.Stop() - ns.Wait() - - // TODO(mr): remove this sleep call after we are using a version of Comet - // that includes a fix for https://github.com/cometbft/cometbft/issues/646. - // - // On my machine, this sleep appears to completely eliminate the late file write. - // It also almost always works around https://github.com/cometbft/cometbft/issues/652. - // - // Presumably the fix for those two issues will be included in a v0.37.1 release. - // If not, I assume they will be part of the first v0.38 series release. - time.Sleep(250 * time.Millisecond) - - return err -} diff --git a/testutil/testnet/nodes_test.go b/testutil/testnet/nodes_test.go deleted file mode 100644 index f7737960c541..000000000000 --- a/testutil/testnet/nodes_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package testnet_test - -import ( - "testing" - - "github.com/cometbft/cometbft/node" - "github.com/stretchr/testify/require" - - "github.com/cosmos/cosmos-sdk/testutil/testnet" -) - -// Nil entries in a Nodes slice don't fail Stop or Wait. -func TestNodes_StopWaitNil(t *testing.T) { - for _, tc := range []struct { - Name string - Nodes []*node.Node - }{ - {Name: "nil slice", Nodes: nil}, - {Name: "slice with nil elements", Nodes: []*node.Node{nil}}, - } { - ns := testnet.Nodes(tc.Nodes) - t.Run(tc.Name, func(t *testing.T) { - require.NoError(t, ns.Stop()) - - // Nothing special to assert about Wait(). - // It should return immediately, without panicking. - ns.Wait() - }) - } -} diff --git a/testutil/testnet/validator.go b/testutil/testnet/validator.go deleted file mode 100644 index 973206a1f682..000000000000 --- a/testutil/testnet/validator.go +++ /dev/null @@ -1,189 +0,0 @@ -package testnet - -import ( - "fmt" - - cmted25519 "github.com/cometbft/cometbft/crypto/ed25519" - cmttypes "github.com/cometbft/cometbft/types" - - sdkmath "cosmossdk.io/math" - - codectypes "github.com/cosmos/cosmos-sdk/codec/types" - cryptocodec "github.com/cosmos/cosmos-sdk/crypto/codec" - "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" - sdk "github.com/cosmos/cosmos-sdk/types" - "github.com/cosmos/cosmos-sdk/types/bech32" - authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" - banktypes "github.com/cosmos/cosmos-sdk/x/bank/types" - stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types" -) - -// ValidatorPrivKeys is a slice of [*ValidatorPrivKey]. -type ValidatorPrivKeys []*ValidatorPrivKey - -// ValidatorPrivKey holds a validator key (a comet ed25519 key) -// and the validator's delegator or account key (a Cosmos SDK secp256k1 key). -type ValidatorPrivKey struct { - Val cmted25519.PrivKey - Del *secp256k1.PrivKey -} - -// NewValidatorPrivKeys returns a ValidatorPrivKeys of length n, -// where each set of keys is dynamically generated. -// -// If writing a test where deterministic keys are required, -// the caller should manually construct a slice and assign each key as needed. -func NewValidatorPrivKeys(n int) ValidatorPrivKeys { - vpk := make(ValidatorPrivKeys, n) - - for i := range vpk { - vpk[i] = &ValidatorPrivKey{ - Val: cmted25519.GenPrivKey(), - Del: secp256k1.GenPrivKey(), - } - } - - return vpk -} - -// CometGenesisValidators derives the CometGenesisValidators belonging to vpk. -func (vpk ValidatorPrivKeys) CometGenesisValidators() CometGenesisValidators { - cgv := make(CometGenesisValidators, len(vpk)) - - for i, pk := range vpk { - pubKey := pk.Val.PubKey() - - const votingPower = 1 - cmtVal := cmttypes.NewValidator(pubKey, votingPower) - - cgv[i] = &CometGenesisValidator{ - V: cmttypes.GenesisValidator{ - Address: cmtVal.Address, - PubKey: cmtVal.PubKey, - Power: cmtVal.VotingPower, - Name: fmt.Sprintf("val-%d", i), - }, - PK: pk, - } - } - - return cgv -} - -// CometGenesisValidators is a slice of [*CometGenesisValidator]. -type CometGenesisValidators []*CometGenesisValidator - -// CometGenesisValidator holds a comet GenesisValidator -// and a reference to the ValidatorPrivKey from which the CometGenesisValidator was derived. -type CometGenesisValidator struct { - V cmttypes.GenesisValidator - PK *ValidatorPrivKey -} - -// ToComet returns a new slice of [cmttypes.GenesisValidator], -// useful for some interactions. -func (cgv CometGenesisValidators) ToComet() []cmttypes.GenesisValidator { - vs := make([]cmttypes.GenesisValidator, len(cgv)) - for i, v := range cgv { - vs[i] = v.V - } - return vs -} - -// StakingValidators derives the StakingValidators belonging to cgv. -func (cgv CometGenesisValidators) StakingValidators() StakingValidators { - vals := make(StakingValidators, len(cgv)) - for i, v := range cgv { - pk, err := cryptocodec.FromCmtPubKeyInterface(v.V.PubKey) - if err != nil { - panic(fmt.Errorf("failed to extract comet pub key: %w", err)) - } - - pkAny, err := codectypes.NewAnyWithValue(pk) - if err != nil { - panic(fmt.Errorf("failed to wrap pub key in any type: %w", err)) - } - - vals[i] = &StakingValidator{ - V: stakingtypes.Validator{ - OperatorAddress: sdk.ValAddress(v.V.Address).String(), // TODO: this relies on global bech32 config. - ConsensusPubkey: pkAny, - Status: stakingtypes.Bonded, - Tokens: sdk.DefaultPowerReduction, - DelegatorShares: sdkmath.LegacyOneDec(), - MinSelfDelegation: sdkmath.ZeroInt(), - - // more fields uncopied from testutil/sims/app_helpers.go:220 - }, - C: v, - PK: v.PK, - } - } - - return vals -} - -// StakingValidators is a slice of [*StakingValidator]. -type StakingValidators []*StakingValidator - -// StakingValidator holds a [stakingtypes.Validator], -// and the CometGenesisValidator and ValidatorPrivKey required to derive the StakingValidator. -type StakingValidator struct { - V stakingtypes.Validator - C *CometGenesisValidator - PK *ValidatorPrivKey -} - -// ToStakingType returns a new slice of [stakingtypes.Validator], -// useful for some interactions. -func (sv StakingValidators) ToStakingType() []stakingtypes.Validator { - vs := make([]stakingtypes.Validator, len(sv)) - for i, v := range sv { - vs[i] = v.V - } - return vs -} - -// BaseAccounts returns the BaseAccounts for this set of StakingValidators. -// The base accounts are important for [*GenesisBuilder.BaseAccounts]. -func (sv StakingValidators) BaseAccounts() BaseAccounts { - ba := make(BaseAccounts, len(sv)) - - for i, v := range sv { - const accountNumber = 0 - const sequenceNumber = 0 - - pubKey := v.PK.Del.PubKey() - bech, err := bech32.ConvertAndEncode("cosmos", pubKey.Address().Bytes()) // TODO: this shouldn't be hardcoded to cosmos! - if err != nil { - panic(err) - } - accAddr, err := sdk.AccAddressFromBech32(bech) - if err != nil { - panic(err) - } - ba[i] = authtypes.NewBaseAccount( - accAddr, pubKey, accountNumber, sequenceNumber, - ) - } - - return ba -} - -// Balances returns the balances held by this set of StakingValidators. -func (sv StakingValidators) Balances() []banktypes.Balance { - bals := make([]banktypes.Balance, len(sv)) - - for i, v := range sv { - addr, err := bech32.ConvertAndEncode("cosmos", v.PK.Del.PubKey().Address().Bytes()) // TODO: this shouldn't be hardcoded to cosmos! - if err != nil { - panic(err) - } - bals[i] = banktypes.Balance{ - Address: addr, - Coins: sdk.Coins{sdk.NewCoin(sdk.DefaultBondDenom, v.V.Tokens)}, - } - } - - return bals -}