Merge branch 'development' into 1garo/move-sig-verifier-to-crypto

.github/CODEOWNERS

@@ -1,4 +1,4 @@
 # CODEOWNERS: https://help.github.com/articles/about-codeowners/
 
 # Primary repo maintainers
-*       @noot @arijitAD @edwardmack @timwu20 @EclesioMeloJunior @jimjbrettj @kishansagathiya @omar391
+*       @noot @arijitAD @edwardmack @timwu20 @EclesioMeloJunior @jimjbrettj @kishansagathiya @qdm12

.github/workflows/release.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Setup Node.js
         uses: actions/setup-node@v2.4.1
         with:
-          node-version: 12
+          node-version: 16
       - name: Install dependencies
         run: npm install @semantic-release/changelog @semantic-release/git @semantic-release/changelog @semantic-release/commit-analyzer @semantic-release/exec @semantic-release/git semantic-release/release-notes-generator
       - name: Release

.golangci.yml

@@ -27,7 +27,6 @@ run:
   skip-files:
     - .*mock_.*\.go
     - .*mocks\/.*\.go
-    - author_integration_test\.go # TODO remove once fixed
 
 # all available settings of specific linters
 linters-settings:

chain/gssmr/config.toml

@@ -32,6 +32,7 @@ nobootstrap = false
 nomdns = false
 discovery-interval = 10
 min-peers = 1
+max-peers = 50
 
 [rpc]
 enabled = false

chain/gssmr/defaults.go

@@ -74,6 +74,8 @@ var (
 	DefaultNoMDNS = false
 	// DefaultMinPeers is the default minimum desired peer count
 	DefaultMinPeers = 1
+	// DefaultMaxPeers is the default maximum desired peer count
+	DefaultMaxPeers = 50
 
 	// DefaultDiscoveryInterval is the default interval for searching for DHT peers
 	DefaultDiscoveryInterval = time.Second * 10

cmd/gossamer/config.go

@@ -121,7 +121,9 @@ func createDotConfig(ctx *cli.Context) (*dot.Config, error) {
 		return nil, err
 	}
 
-	logger.Infof("loaded package log configuration: %v", cfg.Log)
+	// TODO: log this better.
+	// See https://github.com/ChainSafe/gossamer/issues/1945
+	logger.Infof("loaded package log configuration: %#v", cfg.Log)
 
 	// set global configuration values
 	if err := setDotGlobalConfig(ctx, tomlCfg, &cfg.Global); err != nil {

cmd/gossamer/config_test.go

@@ -458,6 +458,7 @@ func TestNetworkConfigFromFlags(t *testing.T) {
 				NoMDNS:            testCfg.Network.NoMDNS,
 				DiscoveryInterval: time.Second * 10,
 				MinPeers:          testCfg.Network.MinPeers,
+				MaxPeers:          testCfg.Network.MaxPeers,
 			},
 		},
 		{
@@ -472,6 +473,7 @@ func TestNetworkConfigFromFlags(t *testing.T) {
 				NoMDNS:            testCfg.Network.NoMDNS,
 				DiscoveryInterval: time.Second * 10,
 				MinPeers:          testCfg.Network.MinPeers,
+				MaxPeers:          testCfg.Network.MaxPeers,
 			},
 		},
 		{
@@ -486,6 +488,7 @@ func TestNetworkConfigFromFlags(t *testing.T) {
 				NoMDNS:            testCfg.Network.NoMDNS,
 				DiscoveryInterval: time.Second * 10,
 				MinPeers:          testCfg.Network.MinPeers,
+				MaxPeers:          testCfg.Network.MaxPeers,
 			},
 		},
 		{
@@ -500,6 +503,7 @@ func TestNetworkConfigFromFlags(t *testing.T) {
 				NoMDNS:            testCfg.Network.NoMDNS,
 				DiscoveryInterval: time.Second * 10,
 				MinPeers:          testCfg.Network.MinPeers,
+				MaxPeers:          testCfg.Network.MaxPeers,
 			},
 		},
 		{
@@ -514,6 +518,7 @@ func TestNetworkConfigFromFlags(t *testing.T) {
 				NoMDNS:            true,
 				DiscoveryInterval: time.Second * 10,
 				MinPeers:          testCfg.Network.MinPeers,
+				MaxPeers:          testCfg.Network.MaxPeers,
 			},
 		},
 		{
@@ -528,6 +533,7 @@ func TestNetworkConfigFromFlags(t *testing.T) {
 				NoMDNS:            false,
 				DiscoveryInterval: time.Second * 10,
 				MinPeers:          testCfg.Network.MinPeers,
+				MaxPeers:          testCfg.Network.MaxPeers,
 				PublicIP:          "10.0.5.2",
 			},
 		},
@@ -909,6 +915,7 @@ func TestUpdateConfigFromGenesisData(t *testing.T) {
 			NoMDNS:            testCfg.Network.NoMDNS,
 			DiscoveryInterval: testCfg.Network.DiscoveryInterval,
 			MinPeers:          testCfg.Network.MinPeers,
+			MaxPeers:          testCfg.Network.MaxPeers,
 		},
 		RPC:    testCfg.RPC,
 		System: testCfg.System,

cmd/gossamer/export_test.go

@@ -75,6 +75,7 @@ func TestExportCommand(t *testing.T) {
 					NoMDNS:            testCfg.Network.NoMDNS,
 					DiscoveryInterval: testCfg.Network.DiscoveryInterval,
 					MinPeers:          testCfg.Network.MinPeers,
+					MaxPeers:          testCfg.Network.MaxPeers,
 				},
 				RPC:   testCfg.RPC,
 				Pprof: testCfg.Pprof,
@@ -112,6 +113,7 @@ func TestExportCommand(t *testing.T) {
 					NoMDNS:            testCfg.Network.NoMDNS,
 					DiscoveryInterval: testCfg.Network.DiscoveryInterval,
 					MinPeers:          testCfg.Network.MinPeers,
+					MaxPeers:          testCfg.Network.MaxPeers,
 				},
 				RPC:   testCfg.RPC,
 				Pprof: testCfg.Pprof,
@@ -149,6 +151,7 @@ func TestExportCommand(t *testing.T) {
 					NoMDNS:            testCfg.Network.NoMDNS,
 					DiscoveryInterval: testCfg.Network.DiscoveryInterval,
 					MinPeers:          testCfg.Network.MinPeers,
+					MaxPeers:          testCfg.Network.MaxPeers,
 				},
 				RPC:   testCfg.RPC,
 				Pprof: testCfg.Pprof,

dot/config.go

@@ -207,6 +207,7 @@ func GssmrConfig() *Config {
 			NoMDNS:            gssmr.DefaultNoMDNS,
 			DiscoveryInterval: gssmr.DefaultDiscoveryInterval,
 			MinPeers:          gssmr.DefaultMinPeers,
+			MaxPeers:          gssmr.DefaultMaxPeers,
 		},
 		RPC: RPCConfig{
 			Port:    gssmr.DefaultRPCHTTPPort,

dot/network/config.go

@@ -8,8 +8,9 @@ import (
 	"path"
 	"time"
 
-	"github.com/ChainSafe/gossamer/internal/log"
 	"github.com/libp2p/go-libp2p-core/crypto"
+
+	"github.com/ChainSafe/gossamer/internal/log"
 )
 
 const (
@@ -39,6 +40,8 @@ const (
 
 	// DefaultDiscoveryInterval is the default interval for searching for DHT peers
 	DefaultDiscoveryInterval = time.Minute * 5
+
+	defaultTxnBatchSize = 100
 )
 
 // DefaultBootnodes the default value for Config.Bootnodes
@@ -93,6 +96,11 @@ type Config struct {
 	telemetryInterval time.Duration
 
 	noPreAllocate bool // internal option
+
+	batchSize int // internal option
+
+	// SlotDuration is the slot duration to produce a block
+	SlotDuration time.Duration
 }
 
 // build checks the configuration, sets up the private key for the network service,

dot/network/connmgr.go

@@ -5,8 +5,6 @@ package network
 
 import (
 	"context"
-	"crypto/rand"
-	"math/big"
 	"sync"
 
 	"github.com/libp2p/go-libp2p-core/connmgr"
@@ -130,40 +128,10 @@ func (cm *ConnManager) unprotectedPeers(peers []peer.ID) []peer.ID {
 func (cm *ConnManager) Connected(n network.Network, c network.Conn) {
 	logger.Tracef(
 		"Host %s connected to peer %s", n.LocalPeer(), c.RemotePeer())
+
 	if cm.connectHandler != nil {
 		cm.connectHandler(c.RemotePeer())
 	}
-
-	cm.Lock()
-	defer cm.Unlock()
-
-	over := len(n.Peers()) - cm.max
-	if over <= 0 {
-		return
-	}
-
-	// TODO: peer scoring doesn't seem to prevent us from going over the max.
-	// if over the max peer count, disconnect from (total_peers - maximum) peers
-	// (#2039)
-	for i := 0; i < over; i++ {
-		unprotPeers := cm.unprotectedPeers(n.Peers())
-		if len(unprotPeers) == 0 {
-			return
-		}
-
-		i, err := rand.Int(rand.Reader, big.NewInt(int64(len(unprotPeers))))
-		if err != nil {
-			logger.Errorf("error generating random number: %s", err)
-			return
-		}
-
-		up := unprotPeers[i.Int64()]
-		logger.Tracef("Over max peer count, disconnecting from random unprotected peer %s", up)
-		err = n.ClosePeer(up)
-		if err != nil {
-			logger.Tracef("failed to close connection to peer %s", up)
-		}
-	}
 }
 
 // Disconnected is called when a connection closed

dot/network/connmgr_test.go

@@ -43,10 +43,15 @@ func TestMinPeers(t *testing.T) {
 	}
 
 	nodeB := createTestService(t, configB)
-	require.Equal(t, min, nodeB.host.peerCount())
+	require.GreaterOrEqual(t, nodeB.host.peerCount(), len(nodes))
 
-	nodeB.host.cm.peerSetHandler.DisconnectPeer(0, nodes[0].host.id())
-	require.GreaterOrEqual(t, min, nodeB.host.peerCount())
+	// check that peer count is at least greater than minimum number of peers,
+	// even after trying to disconnect from all peers
+	for _, node := range nodes {
+		nodeB.host.cm.peerSetHandler.DisconnectPeer(0, node.host.id())
+	}
+
+	require.GreaterOrEqual(t, nodeB.host.peerCount(), min)
 }
 
 func TestMaxPeers(t *testing.T) {

dot/network/discovery.go

@@ -152,7 +152,7 @@ func (d *discovery) advertise() {
 
 			ttl, err = d.rd.Advertise(d.ctx, string(d.pid))
 			if err != nil {
-				logger.Debugf("failed to advertise in the DHT: %s", err)
+				logger.Warnf("failed to advertise in the DHT: %s", err)
 				ttl = tryAdvertiseTimeout
 			}
 		case <-d.ctx.Done():
@@ -199,21 +199,9 @@ func (d *discovery) findPeers(ctx context.Context) {
 
 			logger.Tracef("found new peer %s via DHT", peer.ID)
 
-			// TODO: this isn't working on the devnet (#2026)
-			// can remove the code block below which directly connects
-			// once that's fixed
 			d.h.Peerstore().AddAddrs(peer.ID, peer.Addrs, peerstore.PermanentAddrTTL)
 			d.handler.AddPeer(0, peer.ID)
 
-			// found a peer, try to connect if we need more peers
-			if len(d.h.Network().Peers()) >= d.maxPeers {
-				d.h.Peerstore().AddAddrs(peer.ID, peer.Addrs, peerstore.PermanentAddrTTL)
-				return
-			}
-
-			if err = d.h.Connect(d.ctx, peer); err != nil {
-				logger.Tracef("failed to connect to discovered peer %s: %s", peer.ID, err)
-			}
 		}
 	}
 }

dot/network/discovery_test.go

@@ -9,12 +9,12 @@ import (
 	"testing"
 	"time"
 
-	"github.com/ChainSafe/gossamer/lib/utils"
 	badger "github.com/ipfs/go-ds-badger2"
 	"github.com/libp2p/go-libp2p-core/peer"
 	"github.com/libp2p/go-libp2p-core/routing"
-
 	"github.com/stretchr/testify/require"
+
+	"github.com/ChainSafe/gossamer/lib/utils"
 )
 
 func newTestDiscovery(t *testing.T, num int) []*discovery {

dot/network/host.go

@@ -98,12 +98,16 @@ func newHost(ctx context.Context, cfg *Config) (*host, error) {
 		return nil, err
 	}
 
+	// We have tried to set maxInPeers and maxOutPeers such that number of peer
+	// connections remain between min peers and max peers
 	const reservedOnly = false
 	peerCfgSet := peerset.NewConfigSet(
 		uint32(cfg.MaxPeers-cfg.MinPeers),
-		uint32(cfg.MinPeers),
+		uint32(cfg.MaxPeers/2),
 		reservedOnly,
-		peerSetSlotAllocTime)
+		peerSetSlotAllocTime,
+	)
+
 	// create connection manager
 	cm, err := newConnManager(cfg.MinPeers, cfg.MaxPeers, peerCfgSet)
 	if err != nil {
@@ -394,3 +398,18 @@ func (h *host) protocols() []string {
 func (h *host) closePeer(peer peer.ID) error {
 	return h.h.Network().ClosePeer(peer)
 }
+
+func (h *host) closeProtocolStream(pID protocol.ID, p peer.ID) {
+	connToPeer := h.h.Network().ConnsToPeer(p)
+	for _, c := range connToPeer {
+		for _, st := range c.GetStreams() {
+			if st.Protocol() != pID {
+				continue
+			}
+			err := st.Close()
+			if err != nil {
+				logger.Tracef("Failed to close stream for protocol %s: %s", pID, err)
+			}
+		}
+	}
+}

dot/network/message.go

@@ -8,12 +8,13 @@ import (
 	"errors"
 	"fmt"
 
+	"google.golang.org/protobuf/proto"
+
 	pb "github.com/ChainSafe/gossamer/dot/network/proto"
 	"github.com/ChainSafe/gossamer/dot/types"
 	"github.com/ChainSafe/gossamer/lib/common"
 	"github.com/ChainSafe/gossamer/lib/common/variadic"
 	"github.com/ChainSafe/gossamer/pkg/scale"
-	"google.golang.org/protobuf/proto"
 )
 
 // Message types for notifications protocol messages. Used internally to map message to protocol.
@@ -305,8 +306,7 @@ func blockDataToProtobuf(bd *types.BlockData) (*pb.BlockData, error) {
 
 func protobufToBlockData(pbd *pb.BlockData) (*types.BlockData, error) {
 	bd := &types.BlockData{
-		Hash:   common.BytesToHash(pbd.Hash),
-		Header: types.NewEmptyHeader(),
+		Hash: common.BytesToHash(pbd.Hash),
 	}
 
 	if pbd.Header != nil {