Merge pull request #654 from ipfs/release-v0.22.0

Release v0.22.0

CHANGELOG.md

@@ -24,12 +24,32 @@ The following emojis are used to highlight certain changes:
 
 ### Security
 
+## [v0.22.0]
+
+### Changed
+
+- `go-libp2p` dependency updated to [v0.36 (release notes)](https://github.com/libp2p/go-libp2p/releases/tag/v0.36.1)
+- `bitswap/server` minor memory use and performance improvements
+- `bitswap` unify logger names to use uniform format bitswap/path/pkgname
+- `gateway` now always returns meaningful cache-control headers for generated HTML listings of UnixFS directories
+- generate random test data using `ipfs/go-test` instead of internal util code
+
+### Removed
+
+- `util` logic for generating random test data moved to [`ipfs/go-test/random`](https://github.com/ipfs/go-test)
+
+### Fixed
+
+- `boxo/gateway` now correctly returns 404 Status Not Found instead of 500 when the requested content cannot be found due to offline exchange, gateway running in no-fetch (non-recursive) mode, or a similar restriction that only serves a specific set of CIDs.
+- `bitswap/client` fix memory leak in BlockPresenceManager due to unlimited map growth.
+- `bitswap/network` fixed race condition when a timeout occurred before hole punching completed while establishing a first-time stream to a peer behind a NAT
+
 ## [v0.21.0]
 
 ### Changed
 
 - `boxo/gateway` is now tested against [gateway-conformance v6](https://github.com/ipfs/gateway-conformance/releases/tag/v0.6.0)
-- `bitswap/client` supports additional tracing 
+- `bitswap/client` supports additional tracing
 
 ### Removed
 
@@ -39,6 +59,7 @@ The following emojis are used to highlight certain changes:
 
 - `routing/http`: the `FindPeer` now returns `routing.ErrNotFound` when no addresses are found
 - `routing/http`: the `FindProvidersAsync` no longer causes a goroutine buildup
+- `bitswap`: wantlist overflow handling now cancels existing entries to make room for newer entries. This fix prevents the wantlist from filling up with CIDs that the server does not have.
 
 ## [v0.20.0]
 

README.md

@@ -85,7 +85,7 @@ Boxo includes high-quality components useful for interacting with IPFS protocols
 - Interacting with public and private IPFS networks
 - Working with content-addressed data
 
-Boxo aims to provide a cohesive interface into these components. Note that not all of the underlying components necessarily reside in this respository.
+Boxo aims to provide a cohesive interface into these components. Note that not all of the underlying components necessarily reside in this repository.
 
 ### Does Boxo == IPFS?
 

bitswap/benchmarks_test.go

@@ -12,8 +12,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/ipfs/boxo/bitswap/internal/testutil"
 	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-test/random"
 	protocol "github.com/libp2p/go-libp2p/core/protocol"
 
 	"github.com/ipfs/boxo/bitswap"
@@ -169,8 +169,8 @@ func BenchmarkFetchFromOldBitswap(b *testing.B) {
 			testinstance.ConnectInstances(instances)
 
 			// Generate blocks, with a smaller root block
-			rootBlock := testutil.GenerateBlocksOfSize(1, rootBlockSize)
-			blocks := testutil.GenerateBlocksOfSize(bch.blockCount, stdBlockSize)
+			rootBlock := random.BlocksOfSize(1, rootBlockSize)
+			blocks := random.BlocksOfSize(bch.blockCount, stdBlockSize)
 			blocks[0] = rootBlock[0]
 
 			// Run the distribution
@@ -300,7 +300,7 @@ func BenchmarkDatacenterMultiLeechMultiSeed(b *testing.B) {
 			defer ig.Close()
 
 			instances := ig.Instances(numnodes)
-			blocks := testutil.GenerateBlocksOfSize(numblks, blockSize)
+			blocks := random.BlocksOfSize(numblks, int(blockSize))
 			runDistributionMulti(b, instances[:3], instances[3:], blocks, bstoreLatency, df, ff)
 		}
 	})
@@ -317,8 +317,8 @@ func subtestDistributeAndFetch(b *testing.B, numnodes, numblks int, d delay.D, b
 		ig := testinstance.NewTestInstanceGenerator(net, nil, nil)
 
 		instances := ig.Instances(numnodes)
-		rootBlock := testutil.GenerateBlocksOfSize(1, rootBlockSize)
-		blocks := testutil.GenerateBlocksOfSize(numblks, stdBlockSize)
+		rootBlock := random.BlocksOfSize(1, rootBlockSize)
+		blocks := random.BlocksOfSize(numblks, stdBlockSize)
 		blocks[0] = rootBlock[0]
 		runDistribution(b, instances, blocks, bstoreLatency, df, ff)
 		ig.Close()
@@ -333,8 +333,8 @@ func subtestDistributeAndFetchRateLimited(b *testing.B, numnodes, numblks int, d
 		defer ig.Close()
 
 		instances := ig.Instances(numnodes)
-		rootBlock := testutil.GenerateBlocksOfSize(1, rootBlockSize)
-		blocks := testutil.GenerateBlocksOfSize(numblks, blockSize)
+		rootBlock := random.BlocksOfSize(1, rootBlockSize)
+		blocks := random.BlocksOfSize(numblks, int(blockSize))
 		blocks[0] = rootBlock[0]
 		runDistribution(b, instances, blocks, bstoreLatency, df, ff)
 	}

bitswap/client/client.go

@@ -38,7 +38,7 @@ import (
 	"go.opentelemetry.io/otel/trace"
 )
 
-var log = logging.Logger("bitswap-client")
+var log = logging.Logger("bitswap/client")
 
 // Option defines the functional option type that can be used to configure
 // bitswap instances

bitswap/client/internal/blockpresencemanager/blockpresencemanager.go

@@ -15,9 +15,7 @@ type BlockPresenceManager struct {
 }
 
 func New() *BlockPresenceManager {
-	return &BlockPresenceManager{
-		presence: make(map[cid.Cid]map[peer.ID]bool),
-	}
+	return &BlockPresenceManager{}
 }
 
 // ReceiveFrom is called when a peer sends us information about which blocks
@@ -26,6 +24,10 @@ func (bpm *BlockPresenceManager) ReceiveFrom(p peer.ID, haves []cid.Cid, dontHav
 	bpm.Lock()
 	defer bpm.Unlock()
 
+	if bpm.presence == nil {
+		bpm.presence = make(map[cid.Cid]map[peer.ID]bool)
+	}
+
 	for _, c := range haves {
 		bpm.updateBlockPresence(p, c, true)
 	}
@@ -75,6 +77,10 @@ func (bpm *BlockPresenceManager) AllPeersDoNotHaveBlock(peers []peer.ID, ks []ci
 	bpm.RLock()
 	defer bpm.RUnlock()
 
+	if len(bpm.presence) == 0 {
+		return nil
+	}
+
 	var res []cid.Cid
 	for _, c := range ks {
 		if bpm.allDontHave(peers, c) {
@@ -90,6 +96,9 @@ func (bpm *BlockPresenceManager) allDontHave(peers []peer.ID, c cid.Cid) bool {
 	if !cok {
 		return false
 	}
+	if len(ps) == 0 {
+		return false
+	}
 
 	// Check if we explicitly know that all the given peers do not have the cid
 	for _, p := range peers {
@@ -108,6 +117,25 @@ func (bpm *BlockPresenceManager) RemoveKeys(ks []cid.Cid) {
 	for _, c := range ks {
 		delete(bpm.presence, c)
 	}
+	if len(bpm.presence) == 0 {
+		bpm.presence = nil
+	}
+}
+
+// RemovePeer removes the given peer from every cid key in the presence map.
+func (bpm *BlockPresenceManager) RemovePeer(p peer.ID) {
+	bpm.Lock()
+	defer bpm.Unlock()
+
+	for c, pm := range bpm.presence {
+		delete(pm, p)
+		if len(pm) == 0 {
+			delete(bpm.presence, c)
+		}
+	}
+	if len(bpm.presence) == 0 {
+		bpm.presence = nil
+	}
 }
 
 // HasKey indicates whether the BlockPresenceManager is tracking the given key