Merge pull request #417 from ipfs/release-v0.11.0

Release v0.11.0

.github/pull_request_template.md

@@ -1,4 +1,5 @@
 <!--
-PR Creation Checklist
-- [ ] Update Changelog
+Please update the CHANGELOG.md if you're modifying Go files. If your change does not require a changelog entry, please do one of the following:
+- add `[skip changelog]` to the PR title
+- label the PR with `skip/changelog`
 -->

.github/workflows/changelog.yml

@@ -0,0 +1,39 @@
+name: Changelog
+
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - synchronize
+      - reopened
+      - labeled
+      - unlabeled
+    paths:
+      - '**.go'
+      - '**/go.mod'
+      - '**/go.sum'
+
+jobs:
+  changelog:
+    if: contains(github.event.pull_request.title, '[skip changelog]') == false &&
+        contains(github.event.pull_request.labels.*.name, 'skip/changelog') == false
+    runs-on: ubuntu-latest
+    name: Changelog
+    steps:
+      - id: changelog
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+          ENDPOINT: repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files
+          SELECTOR: 'map(select(.filename == "CHANGELOG.md")) | length'
+        run: gh api "$ENDPOINT" --jq "$SELECTOR" | xargs -I{} echo "modified={}" | tee -a $GITHUB_OUTPUT
+      - if: steps.changelog.outputs.modified == '0'
+        env:
+          MESSAGE: |
+            CHANGELOG.md was not modified in this PR. Please do one of the following:
+            - add a changelog entry
+            - add `[skip changelog]` to the PR title
+            - label the PR with `skip/changelog`
+        run: |
+          echo "::error::${MESSAGE//$'\n'/%0A}"
+          exit 1

.github/workflows/gateway-conformance.yml

@@ -16,7 +16,7 @@ jobs:
     steps:
       # 1. Download the gateway-conformance fixtures
       - name: Download gateway-conformance fixtures
-        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.1
+        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.2
         with:
           output: fixtures
           merged: true
@@ -40,15 +40,15 @@ jobs:
 
       # 4. Run the gateway-conformance tests
       - name: Run gateway-conformance tests
-        uses: ipfs/gateway-conformance/.github/actions/test@v0.1
+        uses: ipfs/gateway-conformance/.github/actions/test@v0.2
         with:
           gateway-url: http://127.0.0.1:8040
           json: output.json
           xml: output.xml
           html: output.html
           markdown: output.md
           subdomain-url: http://example.net
-          specs: -dnslink-resolver,-ipns-resolver
+          specs: -trustless-ipns-gateway,-path-ipns-gateway,-subdomain-ipns-gateway,-dnslink-gateway
           args: -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length'
 
       # 5. Upload the results

.github/workflows/stale.yml

@@ -2,7 +2,11 @@ name: Close and mark stale issue
 
 on:
   schedule:
-  - cron: '0 0 * * *'
+    - cron: '0 0 * * *'
+
+permissions:
+  issues: write
+  pull-requests: write
 
 jobs:
   stale:

CHANGELOG.md

@@ -24,6 +24,48 @@ The following emojis are used to highlight certain changes:
 
 ### Security
 
+## [v0.11.0]
+
+### Added
+
+* ✨ The gateway now supports the optional `order` and `dups` CAR parameters
+  from [IPIP-412](https://github.com/ipfs/specs/pull/412).
+  * The `BlocksBackend` only implements `order=dfs` (Depth-First Search)
+    ordering, which was already the default behavior.
+  * If a request specifies no `dups`, response with `dups=n` is returned, which
+    was already the default behavior.
+  * If a request explicitly specifies a CAR `order` other than `dfs`, it will
+    result in an error.
+  * The only change to the default behavior on CAR responses is that we follow
+    IPIP-412 and make `order=dfs;dups=n` explicit in the returned
+    `Content-Type` HTTP header.
+* ✨ While the call signature remains the same, the blocks that Bitswap returns can now be cast to [traceability.Block](./bitswap/client/traceability/block.go), which will additionally tell you where the Block came from and how long it took to fetch. This helps consumers of Bitswap collect better metrics on Bitswap behavior.
+
+### Changed
+
+* 🛠 The `ipns` package has been refactored.
+  * You should no longer use the direct Protobuf version of the IPNS Record.
+    Instead, we have a shiny new `ipns.Record` type that wraps all the required
+    functionality to work the best as possible with IPNS v2 Records. Please
+    check the [documentation](https://pkg.go.dev/github.com/ipfs/boxo/ipns) for
+    more information, and follow
+    [ipfs/specs#376](https://github.com/ipfs/specs/issues/376) for related
+    IPIP.
+  * There is no change to IPNS Records produced by `boxo/ipns`, it still
+    produces both V1 and V2 signatures by default, it is still backward-compatible.
+
+### Removed
+
+- 🛠 `ipld/car`  has been removed. Please use [ipld/go-car](https://github.com/ipld/go-car) instead.
+  More information regarding this decision can be found in [issue 218](https://github.com/ipfs/boxo/issues/218).
+
+### Fixed
+
+- Removed mentions of unused ARC algorithm ([#336](https://github.com/ipfs/boxo/issues/366#issuecomment-1597253540))
+- Handle `_redirects` file when `If-None-Match` header is present ([#412](https://github.com/ipfs/boxo/pull/412))
+
+### Security
+
 ## [0.10.2] - 2023-06-29
 
 ### Fixed
@@ -105,8 +147,9 @@ None.
   - `RecursiveKeys`
   - `InternalKeys`
 - 🛠 `provider/batched.New` has been moved to `provider.New` and arguments has been changed. (https://github.com/ipfs/boxo/pulls/273)
-  - a routing system is now passed with the `provider.Online` option, by default the system run in offline mode (push stuff onto the queue); and
-  - you do not have to pass a queue anymore, you pass a `datastore.Datastore` exclusively.
+  - A routing system is now passed with the `provider.Online` option, by default the system run in offline mode (push stuff onto the queue).
+  - When using `provider.Online` calling the `.Run` method is not required anymore, the background worker is implicitely started in the background by `provider.New`.
+  - You do not have to pass a queue anymore, you pass a `datastore.Datastore` exclusively.
 - 🛠 `provider.NewOfflineProvider` has been renamed to `provider.NewNoopProvider` to show more clearly that is does nothing. (https://github.com/ipfs/boxo/pulls/273)
 - 🛠 `provider.Provider` and `provider.Reprovider` has been merged under one `provider.System`. (https://github.com/ipfs/boxo/pulls/273)
 - 🛠 `routing/http` responses now return a streaming `iter.ResultIter` generic interface. (https://github.com/ipfs/boxo/pulls/18)

README.md

@@ -1,10 +1,9 @@
 <h1 align="center">
-Boxo 🍌
-<br>
-<img src="https://raw.githubusercontent.com/ipfs/boxo/main/logo.svg" alt="Boxo logo" title="Boxo logo" width="200">
+<img src="https://github.com/ipfs/boxo/assets/157609/3c5e7391-fbc2-405b-9efc-920f4fd13b39" alt="Boxo logo" title="Boxo logo" width="200">
 <br>
+BOXO: IPFS SDK for GO
 </h1>
-<p align="center" style="font-size: 1.2rem;">A library for building IPFS applications and implementations.</p>
+<p align="center" style="font-size: 1.2rem;">A set of libraries for building IPFS applications and implementations in GO.</p>
 
 <hr />
 

bitswap/client/bitswap_with_sessions_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/ipfs/boxo/bitswap"
 	"github.com/ipfs/boxo/bitswap/client/internal/session"
+	"github.com/ipfs/boxo/bitswap/client/traceability"
 	testinstance "github.com/ipfs/boxo/bitswap/testinstance"
 	tn "github.com/ipfs/boxo/bitswap/testnet"
 	"github.com/ipfs/boxo/internal/test"
@@ -17,6 +18,7 @@ import (
 	blocksutil "github.com/ipfs/go-ipfs-blocksutil"
 	delay "github.com/ipfs/go-ipfs-delay"
 	tu "github.com/libp2p/go-libp2p-testing/etc"
+	"github.com/libp2p/go-libp2p/core/peer"
 )
 
 func getVirtualNetwork() tn.Network {
@@ -71,16 +73,32 @@ func TestBasicSessions(t *testing.T) {
 	if !blkout.Cid().Equals(block.Cid()) {
 		t.Fatal("got wrong block")
 	}
+
+	traceBlock, ok := blkout.(traceability.Block)
+	if !ok {
+		t.Fatal("did not get tracable block")
+	}
+
+	if traceBlock.From != b.Peer {
+		t.Fatal("should have received block from peer B, did not")
+	}
 }
 
-func assertBlockLists(got, exp []blocks.Block) error {
+func assertBlockListsFrom(from peer.ID, got, exp []blocks.Block) error {
 	if len(got) != len(exp) {
 		return fmt.Errorf("got wrong number of blocks, %d != %d", len(got), len(exp))
 	}
 
 	h := cid.NewSet()
 	for _, b := range got {
 		h.Add(b.Cid())
+		traceableBlock, ok := b.(traceability.Block)
+		if !ok {
+			return fmt.Errorf("not a traceable block: %s", b.Cid())
+		}
+		if traceableBlock.From != from {
+			return fmt.Errorf("incorrect peer sent block, expect %s, got %s", from, traceableBlock.From)
+		}
 	}
 	for _, b := range exp {
 		if !h.Has(b.Cid()) {
@@ -133,7 +151,7 @@ func TestSessionBetweenPeers(t *testing.T) {
 		for b := range ch {
 			got = append(got, b)
 		}
-		if err := assertBlockLists(got, blks[i*10:(i+1)*10]); err != nil {
+		if err := assertBlockListsFrom(inst[0].Peer, got, blks[i*10:(i+1)*10]); err != nil {
 			t.Fatal(err)
 		}
 	}
@@ -192,7 +210,7 @@ func TestSessionSplitFetch(t *testing.T) {
 		for b := range ch {
 			got = append(got, b)
 		}
-		if err := assertBlockLists(got, blks[i*10:(i+1)*10]); err != nil {
+		if err := assertBlockListsFrom(inst[i].Peer, got, blks[i*10:(i+1)*10]); err != nil {
 			t.Fatal(err)
 		}
 	}
@@ -238,7 +256,7 @@ func TestFetchNotConnected(t *testing.T) {
 	for b := range ch {
 		got = append(got, b)
 	}
-	if err := assertBlockLists(got, blks); err != nil {
+	if err := assertBlockListsFrom(other.Peer, got, blks); err != nil {
 		t.Fatal(err)
 	}
 }
@@ -289,7 +307,7 @@ func TestFetchAfterDisconnect(t *testing.T) {
 		got = append(got, b)
 	}
 
-	if err := assertBlockLists(got, blks[:5]); err != nil {
+	if err := assertBlockListsFrom(peerA.Peer, got, blks[:5]); err != nil {
 		t.Fatal(err)
 	}
 
@@ -318,7 +336,7 @@ func TestFetchAfterDisconnect(t *testing.T) {
 		}
 	}
 
-	if err := assertBlockLists(got, blks); err != nil {
+	if err := assertBlockListsFrom(peerA.Peer, got, blks); err != nil {
 		t.Fatal(err)
 	}
 }

bitswap/client/client.go

@@ -5,14 +5,9 @@ package client
 import (
 	"context"
 	"errors"
-
 	"sync"
 	"time"
 
-	delay "github.com/ipfs/go-ipfs-delay"
-	"go.opentelemetry.io/otel/attribute"
-	"go.opentelemetry.io/otel/trace"
-
 	bsbpm "github.com/ipfs/boxo/bitswap/client/internal/blockpresencemanager"
 	bsgetter "github.com/ipfs/boxo/bitswap/client/internal/getter"
 	bsmq "github.com/ipfs/boxo/bitswap/client/internal/messagequeue"
@@ -33,11 +28,14 @@ import (
 	exchange "github.com/ipfs/boxo/exchange"
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
+	delay "github.com/ipfs/go-ipfs-delay"
 	logging "github.com/ipfs/go-log/v2"
 	"github.com/ipfs/go-metrics-interface"
 	process "github.com/jbenet/goprocess"
 	procctx "github.com/jbenet/goprocess/context"
 	"github.com/libp2p/go-libp2p/core/peer"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 )
 
 var log = logging.Logger("bitswap-client")
@@ -239,6 +237,7 @@ type counters struct {
 
 // GetBlock attempts to retrieve a particular block from peers within the
 // deadline enforced by the context.
+// It returns a [github.com/ipfs/boxo/bitswap/client/traceability.Block] assertable [blocks.Block].
 func (bs *Client) GetBlock(ctx context.Context, k cid.Cid) (blocks.Block, error) {
 	ctx, span := internal.StartSpan(ctx, "GetBlock", trace.WithAttributes(attribute.String("Key", k.String())))
 	defer span.End()
@@ -248,6 +247,7 @@ func (bs *Client) GetBlock(ctx context.Context, k cid.Cid) (blocks.Block, error)
 // GetBlocks returns a channel where the caller may receive blocks that
 // correspond to the provided |keys|. Returns an error if BitSwap is unable to
 // begin this request within the deadline enforced by the context.
+// It returns a [github.com/ipfs/boxo/bitswap/client/traceability.Block] assertable [blocks.Block].
 //
 // NB: Your request remains open until the context expires. To conserve
 // resources, provide a context with a reasonably short deadline (ie. not one
@@ -284,7 +284,8 @@ func (bs *Client) NotifyNewBlocks(ctx context.Context, blks ...blocks.Block) err
 	// Publish the block to any Bitswap clients that had requested blocks.
 	// (the sessions use this pubsub mechanism to inform clients of incoming
 	// blocks)
-	bs.notif.Publish(blks...)
+	var zero peer.ID
+	bs.notif.Publish(zero, blks...)
 
 	return nil
 }
@@ -325,7 +326,7 @@ func (bs *Client) receiveBlocksFrom(ctx context.Context, from peer.ID, blks []bl
 	// (the sessions use this pubsub mechanism to inform clients of incoming
 	// blocks)
 	for _, b := range wanted {
-		bs.notif.Publish(b)
+		bs.notif.Publish(from, b)
 	}
 
 	for _, b := range wanted {

bitswap/client/internal/notifications/notifications.go

@@ -3,10 +3,13 @@ package notifications
 import (
 	"context"
 	"sync"
+	"time"
 
 	pubsub "github.com/cskr/pubsub"
+	"github.com/ipfs/boxo/bitswap/client/traceability"
 	blocks "github.com/ipfs/go-block-format"
 	cid "github.com/ipfs/go-cid"
+	"github.com/libp2p/go-libp2p/core/peer"
 )
 
 const bufferSize = 16
@@ -15,7 +18,7 @@ const bufferSize = 16
 // for cids. It's used internally by bitswap to decouple receiving blocks
 // and actually providing them back to the GetBlocks caller.
 type PubSub interface {
-	Publish(blocks ...blocks.Block)
+	Publish(from peer.ID, blocks ...blocks.Block)
 	Subscribe(ctx context.Context, keys ...cid.Cid) <-chan blocks.Block
 	Shutdown()
 }
@@ -35,7 +38,7 @@ type impl struct {
 	closed chan struct{}
 }
 
-func (ps *impl) Publish(blocks ...blocks.Block) {
+func (ps *impl) Publish(from peer.ID, blocks ...blocks.Block) {
 	ps.lk.RLock()
 	defer ps.lk.RUnlock()
 	select {
@@ -45,7 +48,7 @@ func (ps *impl) Publish(blocks ...blocks.Block) {
 	}
 
 	for _, block := range blocks {
-		ps.wrapped.Pub(block, block.Cid().KeyString())
+		ps.wrapped.Pub(traceability.Block{Block: block, From: from}, block.Cid().KeyString())
 	}
 }
 
@@ -84,6 +87,8 @@ func (ps *impl) Subscribe(ctx context.Context, keys ...cid.Cid) <-chan blocks.Bl
 	default:
 	}
 
+	subscribe := time.Now()
+
 	// AddSubOnceEach listens for each key in the list, and closes the channel
 	// once all keys have been received
 	ps.wrapped.AddSubOnceEach(valuesCh, toStrings(keys)...)
@@ -113,10 +118,13 @@ func (ps *impl) Subscribe(ctx context.Context, keys ...cid.Cid) <-chan blocks.Bl
 				if !ok {
 					return
 				}
-				block, ok := val.(blocks.Block)
+				block, ok := val.(traceability.Block)
 				if !ok {
+					// FIXME: silently dropping errors wtf ?
 					return
 				}
+				block.Delay = time.Since(subscribe)
+
 				select {
 				case <-ctx.Done():
 					return