feat(gateway): support for order=, dups= parameters from IPIP-412

CHANGELOG.md

@@ -16,6 +16,11 @@ The following emojis are used to highlight certain changes:
 
 ### Added
 
+* ✨ The gateway now supports the optional `order` and `dups` CAR parameters
+  from [IPIP-412](https://github.com/ipfs/specs/pull/412). `BlocksBackend` only
+  DFS ordering. If the request explicitly requests an ordering other than `dfs`
+  and `unk`, the request will return an error.
+
 ### Changed
 
 * 🛠 The `ipns` package has been refactored. You should no longer use the direct Protobuf

gateway/blocks_backend.go

@@ -232,7 +232,25 @@ func (bb *BlocksBackend) Head(ctx context.Context, path ImmutablePath) (ContentP
 	return md, fileNode, nil
 }
 
-func (bb *BlocksBackend) GetCAR(ctx context.Context, p ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) {
+func (bb *BlocksBackend) GetCAR(ctx context.Context, p ImmutablePath, params *CarParams) (ContentPathMetadata, io.ReadCloser, error) {
+	// Check if we support the request order. On unknown, change it to DFS. We change
+	// the parameter directly, which means that the caller can use the value to later construct
+	// the Content-Type header.
+	switch params.Order {
+	case DagOrderUnknown:
+		params.Order = DagOrderDFS
+	case DagOrderDFS:
+		// Do nothing
+	default:
+		return ContentPathMetadata{}, nil, fmt.Errorf("unsupported order: %s", params.Order)
+	}
+
+	// Similarly, if params.Duplicates is not set, let's set it to false.
+	if params.Duplicates == nil {
+		v := false
+		params.Duplicates = &v
+	}
+
 	pathMetadata, err := bb.ResolvePath(ctx, p)
 	if err != nil {
 		return ContentPathMetadata{}, nil, err
@@ -245,7 +263,12 @@ func (bb *BlocksBackend) GetCAR(ctx context.Context, p ImmutablePath, params Car
 
 	r, w := io.Pipe()
 	go func() {
-		cw, err := storage.NewWritable(w, []cid.Cid{pathMetadata.LastSegment.Cid()}, car.WriteAsCarV1(true))
+		cw, err := storage.NewWritable(
+			w,
+			[]cid.Cid{pathMetadata.LastSegment.Cid()},
+			car.WriteAsCarV1(true),
+			car.AllowDuplicatePuts(*params.Duplicates),
+		)
 		if err != nil {
 			// io.PipeWriter.CloseWithError always returns nil.
 			_ = w.CloseWithError(err)
@@ -279,7 +302,7 @@ func (bb *BlocksBackend) GetCAR(ctx context.Context, p ImmutablePath, params Car
 }
 
 // walkGatewaySimpleSelector walks the subgraph described by the path and terminal element parameters
-func walkGatewaySimpleSelector(ctx context.Context, p ipfspath.Path, params CarParams, lsys *ipld.LinkSystem, pathResolver resolver.Resolver) error {
+func walkGatewaySimpleSelector(ctx context.Context, p ipfspath.Path, params *CarParams, lsys *ipld.LinkSystem, pathResolver resolver.Resolver) error {
 	// First resolve the path since we always need to.
 	lastCid, remainder, err := pathResolver.ResolveToLastNode(ctx, p)
 	if err != nil {
@@ -312,7 +335,7 @@ func walkGatewaySimpleSelector(ctx context.Context, p ipfspath.Path, params CarP
 				Ctx:                            ctx,
 				LinkSystem:                     *lsys,
 				LinkTargetNodePrototypeChooser: bsfetcher.DefaultPrototypeChooser,
-				LinkVisitOnlyOnce:              true, // This is safe for the "all" selector
+				LinkVisitOnlyOnce:              !*params.Duplicates,
 			},
 		}
 

gateway/gateway.go

@@ -121,8 +121,10 @@ func (i ImmutablePath) IsValid() error {
 var _ path.Path = (*ImmutablePath)(nil)
 
 type CarParams struct {
-	Range *DagByteRange
-	Scope DagScope
+	Range      *DagByteRange
+	Scope      DagScope
+	Order      DagOrder
+	Duplicates *bool
 }
 
 // DagByteRange describes a range request within a UnixFS file. "From" and
@@ -189,6 +191,13 @@ const (
 	DagScopeBlock  DagScope = "block"
 )
 
+type DagOrder string
+
+const (
+	DagOrderDFS     DagOrder = "dfs"
+	DagOrderUnknown DagOrder = "unk"
+)
+
 type ContentPathMetadata struct {
 	PathSegmentRoots []cid.Cid
 	LastSegment      path.Resolved
@@ -278,8 +287,11 @@ type IPFSBackend interface {
 	ResolvePath(context.Context, ImmutablePath) (ContentPathMetadata, error)
 
 	// GetCAR returns a CAR file for the given immutable path. It returns an error
-	// if there was an issue before the CAR streaming begins.
-	GetCAR(context.Context, ImmutablePath, CarParams) (ContentPathMetadata, io.ReadCloser, error)
+	// if there was an issue before the CAR streaming begins. If [CarParams.Duplicates]
+	// is nil, or if [CaraParams.Order] is Unknown, the implementer should change it
+	// such that the caller can form the response "Content-Type" header with the most
+	// amount of information.
+	GetCAR(context.Context, ImmutablePath, *CarParams) (ContentPathMetadata, io.ReadCloser, error)
 
 	// IsCached returns whether or not the path exists locally.
 	IsCached(context.Context, path.Path) bool

gateway/gateway_test.go

@@ -669,7 +669,7 @@ func (mb *errorMockBackend) Head(ctx context.Context, path ImmutablePath) (Conte
 	return ContentPathMetadata{}, nil, mb.err
 }
 
-func (mb *errorMockBackend) GetCAR(ctx context.Context, path ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) {
+func (mb *errorMockBackend) GetCAR(ctx context.Context, path ImmutablePath, params *CarParams) (ContentPathMetadata, io.ReadCloser, error) {
 	return ContentPathMetadata{}, nil, mb.err
 }
 
@@ -753,7 +753,7 @@ func (mb *panicMockBackend) Head(ctx context.Context, immutablePath ImmutablePat
 	panic("i am panicking")
 }
 
-func (mb *panicMockBackend) GetCAR(ctx context.Context, immutablePath ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) {
+func (mb *panicMockBackend) GetCAR(ctx context.Context, immutablePath ImmutablePath, params *CarParams) (ContentPathMetadata, io.ReadCloser, error) {
 	panic("i am panicking")
 }
 

gateway/handler_car.go

@@ -39,7 +39,7 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R
 		return false
 	}
 
-	params, err := getCarParams(r)
+	params, err := getCarParams(r, rq.responseParams)
 	if err != nil {
 		i.webError(w, r, err, http.StatusBadRequest)
 		return false
@@ -90,7 +90,7 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R
 	// sub-DAGs and IPLD selectors: https://github.com/ipfs/go-ipfs/issues/8769
 	w.Header().Set("Accept-Ranges", "none")
 
-	w.Header().Set("Content-Type", carResponseFormat+"; version=1")
+	w.Header().Set("Content-Type", getContentTypeFromCarParams(params))
 	w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^)
 
 	_, copyErr := io.Copy(w, carFile)
@@ -113,7 +113,7 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R
 	return true
 }
 
-func getCarParams(r *http.Request) (CarParams, error) {
+func getCarParams(r *http.Request, formatParams map[string]string) (*CarParams, error) {
 	queryParams := r.URL.Query()
 	rangeStr, hasRange := queryParams.Get(carRangeBytesKey), queryParams.Has(carRangeBytesKey)
 	scopeStr, hasScope := queryParams.Get(carTerminalElementTypeKey), queryParams.Has(carTerminalElementTypeKey)
@@ -123,7 +123,7 @@ func getCarParams(r *http.Request) (CarParams, error) {
 		rng, err := NewDagByteRange(rangeStr)
 		if err != nil {
 			err = fmt.Errorf("invalid entity-bytes: %w", err)
-			return CarParams{}, err
+			return nil, err
 		}
 		params.Range = &rng
 	}
@@ -134,13 +134,58 @@ func getCarParams(r *http.Request) (CarParams, error) {
 			params.Scope = s
 		default:
 			err := fmt.Errorf("unsupported dag-scope %s", scopeStr)
-			return CarParams{}, err
+			return nil, err
 		}
 	} else {
 		params.Scope = DagScopeAll
 	}
 
-	return params, nil
+	switch order := DagOrder(formatParams["order"]); order {
+	case DagOrderUnknown, DagOrderDFS:
+		params.Order = order
+	case "":
+		params.Order = DagOrderUnknown
+	default:
+		return nil, fmt.Errorf("unsupported order %s", order)
+	}
+
+	switch dups := formatParams["dups"]; dups {
+	case "y":
+		v := true
+		params.Duplicates = &v
+	case "n":
+		v := false
+		params.Duplicates = &v
+	case "":
+		// Acceptable, we do not set anything.
+	default:
+		return nil, fmt.Errorf("unsupported dups %s", dups)
+	}
+
+	return &params, nil
+}
+
+func getContentTypeFromCarParams(params *CarParams) string {
+	h := strings.Builder{}
+	h.WriteString(carResponseFormat)
+	h.WriteString("; version=1; order=")
+
+	if params.Order != "" {
+		h.WriteString(string(params.Order))
+	} else {
+		h.WriteString(string(DagOrderUnknown))
+	}
+
+	if params.Duplicates != nil {
+		h.WriteString("; dups=")
+		if *params.Duplicates {
+			h.WriteString("y")
+		} else {
+			h.WriteString("n")
+		}
+	}
+
+	return h.String()
 }
 
 func getCarRootCidAndLastSegment(imPath ImmutablePath) (cid.Cid, string, error) {
@@ -164,7 +209,7 @@ func getCarRootCidAndLastSegment(imPath ImmutablePath) (cid.Cid, string, error)
 	return rootCid, lastSegment, err
 }
 
-func getCarEtag(imPath ImmutablePath, params CarParams, rootCid cid.Cid) string {
+func getCarEtag(imPath ImmutablePath, params *CarParams, rootCid cid.Cid) string {
 	data := imPath.String()
 	if params.Scope != DagScopeAll {
 		data += "." + string(params.Scope)

gateway/handler_car_test.go

@@ -28,7 +28,7 @@ func TestCarParams(t *testing.T) {
 		}
 		for _, test := range tests {
 			r := mustNewRequest(t, http.MethodGet, "http://example.com/?"+test.query, nil)
-			params, err := getCarParams(r)
+			params, err := getCarParams(r, map[string]string{})
 			if test.expectedError {
 				assert.Error(t, err)
 			} else {
@@ -60,7 +60,7 @@ func TestCarParams(t *testing.T) {
 		}
 		for _, test := range tests {
 			r := mustNewRequest(t, http.MethodGet, "http://example.com/?"+test.query, nil)
-			params, err := getCarParams(r)
+			params, err := getCarParams(r, map[string]string{})
 			if test.hasError {
 				assert.Error(t, err)
 			} else {
@@ -73,6 +73,67 @@ func TestCarParams(t *testing.T) {
 			}
 		}
 	})
+
+	t.Run("order and duplicates parsing", func(t *testing.T) {
+		t.Parallel()
+
+		T := true
+		F := false
+
+		tests := []struct {
+			acceptHeader       string
+			expectedOrder      DagOrder
+			expectedDuplicates *bool
+		}{
+			{"application/vnd.ipld.car; order=dfs; dups=y", DagOrderDFS, &T},
+			{"application/vnd.ipld.car; order=unk; dups=n", DagOrderUnknown, &F},
+			{"application/vnd.ipld.car; order=unk", DagOrderUnknown, nil},
+			{"application/vnd.ipld.car; dups=y", DagOrderUnknown, &T},
+			{"application/vnd.ipld.car; dups=n", DagOrderUnknown, &F},
+			{"application/vnd.ipld.car", DagOrderUnknown, nil},
+		}
+		for _, test := range tests {
+			r := mustNewRequest(t, http.MethodGet, "http://example.com/", nil)
+			r.Header.Set("Accept", test.acceptHeader)
+
+			mediaType, formatParams, err := customResponseFormat(r)
+			assert.NoError(t, err)
+			assert.Equal(t, carResponseFormat, mediaType)
+
+			params, err := getCarParams(r, formatParams)
+			assert.NoError(t, err)
+			require.Equal(t, test.expectedOrder, params.Order)
+
+			if test.expectedDuplicates == nil {
+				require.Nil(t, params.Duplicates)
+			} else {
+				require.Equal(t, *test.expectedDuplicates, *params.Duplicates)
+			}
+		}
+	})
+}
+
+func TestContentTypeFromCarParams(t *testing.T) {
+	t.Parallel()
+
+	T := true
+	F := false
+
+	tests := []struct {
+		params CarParams
+		header string
+	}{
+		{CarParams{}, "application/vnd.ipld.car; version=1; order=unk"},
+		{CarParams{Order: DagOrderDFS, Duplicates: &T}, "application/vnd.ipld.car; version=1; order=dfs; dups=y"},
+		{CarParams{Order: DagOrderUnknown, Duplicates: &T}, "application/vnd.ipld.car; version=1; order=unk; dups=y"},
+		{CarParams{Order: DagOrderUnknown}, "application/vnd.ipld.car; version=1; order=unk"},
+		{CarParams{Duplicates: &T}, "application/vnd.ipld.car; version=1; order=unk; dups=y"},
+		{CarParams{Duplicates: &F}, "application/vnd.ipld.car; version=1; order=unk; dups=n"},
+	}
+	for _, test := range tests {
+		header := getContentTypeFromCarParams(&test.params)
+		assert.Equal(t, test.header, header)
+	}
 }
 
 func TestGetCarEtag(t *testing.T) {
@@ -87,24 +148,24 @@ func TestGetCarEtag(t *testing.T) {
 	t.Run("Etag with entity-bytes=0:* is the same as without query param", func(t *testing.T) {
 		t.Parallel()
 
-		noRange := getCarEtag(imPath, CarParams{}, cid)
-		withRange := getCarEtag(imPath, CarParams{Range: &DagByteRange{From: 0}}, cid)
+		noRange := getCarEtag(imPath, &CarParams{}, cid)
+		withRange := getCarEtag(imPath, &CarParams{Range: &DagByteRange{From: 0}}, cid)
 		require.Equal(t, noRange, withRange)
 	})
 
 	t.Run("Etag with entity-bytes=1:* is different than without query param", func(t *testing.T) {
 		t.Parallel()
 
-		noRange := getCarEtag(imPath, CarParams{}, cid)
-		withRange := getCarEtag(imPath, CarParams{Range: &DagByteRange{From: 1}}, cid)
+		noRange := getCarEtag(imPath, &CarParams{}, cid)
+		withRange := getCarEtag(imPath, &CarParams{Range: &DagByteRange{From: 1}}, cid)
 		require.NotEqual(t, noRange, withRange)
 	})
 
 	t.Run("Etags with different dag-scope are different", func(t *testing.T) {
 		t.Parallel()
 
-		a := getCarEtag(imPath, CarParams{Scope: DagScopeAll}, cid)
-		b := getCarEtag(imPath, CarParams{Scope: DagScopeEntity}, cid)
+		a := getCarEtag(imPath, &CarParams{Scope: DagScopeAll}, cid)
+		b := getCarEtag(imPath, &CarParams{Scope: DagScopeEntity}, cid)
 		require.NotEqual(t, a, b)
 	})
 }

gateway/metrics.go

@@ -120,7 +120,7 @@ func (b *ipfsBackendWithMetrics) ResolvePath(ctx context.Context, path Immutable
 	return md, err
 }
 
-func (b *ipfsBackendWithMetrics) GetCAR(ctx context.Context, path ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) {
+func (b *ipfsBackendWithMetrics) GetCAR(ctx context.Context, path ImmutablePath, params *CarParams) (ContentPathMetadata, io.ReadCloser, error) {
 	begin := time.Now()
 	name := "IPFSBackend.GetCAR"
 	ctx, span := spanTrace(ctx, name, trace.WithAttributes(attribute.String("path", path.String())))

gateway/utilities_test.go

@@ -149,7 +149,7 @@ func (mb *mockBackend) Head(ctx context.Context, immutablePath ImmutablePath) (C
 	return mb.gw.Head(ctx, immutablePath)
 }
 
-func (mb *mockBackend) GetCAR(ctx context.Context, immutablePath ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) {
+func (mb *mockBackend) GetCAR(ctx context.Context, immutablePath ImmutablePath, params *CarParams) (ContentPathMetadata, io.ReadCloser, error) {
 	return mb.gw.GetCAR(ctx, immutablePath, params)
 }