feat(gateway): support duplicates

gateway/blocks_backend.go

@@ -232,7 +232,7 @@ func (bb *BlocksBackend) Head(ctx context.Context, path ImmutablePath) (ContentP
 	return md, fileNode, nil
 }
 
-func (bb *BlocksBackend) GetCAR(ctx context.Context, p ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) {
+func (bb *BlocksBackend) GetCAR(ctx context.Context, p ImmutablePath, params *CarParams) (ContentPathMetadata, io.ReadCloser, error) {
 	pathMetadata, err := bb.ResolvePath(ctx, p)
 	if err != nil {
 		return ContentPathMetadata{}, nil, err
@@ -243,6 +243,12 @@ func (bb *BlocksBackend) GetCAR(ctx context.Context, p ImmutablePath, params Car
 		return ContentPathMetadata{}, nil, fmt.Errorf("path does not have /ipfs/ prefix")
 	}
 
+	// If params.Duplicates is not set, let's set it to false.
+	if params.Duplicates == nil {
+		v := false
+		params.Duplicates = &v
+	}
+
 	r, w := io.Pipe()
 	go func() {
 		cw, err := storage.NewWritable(w, []cid.Cid{pathMetadata.LastSegment.Cid()}, car.WriteAsCarV1(true))
@@ -279,7 +285,7 @@ func (bb *BlocksBackend) GetCAR(ctx context.Context, p ImmutablePath, params Car
 }
 
 // walkGatewaySimpleSelector walks the subgraph described by the path and terminal element parameters
-func walkGatewaySimpleSelector(ctx context.Context, p ipfspath.Path, params CarParams, lsys *ipld.LinkSystem, pathResolver resolver.Resolver) error {
+func walkGatewaySimpleSelector(ctx context.Context, p ipfspath.Path, params *CarParams, lsys *ipld.LinkSystem, pathResolver resolver.Resolver) error {
 	// First resolve the path since we always need to.
 	lastCid, remainder, err := pathResolver.ResolveToLastNode(ctx, p)
 	if err != nil {
@@ -312,7 +318,7 @@ func walkGatewaySimpleSelector(ctx context.Context, p ipfspath.Path, params CarP
 				Ctx:                            ctx,
 				LinkSystem:                     *lsys,
 				LinkTargetNodePrototypeChooser: bsfetcher.DefaultPrototypeChooser,
-				LinkVisitOnlyOnce:              true, // This is safe for the "all" selector
+				LinkVisitOnlyOnce:              !*params.Duplicates,
 			},
 		}
 

gateway/gateway.go

@@ -288,7 +288,7 @@ type IPFSBackend interface {
 
 	// GetCAR returns a CAR file for the given immutable path. It returns an error
 	// if there was an issue before the CAR streaming begins.
-	GetCAR(context.Context, ImmutablePath, CarParams) (ContentPathMetadata, io.ReadCloser, error)
+	GetCAR(context.Context, ImmutablePath, *CarParams) (ContentPathMetadata, io.ReadCloser, error)
 
 	// IsCached returns whether or not the path exists locally.
 	IsCached(context.Context, path.Path) bool

gateway/gateway_test.go

@@ -556,7 +556,7 @@ func (mb *errorMockBackend) Head(ctx context.Context, path ImmutablePath) (Conte
 	return ContentPathMetadata{}, nil, mb.err
 }
 
-func (mb *errorMockBackend) GetCAR(ctx context.Context, path ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) {
+func (mb *errorMockBackend) GetCAR(ctx context.Context, path ImmutablePath, params *CarParams) (ContentPathMetadata, io.ReadCloser, error) {
 	return ContentPathMetadata{}, nil, mb.err
 }
 
@@ -640,7 +640,7 @@ func (mb *panicMockBackend) Head(ctx context.Context, immutablePath ImmutablePat
 	panic("i am panicking")
 }
 
-func (mb *panicMockBackend) GetCAR(ctx context.Context, immutablePath ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) {
+func (mb *panicMockBackend) GetCAR(ctx context.Context, immutablePath ImmutablePath, params *CarParams) (ContentPathMetadata, io.ReadCloser, error) {
 	panic("i am panicking")
 }
 

gateway/handler_car.go

@@ -68,7 +68,7 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R
 	addCacheControlHeaders(w, r, rq.contentPath, rootCid, carResponseFormat)
 
 	// Generate the CAR Etag.
-	etag := getCarEtag(rq.immutablePath, params, rootCid)
+	etag := getCarEtag(rq.immutablePath, *params, rootCid)
 	w.Header().Set("Etag", etag)
 
 	// Terminate early if Etag matches. We cannot rely on handleIfNoneMatch since
@@ -90,7 +90,7 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R
 	// sub-DAGs and IPLD selectors: https://github.com/ipfs/go-ipfs/issues/8769
 	w.Header().Set("Accept-Ranges", "none")
 
-	w.Header().Set("Content-Type", getContentTypeFromCarParams(params))
+	w.Header().Set("Content-Type", getContentTypeFromCarParams(*params))
 	w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^)
 
 	_, copyErr := io.Copy(w, carFile)
@@ -136,7 +136,7 @@ func getContentTypeFromCarParams(params CarParams) string {
 	return h.String()
 }
 
-func getCarParams(r *http.Request, formatParams map[string]string) (CarParams, error) {
+func getCarParams(r *http.Request, formatParams map[string]string) (*CarParams, error) {
 	queryParams := r.URL.Query()
 	rangeStr, hasRange := queryParams.Get(carRangeBytesKey), queryParams.Has(carRangeBytesKey)
 	scopeStr, hasScope := queryParams.Get(carTerminalElementTypeKey), queryParams.Has(carTerminalElementTypeKey)
@@ -146,7 +146,7 @@ func getCarParams(r *http.Request, formatParams map[string]string) (CarParams, e
 		rng, err := NewDagByteRange(rangeStr)
 		if err != nil {
 			err = fmt.Errorf("invalid entity-bytes: %w", err)
-			return CarParams{}, err
+			return nil, err
 		}
 		params.Range = &rng
 	}
@@ -157,7 +157,7 @@ func getCarParams(r *http.Request, formatParams map[string]string) (CarParams, e
 			params.Scope = s
 		default:
 			err := fmt.Errorf("unsupported dag-scope %s", scopeStr)
-			return CarParams{}, err
+			return nil, err
 		}
 	} else {
 		params.Scope = DagScopeAll
@@ -169,7 +169,7 @@ func getCarParams(r *http.Request, formatParams map[string]string) (CarParams, e
 	case "":
 		params.Order = DagOrderUnknown
 	default:
-		return CarParams{}, fmt.Errorf("unsupported order %s", order)
+		return nil, fmt.Errorf("unsupported order %s", order)
 	}
 
 	switch dups := formatParams["dups"]; dups {
@@ -182,10 +182,10 @@ func getCarParams(r *http.Request, formatParams map[string]string) (CarParams, e
 	case "":
 		// Acceptable, we do not set anything.
 	default:
-		return CarParams{}, fmt.Errorf("unsupported dups %s", dups)
+		return nil, fmt.Errorf("unsupported dups %s", dups)
 	}
 
-	return params, nil
+	return &params, nil
 }
 
 func getCarRootCidAndLastSegment(imPath ImmutablePath) (cid.Cid, string, error) {

gateway/metrics.go

@@ -120,7 +120,7 @@ func (b *ipfsBackendWithMetrics) ResolvePath(ctx context.Context, path Immutable
 	return md, err
 }
 
-func (b *ipfsBackendWithMetrics) GetCAR(ctx context.Context, path ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) {
+func (b *ipfsBackendWithMetrics) GetCAR(ctx context.Context, path ImmutablePath, params *CarParams) (ContentPathMetadata, io.ReadCloser, error) {
 	begin := time.Now()
 	name := "IPFSBackend.GetCAR"
 	ctx, span := spanTrace(ctx, name, trace.WithAttributes(attribute.String("path", path.String())))

gateway/utilities_test.go

@@ -148,7 +148,7 @@ func (mb *mockBackend) Head(ctx context.Context, immutablePath ImmutablePath) (C
 	return mb.gw.Head(ctx, immutablePath)
 }
 
-func (mb *mockBackend) GetCAR(ctx context.Context, immutablePath ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) {
+func (mb *mockBackend) GetCAR(ctx context.Context, immutablePath ImmutablePath, params *CarParams) (ContentPathMetadata, io.ReadCloser, error) {
 	return mb.gw.GetCAR(ctx, immutablePath, params)
 }