From 1d2b5c45348e5fcfdc2a267545efaf55505d6651 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 7 Sep 2023 14:09:52 -0400 Subject: [PATCH] fix(gateway): fix codec range requests --- gateway/blocks_backend.go | 8 ++++++-- gateway/gateway.go | 3 +++ gateway/handler.go | 2 +- gateway/handler_codec.go | 14 +++++++------- gateway/handler_defaults.go | 9 +++++++-- 5 files changed, 24 insertions(+), 12 deletions(-) diff --git a/gateway/blocks_backend.go b/gateway/blocks_backend.go index d3979b583..fc0685d62 100644 --- a/gateway/blocks_backend.go +++ b/gateway/blocks_backend.go @@ -162,8 +162,10 @@ func (bb *BlocksBackend) Get(ctx context.Context, path ImmutablePath, ranges ... } rootCodec := nd.Cid().Prefix().GetCodec() + // This covers both Raw blocks and terminal IPLD codecs like dag-cbor and dag-json // Note: while only cbor, json, dag-cbor, and dag-json are currently supported by gateways this could change + // Note: For the raw codec we return just the relevant range rather than the entire block if rootCodec != uint64(mc.DagPb) { f := files.NewBytesFile(nd.RawData()) @@ -172,8 +174,10 @@ func (bb *BlocksBackend) Get(ctx context.Context, path ImmutablePath, ranges ... return ContentPathMetadata{}, nil, err } - if err := seekToRangeStart(f, ra); err != nil { - return ContentPathMetadata{}, nil, err + if rootCodec == uint64(mc.Raw) { + if err := seekToRangeStart(f, ra); err != nil { + return ContentPathMetadata{}, nil, err + } } return md, NewGetResponseFromReader(f, fileSize), nil diff --git a/gateway/gateway.go b/gateway/gateway.go index 04a48b519..089dd236c 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -358,6 +358,9 @@ type IPFSBackend interface { // file will still need magic bytes from the very beginning for content // type sniffing). // - A range request for a directory currently holds no semantic meaning. + // - For non-UnixFS (and non-raw data) such as terminal IPLD dag-cbor/json, etc. blocks the returned response + // bytes should be the complete block and returned as an [io.ReadSeekCloser] starting at the beginning of the + // block rather than as an [io.ReadCloser] that starts at the beginning of the range request. // // [HTTP Byte Ranges]: https://httpwg.org/specs/rfc9110.html#rfc.section.14.1.2 Get(context.Context, ImmutablePath, ...ByteRange) (ContentPathMetadata, *GetResponse, error) diff --git a/gateway/handler.go b/gateway/handler.go index fb1f9c7d9..af20e2b6e 100644 --- a/gateway/handler.go +++ b/gateway/handler.go @@ -92,7 +92,7 @@ func NewHandler(c Config, backend IPFSBackend) http.Handler { return newHandlerWithMetrics(&c, backend) } -// serveContent replies to the request using the content in the provided ReadSeeker +// serveContent replies to the request using the content in the provided Reader // and returns the status code written and any error encountered during a write. // It wraps httpServeContent (a close clone of http.ServeContent) which takes care of If-None-Match+Etag, // Content-Length and range requests. diff --git a/gateway/handler_codec.go b/gateway/handler_codec.go index 1aaadc31a..97dfaad0a 100644 --- a/gateway/handler_codec.go +++ b/gateway/handler_codec.go @@ -75,14 +75,10 @@ func (i *handler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http return false } - if !i.seekToStartOfFirstRange(w, r, data) { - return false - } - return i.renderCodec(ctx, w, r, rq, blockSize, data) } -func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, rq *requestData, blockSize int64, blockData io.ReadCloser) bool { +func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, rq *requestData, blockSize int64, blockData io.ReadSeekCloser) bool { resolvedPath := rq.pathMetadata.LastSegment ctx, span := spanTrace(ctx, "Handler.RenderCodec", trace.WithAttributes(attribute.String("path", resolvedPath.String()), attribute.String("requestedContentType", rq.responseFormat))) defer span.End() @@ -239,9 +235,13 @@ func parseNode(blockCid cid.Cid, blockData io.Reader) *assets.ParsedNode { } // serveCodecRaw returns the raw block without any conversion -func (i *handler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *http.Request, blockSize int64, blockData io.ReadCloser, contentPath ipath.Path, modtime, begin time.Time) bool { +func (i *handler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *http.Request, blockSize int64, blockData io.ReadSeekCloser, contentPath ipath.Path, modtime, begin time.Time) bool { // ServeContent will take care of - // If-None-Match+Etag, Content-Length and range requests + // If-None-Match+Etag, Content-Length and setting range request headers after we've already seeked to the start of + // the first range + if !i.seekToStartOfFirstRange(w, r, blockData) { + return false + } _, dataSent, _ := serveContent(w, r, modtime, blockSize, blockData) if dataSent { diff --git a/gateway/handler_defaults.go b/gateway/handler_defaults.go index 04fc71643..22c397ade 100644 --- a/gateway/handler_defaults.go +++ b/gateway/handler_defaults.go @@ -99,14 +99,19 @@ func (i *handler) serveDefaults(ctx context.Context, w http.ResponseWriter, r *h case mc.Json, mc.DagJson, mc.Cbor, mc.DagCbor: rq.logger.Debugw("serving codec", "path", rq.contentPath) var blockSize int64 - var dataToRender io.ReadCloser + var dataToRender io.ReadSeekCloser if headResp != nil { blockSize = headResp.bytesSize dataToRender = nil } else { blockSize = getResp.bytesSize - dataToRender = getResp.bytes + dataAsReadSeekCloser, ok := getResp.bytes.(io.ReadSeekCloser) + if !ok { + i.webError(w, r, fmt.Errorf("expected returned non-UnixFS data to be seekable"), http.StatusInternalServerError) + } + dataToRender = dataAsReadSeekCloser } + return i.renderCodec(r.Context(), w, r, rq, blockSize, dataToRender) default: rq.logger.Debugw("serving unixfs", "path", rq.contentPath)