From 17cd5dd2d605b58222cd6cde36febce493caf3df Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 4 Sep 2023 21:03:51 +0200 Subject: [PATCH] refactor: futher cleanup --- CHANGELOG.md | 18 ++- gateway/blocks_backend.go | 36 +++--- gateway/gateway.go | 5 +- gateway/gateway_test.go | 79 +++++++++++--- gateway/handler.go | 35 +++--- gateway/handler_block.go | 2 +- gateway/handler_car.go | 2 +- gateway/handler_codec.go | 2 +- gateway/handler_defaults.go | 4 +- gateway/handler_tar.go | 2 +- gateway/handler_unixfs__redirects.go | 2 +- gateway/handler_unixfs_dir.go | 4 +- gateway/handler_unixfs_dir_test.go | 2 +- gateway/handler_unixfs_file.go | 4 +- gateway/hostname_test.go | 4 +- gateway/metrics.go | 6 +- gateway/utilities_test.go | 22 +++- namesys/dns_resolver.go | 2 + namesys/dns_resolver_test.go | 157 +++++++++++++++------------ namesys/interface.go | 8 +- namesys/ipns_publisher.go | 73 ++++++------- namesys/ipns_publisher_test.go | 123 ++++++++++----------- namesys/ipns_resolver.go | 58 +++++++--- namesys/ipns_resolver_test.go | 4 +- namesys/namesys.go | 108 +++++++----------- namesys/namesys_cache.go | 14 ++- namesys/namesys_test.go | 48 +++++--- namesys/republisher/repub.go | 8 +- namesys/republisher/repub_test.go | 8 +- namesys/resolve/resolve.go | 63 ----------- namesys/utilities.go | 31 +++--- 31 files changed, 477 insertions(+), 457 deletions(-) delete mode 100644 namesys/resolve/resolve.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b2ace16d..fdc43e802 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ The following emojis are used to highlight certain changes: ### Added * ✨ The `routing/http` implements Delegated Peer Routing introduced in [IPIP-417](https://github.com/ipfs/specs/pull/417). +* The gateway now sets a `Cache-Control` header for requests under the `/ipns/` namespace + if the TTL for the corresponding IPNS Records or DNSLink entities is known. ### Changed @@ -31,7 +33,19 @@ The following emojis are used to highlight certain changes: condensed the different path-related packages under a single one. Therefore, there are many breaking changes. Please consult the [documentation](https://pkg.go.dev/github.com/ipfs/boxo/path) for more details on how to use the new package. - +* 🛠 The `namesys` package has been refactored. The following are the largest modifications: + * The options in `coreiface/options/namesys` have been moved to `namesys` and their names + have been made more consistent. + * Many of the exported structs and functions have been renamed in order to be consistent with + the remaining packages. + * `namesys.Resolver.Resolve` now returns a TTL, in addition to the resolved path. If the + TTL is unknown, 0 is returned. `IPNSResolver` is able to resolve a TTL, while `DNSResolver` + is not. + * `namesys/resolver.ResolveIPNS` has been moved to `namesys.ResolveIPNS` and now returns a TTL + in addition to the resolved path. +* 🛠 The `gateway`'s `IPFSBackend.ResolveMutable` is now expected to return a TTL in addition to + the resolved path. If the TTL is unknown, 0 should be returned. + ### Removed * 🛠 The `routing/http` package experienced following removals: @@ -39,6 +53,8 @@ The following emojis are used to highlight certain changes: `ProvideBitswap` is still usable, but marked as deprecated. A protocol-agnostic provide mechanism is being worked on in [IPIP-378](https://github.com/ipfs/specs/pull/378). * Server no longer exports `FindProvidersPath` and `ProvidePath`. +* 🛠 The `coreiface/options/namesys` package has been removed. +* 🛠 The `namesys.StartSpan` function is no longer exported. ### Fixed diff --git a/gateway/blocks_backend.go b/gateway/blocks_backend.go index 3a673dad6..15437e75d 100644 --- a/gateway/blocks_backend.go +++ b/gateway/blocks_backend.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "strings" + "time" "github.com/ipfs/boxo/blockservice" blockstore "github.com/ipfs/boxo/blockstore" @@ -17,8 +18,8 @@ import ( "github.com/ipfs/boxo/ipld/merkledag" ufile "github.com/ipfs/boxo/ipld/unixfs/file" uio "github.com/ipfs/boxo/ipld/unixfs/io" + "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/namesys" - "github.com/ipfs/boxo/namesys/resolve" "github.com/ipfs/boxo/path" "github.com/ipfs/boxo/path/resolver" blocks "github.com/ipfs/go-block-format" @@ -38,7 +39,6 @@ import ( "github.com/ipld/go-ipld-prime/traversal/selector" selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" - "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" mc "github.com/multiformats/go-multicodec" @@ -556,18 +556,23 @@ func (bb *BlocksBackend) getPathRoots(ctx context.Context, contentPath path.Immu return pathRoots, lastPath, nil } -func (bb *BlocksBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, error) { +func (bb *BlocksBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, time.Duration, error) { switch p.Namespace() { case path.IPNSNamespace: - p, err := resolve.ResolveIPNS(ctx, bb.namesys, p) + p, ttl, err := namesys.ResolveIPNS(ctx, bb.namesys, p) if err != nil { - return nil, err + return nil, 0, err + } + ip, err := path.NewImmutablePath(p) + if err != nil { + return nil, 0, err } - return path.NewImmutablePath(p) + return ip, ttl, nil case path.IPFSNamespace: - return path.NewImmutablePath(p) + ip, err := path.NewImmutablePath(p) + return ip, 0, err default: - return nil, NewErrorStatusCode(fmt.Errorf("unsupported path namespace: %s", p.Namespace()), http.StatusNotImplemented) + return nil, 0, NewErrorStatusCode(fmt.Errorf("unsupported path namespace: %s", p.Namespace()), http.StatusNotImplemented) } } @@ -576,19 +581,12 @@ func (bb *BlocksBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, return nil, NewErrorStatusCode(errors.New("IPNS Record responses are not supported by this gateway"), http.StatusNotImplemented) } - // Fails fast if the CID is not an encoded Libp2p Key, avoids wasteful - // round trips to the remote routing provider. - if mc.Code(c.Type()) != mc.Libp2pKey { - return nil, NewErrorStatusCode(errors.New("cid codec must be libp2p-key"), http.StatusBadRequest) - } - - // The value store expects the key itself to be encoded as a multihash. - id, err := peer.FromCid(c) + name, err := ipns.NameFromCid(c) if err != nil { - return nil, err + return nil, NewErrorStatusCode(err, http.StatusBadRequest) } - return bb.routing.GetValue(ctx, "/ipns/"+string(id)) + return bb.routing.GetValue(ctx, string(name.RoutingKey())) } func (bb *BlocksBackend) GetDNSLinkRecord(ctx context.Context, hostname string) (path.Path, error) { @@ -628,7 +626,7 @@ func (bb *BlocksBackend) ResolvePath(ctx context.Context, path path.ImmutablePat func (bb *BlocksBackend) resolvePath(ctx context.Context, p path.Path) (path.ImmutablePath, error) { var err error if p.Namespace() == path.IPNSNamespace { - p, err = resolve.ResolveIPNS(ctx, bb.namesys, p) + p, _, err = namesys.ResolveIPNS(ctx, bb.namesys, p) if err != nil { return nil, err } diff --git a/gateway/gateway.go b/gateway/gateway.go index 3be6c10f3..1664fa1e7 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -8,6 +8,7 @@ import ( "sort" "strconv" "strings" + "time" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/gateway/assets" @@ -303,11 +304,11 @@ type IPFSBackend interface { GetIPNSRecord(context.Context, cid.Cid) ([]byte, error) // ResolveMutable takes a mutable path and resolves it into an immutable one. This means recursively resolving any - // DNSLink or IPNS records. + // DNSLink or IPNS records. It should also return a TTL. If the TTL is unknown, 0 should be returned. // // For example, given a mapping from `/ipns/dnslink.tld -> /ipns/ipns-id/mydirectory` and `/ipns/ipns-id` to // `/ipfs/some-cid`, the result of passing `/ipns/dnslink.tld/myfile` would be `/ipfs/some-cid/mydirectory/myfile`. - ResolveMutable(context.Context, path.Path) (path.ImmutablePath, error) + ResolveMutable(context.Context, path.Path) (path.ImmutablePath, time.Duration, error) // GetDNSLinkRecord returns the DNSLink TXT record for the provided FQDN. // Unlike ResolvePath, it does not perform recursive resolution. It only diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index 785c338ca..4e112f509 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -37,11 +37,11 @@ func TestGatewayGet(t *testing.T) { return p } - backend.namesys["/ipns/example.com"] = path.NewIPFSPath(k.Cid()) - backend.namesys["/ipns/working.example.com"] = k - backend.namesys["/ipns/double.example.com"] = mustMakeDNSLinkPath("working.example.com") - backend.namesys["/ipns/triple.example.com"] = mustMakeDNSLinkPath("double.example.com") - backend.namesys["/ipns/broken.example.com"] = mustMakeDNSLinkPath(k.Cid().String()) + backend.namesys["/ipns/example.com"] = newMockNamesysItem(path.NewIPFSPath(k.Cid()), 0) + backend.namesys["/ipns/working.example.com"] = newMockNamesysItem(k, 0) + backend.namesys["/ipns/double.example.com"] = newMockNamesysItem(mustMakeDNSLinkPath("working.example.com"), 0) + backend.namesys["/ipns/triple.example.com"] = newMockNamesysItem(mustMakeDNSLinkPath("double.example.com"), 0) + backend.namesys["/ipns/broken.example.com"] = newMockNamesysItem(mustMakeDNSLinkPath(k.Cid().String()), 0) // We picked .man because: // 1. It's a valid TLD. // 2. Go treats it as the file extension for "man" files (even though @@ -49,7 +49,7 @@ func TestGatewayGet(t *testing.T) { // // Unfortunately, this may not work on all platforms as file type // detection is platform dependent. - backend.namesys["/ipns/example.man"] = k + backend.namesys["/ipns/example.man"] = newMockNamesysItem(k, 0) for _, test := range []struct { host string @@ -98,7 +98,7 @@ func TestPretty404(t *testing.T) { t.Logf("test server url: %s", ts.URL) host := "example.net" - backend.namesys["/ipns/"+host] = path.NewIPFSPath(root) + backend.namesys["/ipns/"+host] = newMockNamesysItem(path.NewIPFSPath(root), 0) for _, test := range []struct { path string @@ -158,7 +158,56 @@ func TestHeaders(t *testing.T) { dagCborRoots = dirRoots + "," + dagCborCID ) - t.Run("Cache-Control is not immutable on generated /ipfs/ HTML dir listings", func(t *testing.T) { + t.Run("Cache-Control uses TTL for /ipns/ when it is known", func(t *testing.T) { + t.Parallel() + + ts, backend, root := newTestServerAndNode(t, nil, "ipns-hostname-redirects.car") + backend.namesys["/ipns/example.net"] = newMockNamesysItem(path.NewIPFSPath(root), time.Second*30) + + t.Run("UnixFS generated directory listing without index.html has no Cache-Control", func(t *testing.T) { + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipns/example.net/", nil) + res := mustDoWithoutRedirect(t, req) + require.Empty(t, res.Header["Cache-Control"]) + }) + + t.Run("UnixFS directory with index.html has Cache-Control", func(t *testing.T) { + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipns/example.net/foo/", nil) + res := mustDoWithoutRedirect(t, req) + require.Equal(t, "public, max-age=30", res.Header.Get("Cache-Control")) + }) + + t.Run("UnixFS file has Cache-Control", func(t *testing.T) { + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipns/example.net/foo/index.html", nil) + res := mustDoWithoutRedirect(t, req) + require.Equal(t, "public, max-age=30", res.Header.Get("Cache-Control")) + }) + + t.Run("Raw block has Cache-Control", func(t *testing.T) { + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipns/example.net?format=raw", nil) + res := mustDoWithoutRedirect(t, req) + require.Equal(t, "public, max-age=30", res.Header.Get("Cache-Control")) + }) + + t.Run("DAG-JSON block has Cache-Control", func(t *testing.T) { + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipns/example.net?format=dag-json", nil) + res := mustDoWithoutRedirect(t, req) + require.Equal(t, "public, max-age=30", res.Header.Get("Cache-Control")) + }) + + t.Run("DAG-CBOR block has Cache-Control", func(t *testing.T) { + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipns/example.net?format=dag-cbor", nil) + res := mustDoWithoutRedirect(t, req) + require.Equal(t, "public, max-age=30", res.Header.Get("Cache-Control")) + }) + + t.Run("CAR block has Cache-Control", func(t *testing.T) { + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipns/example.net?format=car", nil) + res := mustDoWithoutRedirect(t, req) + require.Equal(t, "public, max-age=30", res.Header.Get("Cache-Control")) + }) + }) + + t.Run("Cache-Control is not immutable on generated /ipfs/ HTML dir listings", func(t *testing.T) { req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+rootCID+"/", nil) res := mustDoWithoutRedirect(t, req) @@ -500,7 +549,7 @@ func TestRedirects(t *testing.T) { t.Parallel() ts, backend, root := newTestServerAndNode(t, nil, "ipns-hostname-redirects.car") - backend.namesys["/ipns/example.net"] = path.NewIPFSPath(root) + backend.namesys["/ipns/example.net"] = newMockNamesysItem(path.NewIPFSPath(root), 0) // make request to directory containing index.html req := mustNewRequest(t, http.MethodGet, ts.URL+"/foo", nil) @@ -535,7 +584,7 @@ func TestRedirects(t *testing.T) { t.Parallel() backend, root := newMockBackend(t, "redirects-spa.car") - backend.namesys["/ipns/example.com"] = path.NewIPFSPath(root) + backend.namesys["/ipns/example.com"] = newMockNamesysItem(path.NewIPFSPath(root), 0) ts := newTestServerWithConfig(t, backend, Config{ Headers: map[string][]string{}, @@ -672,8 +721,8 @@ func TestDeserializedResponses(t *testing.T) { t.Parallel() backend, root := newMockBackend(t, "fixtures.car") - backend.namesys["/ipns/trustless.com"] = path.NewIPFSPath(root) - backend.namesys["/ipns/trusted.com"] = path.NewIPFSPath(root) + backend.namesys["/ipns/trustless.com"] = newMockNamesysItem(path.NewIPFSPath(root), 0) + backend.namesys["/ipns/trusted.com"] = newMockNamesysItem(path.NewIPFSPath(root), 0) ts := newTestServerWithConfig(t, backend, Config{ Headers: map[string][]string{}, @@ -735,8 +784,8 @@ func (mb *errorMockBackend) GetCAR(ctx context.Context, path path.ImmutablePath, return ContentPathMetadata{}, nil, mb.err } -func (mb *errorMockBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, error) { - return nil, mb.err +func (mb *errorMockBackend) ResolveMutable(ctx context.Context, path path.Path) (path.ImmutablePath, time.Duration, error) { + return nil, 0, mb.err } func (mb *errorMockBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { @@ -819,7 +868,7 @@ func (mb *panicMockBackend) GetCAR(ctx context.Context, immutablePath path.Immut panic("i am panicking") } -func (mb *panicMockBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, error) { +func (mb *panicMockBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, time.Duration, error) { panic("i am panicking") } diff --git a/gateway/handler.go b/gateway/handler.go index c3c19250a..82796daf9 100644 --- a/gateway/handler.go +++ b/gateway/handler.go @@ -34,7 +34,7 @@ var log = logging.Logger("boxo/gateway") const ( ipfsPathPrefix = "/ipfs/" - ipnsPathPrefix = "/ipns/" + ipnsPathPrefix = ipns.NamespacePrefix immutableCacheControl = "public, max-age=29030400, immutable" ) @@ -188,6 +188,7 @@ type requestData struct { // Defined for non IPNS Record requests. immutablePath path.ImmutablePath + ttl time.Duration // Defined if resolution has already happened. pathMetadata *ContentPathMetadata @@ -279,7 +280,7 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) { } if contentPath.Namespace().Mutable() { - rq.immutablePath, err = i.backend.ResolveMutable(r.Context(), contentPath) + rq.immutablePath, rq.ttl, err = i.backend.ResolveMutable(r.Context(), contentPath) if err != nil { err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err) i.webError(w, r, err, http.StatusInternalServerError) @@ -409,32 +410,30 @@ func panicHandler(w http.ResponseWriter) { } } -func addCacheControlHeaders(w http.ResponseWriter, r *http.Request, contentPath path.Path, cid cid.Cid, responseFormat string) (modtime time.Time) { +func addCacheControlHeaders(w http.ResponseWriter, r *http.Request, contentPath path.Path, ttl time.Duration, cid cid.Cid, responseFormat string) (modtime time.Time) { // Best effort attempt to set an Etag based on the CID and response format. // Setting an ETag is handled separately for CARs and IPNS records. if etag := getEtag(r, cid, responseFormat); etag != "" { w.Header().Set("Etag", etag) } - // Set Cache-Control and Last-Modified based on contentPath properties + // Set Cache-Control and Last-Modified based on contentPath properties. if contentPath.Namespace().Mutable() { - // mutable namespaces such as /ipns/ can't be cached forever - - // For now we set Last-Modified to Now() to leverage caching heuristics built into modern browsers: - // https://github.com/ipfs/kubo/pull/8074#pullrequestreview-645196768 - // but we should not set it to fake values and use Cache-Control based on TTL instead - modtime = time.Now() - - // TODO: set Cache-Control based on TTL of IPNS/DNSLink: https://github.com/ipfs/kubo/issues/1818#issuecomment-1015849462 - // TODO: set Last-Modified based on /ipns/ publishing timestamp? + if ttl > 0 { + // When we know the TTL, set the Cache-Control header and disable Last-Modified. + w.Header().Set("Cache-Control", fmt.Sprintf("public, max-age=%d", int(ttl.Seconds()))) + modtime = noModtime + } else { + // Otherwise, we set Last-Modified to the current time to leverage caching heuristics + // built into modern browsers: https://github.com/ipfs/kubo/pull/8074#pullrequestreview-645196768 + modtime = time.Now() + } } else { - // immutable! CACHE ALL THE THINGS, FOREVER! wolololol w.Header().Set("Cache-Control", immutableCacheControl) + modtime = noModtime // disable Last-Modified - // Set modtime to 'zero time' to disable Last-Modified header (superseded by Cache-Control) - modtime = noModtime - - // TODO: set Last-Modified? - TBD - /ipfs/ modification metadata is present in unixfs 1.5 https://github.com/ipfs/kubo/issues/6920? + // TODO: consider setting Last-Modified if UnixFS V1.5 ever gets released + // with metadata: https://github.com/ipfs/kubo/issues/6920 } return modtime diff --git a/gateway/handler_block.go b/gateway/handler_block.go index 487ddf8b6..d7f59bcd1 100644 --- a/gateway/handler_block.go +++ b/gateway/handler_block.go @@ -34,7 +34,7 @@ func (i *handler) serveRawBlock(ctx context.Context, w http.ResponseWriter, r *h setContentDispositionHeader(w, name, "attachment") // Set remaining headers - modtime := addCacheControlHeaders(w, r, rq.contentPath, blockCid, rawResponseFormat) + modtime := addCacheControlHeaders(w, r, rq.contentPath, rq.ttl, blockCid, rawResponseFormat) w.Header().Set("Content-Type", rawResponseFormat) w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^) diff --git a/gateway/handler_car.go b/gateway/handler_car.go index 56aa42e9d..ae27140f4 100644 --- a/gateway/handler_car.go +++ b/gateway/handler_car.go @@ -57,7 +57,7 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R setContentDispositionHeader(w, name, "attachment") // Set Cache-Control (same logic as for a regular files) - addCacheControlHeaders(w, r, rq.contentPath, rootCid, carResponseFormat) + addCacheControlHeaders(w, r, rq.contentPath, rq.ttl, rootCid, carResponseFormat) // Generate the CAR Etag. etag := getCarEtag(rq.immutablePath, params, rootCid) diff --git a/gateway/handler_codec.go b/gateway/handler_codec.go index df222a7c2..62fdb6934 100644 --- a/gateway/handler_codec.go +++ b/gateway/handler_codec.go @@ -104,7 +104,7 @@ func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *htt } // Set HTTP headers (for caching, etc). Etag will be replaced if handled by serveCodecHTML. - modtime := addCacheControlHeaders(w, r, rq.contentPath, resolvedPath.Cid(), responseContentType) + modtime := addCacheControlHeaders(w, r, rq.contentPath, rq.ttl, resolvedPath.Cid(), responseContentType) name := setCodecContentDisposition(w, r, resolvedPath, responseContentType) w.Header().Set("Content-Type", responseContentType) w.Header().Set("X-Content-Type-Options", "nosniff") diff --git a/gateway/handler_defaults.go b/gateway/handler_defaults.go index de31c1fc1..edfc0f993 100644 --- a/gateway/handler_defaults.go +++ b/gateway/handler_defaults.go @@ -128,13 +128,13 @@ func (i *handler) serveDefaults(ctx context.Context, w http.ResponseWriter, r *h // Handling Unixfs file if bytesResponse != nil { rq.logger.Debugw("serving unixfs file", "path", rq.contentPath) - return i.serveFile(ctx, w, r, resolvedPath, rq.contentPath, bytesResponse, pathMetadata.ContentType, rq.begin) + return i.serveFile(ctx, w, r, resolvedPath, rq.contentPath, rq.ttl, bytesResponse, pathMetadata.ContentType, rq.begin) } // Handling Unixfs directory if directoryMetadata != nil || isDirectoryHeadRequest { rq.logger.Debugw("serving unixfs directory", "path", rq.contentPath) - return i.serveDirectory(ctx, w, r, resolvedPath, rq.contentPath, isDirectoryHeadRequest, directoryMetadata, ranges, rq.begin, rq.logger) + return i.serveDirectory(ctx, w, r, resolvedPath, rq.contentPath, rq.ttl, isDirectoryHeadRequest, directoryMetadata, ranges, rq.begin, rq.logger) } i.webError(w, r, fmt.Errorf("unsupported UnixFS type"), http.StatusInternalServerError) diff --git a/gateway/handler_tar.go b/gateway/handler_tar.go index 529e463da..d2f6fa88d 100644 --- a/gateway/handler_tar.go +++ b/gateway/handler_tar.go @@ -31,7 +31,7 @@ func (i *handler) serveTAR(ctx context.Context, w http.ResponseWriter, r *http.R rootCid := pathMetadata.LastSegment.Cid() // Set Cache-Control and read optional Last-Modified time - modtime := addCacheControlHeaders(w, r, rq.contentPath, rootCid, tarResponseFormat) + modtime := addCacheControlHeaders(w, r, rq.contentPath, rq.ttl, rootCid, tarResponseFormat) // Set Content-Disposition var name string diff --git a/gateway/handler_unixfs__redirects.go b/gateway/handler_unixfs__redirects.go index 5641d39f2..f9ba210a0 100644 --- a/gateway/handler_unixfs__redirects.go +++ b/gateway/handler_unixfs__redirects.go @@ -229,7 +229,7 @@ func (i *handler) serve4xx(w http.ResponseWriter, r *http.Request, content4xxPat logger.Debugf("using _redirects: custom %d file at %q", status, content4xxPath) w.Header().Set("Content-Type", "text/html") w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) - addCacheControlHeaders(w, r, content4xxPath, content4xxCid, "") + addCacheControlHeaders(w, r, content4xxPath, 0, content4xxCid, "") w.WriteHeader(status) _, err = io.CopyN(w, content4xxFile, size) return err diff --git a/gateway/handler_unixfs_dir.go b/gateway/handler_unixfs_dir.go index 86329b61c..86c59850c 100644 --- a/gateway/handler_unixfs_dir.go +++ b/gateway/handler_unixfs_dir.go @@ -22,7 +22,7 @@ import ( // serveDirectory returns the best representation of UnixFS directory // // It will return index.html if present, or generate directory listing otherwise. -func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath path.ImmutablePath, contentPath path.Path, isHeadRequest bool, directoryMetadata *directoryMetadata, ranges []ByteRange, begin time.Time, logger *zap.SugaredLogger) bool { +func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath path.ImmutablePath, contentPath path.Path, ttl time.Duration, isHeadRequest bool, directoryMetadata *directoryMetadata, ranges []ByteRange, begin time.Time, logger *zap.SugaredLogger) bool { ctx, span := spanTrace(ctx, "Handler.ServeDirectory", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) defer span.End() @@ -104,7 +104,7 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * if err == nil { logger.Debugw("serving index.html file", "path", idxPath) // write to request - success := i.serveFile(ctx, w, r, resolvedPath, idxPath, idxFile, "text/html", begin) + success := i.serveFile(ctx, w, r, resolvedPath, idxPath, ttl, idxFile, "text/html", begin) if success { i.unixfsDirIndexGetMetric.WithLabelValues(contentPath.Namespace().String()).Observe(time.Since(begin).Seconds()) } diff --git a/gateway/handler_unixfs_dir_test.go b/gateway/handler_unixfs_dir_test.go index fef101315..c3596fd40 100644 --- a/gateway/handler_unixfs_dir_test.go +++ b/gateway/handler_unixfs_dir_test.go @@ -28,7 +28,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) { k3, err := backend.resolvePathNoRootsReturned(ctx, p3) require.NoError(t, err) - backend.namesys["/ipns/example.net"] = path.NewIPFSPath(root) + backend.namesys["/ipns/example.net"] = newMockNamesysItem(path.NewIPFSPath(root), 0) // make request to directory listing req := mustNewRequest(t, http.MethodGet, ts.URL+"/foo%3F%20%23%3C%27/", nil) diff --git a/gateway/handler_unixfs_file.go b/gateway/handler_unixfs_file.go index 5f5b1c52c..6d4109ea5 100644 --- a/gateway/handler_unixfs_file.go +++ b/gateway/handler_unixfs_file.go @@ -19,12 +19,12 @@ import ( // serveFile returns data behind a file along with HTTP headers based on // the file itself, its CID and the contentPath used for accessing it. -func (i *handler) serveFile(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath path.ImmutablePath, contentPath path.Path, file files.File, fileContentType string, begin time.Time) bool { +func (i *handler) serveFile(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath path.ImmutablePath, contentPath path.Path, ttl time.Duration, file files.File, fileContentType string, begin time.Time) bool { _, span := spanTrace(ctx, "Handler.ServeFile", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) defer span.End() // Set Cache-Control and read optional Last-Modified time - modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid(), "") + modtime := addCacheControlHeaders(w, r, contentPath, ttl, resolvedPath.Cid(), "") // Set Content-Disposition name := addContentDispositionHeader(w, r, contentPath) diff --git a/gateway/hostname_test.go b/gateway/hostname_test.go index 7030c88dc..6f77ae73a 100644 --- a/gateway/hostname_test.go +++ b/gateway/hostname_test.go @@ -20,8 +20,8 @@ func TestToSubdomainURL(t *testing.T) { testCID, err := cid.Decode("bafkqaglimvwgy3zakrsxg5cun5jxkyten5wwc2lokvjeycq") require.NoError(t, err) - backend.namesys["/ipns/dnslink.long-name.example.com"] = path.NewIPFSPath(testCID) - backend.namesys["/ipns/dnslink.too-long.f1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5o.example.com"] = path.NewIPFSPath(testCID) + backend.namesys["/ipns/dnslink.long-name.example.com"] = newMockNamesysItem(path.NewIPFSPath(testCID), 0) + backend.namesys["/ipns/dnslink.too-long.f1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5o.example.com"] = newMockNamesysItem(path.NewIPFSPath(testCID), 0) httpRequest := httptest.NewRequest("GET", "http://127.0.0.1:8080", nil) httpsRequest := httptest.NewRequest("GET", "https://https-request-stub.example.com", nil) httpsProxiedRequest := httptest.NewRequest("GET", "http://proxied-https-request-stub.example.com", nil) diff --git a/gateway/metrics.go b/gateway/metrics.go index 0597d56a0..c48f1f3c9 100644 --- a/gateway/metrics.go +++ b/gateway/metrics.go @@ -155,16 +155,16 @@ func (b *ipfsBackendWithMetrics) GetIPNSRecord(ctx context.Context, cid cid.Cid) return r, err } -func (b *ipfsBackendWithMetrics) ResolveMutable(ctx context.Context, path path.Path) (path.ImmutablePath, error) { +func (b *ipfsBackendWithMetrics) ResolveMutable(ctx context.Context, path path.Path) (path.ImmutablePath, time.Duration, error) { begin := time.Now() name := "IPFSBackend.ResolveMutable" ctx, span := spanTrace(ctx, name, trace.WithAttributes(attribute.String("path", path.String()))) defer span.End() - p, err := b.backend.ResolveMutable(ctx, path) + p, ttl, err := b.backend.ResolveMutable(ctx, path) b.updateBackendCallMetric(name, err, begin) - return p, err + return p, ttl, err } func (b *ipfsBackendWithMetrics) GetDNSLinkRecord(ctx context.Context, fqdn string) (path.Path, error) { diff --git a/gateway/utilities_test.go b/gateway/utilities_test.go index 369c6cf2a..2e8318886 100644 --- a/gateway/utilities_test.go +++ b/gateway/utilities_test.go @@ -51,7 +51,16 @@ func mustDo(t *testing.T, req *http.Request) *http.Response { return res } -type mockNamesys map[string]path.Path +type mockNamesysItem struct { + path path.Path + ttl time.Duration +} + +func newMockNamesysItem(p path.Path, ttl time.Duration) *mockNamesysItem { + return &mockNamesysItem{path: p, ttl: ttl} +} + +type mockNamesys map[string]*mockNamesysItem func (m mockNamesys) Resolve(ctx context.Context, name string, opts ...namesys.ResolveOption) (value path.Path, ttl time.Duration, err error) { cfg := namesys.DefaultResolveOptions() @@ -69,14 +78,15 @@ func (m mockNamesys) Resolve(ctx context.Context, name string, opts ...namesys.R } depth-- - var ok bool - value, ok = m[name] + v, ok := m[name] if !ok { return nil, 0, namesys.ErrResolveFailed } + value = v.path + ttl = v.ttl name = value.String() } - return value, 0, nil + return value, ttl, nil } func (m mockNamesys) ResolveAsync(ctx context.Context, name string, opts ...namesys.ResolveOption) <-chan namesys.ResolveResult { @@ -152,7 +162,7 @@ func (mb *mockBackend) GetCAR(ctx context.Context, immutablePath path.ImmutableP return mb.gw.GetCAR(ctx, immutablePath, params) } -func (mb *mockBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, error) { +func (mb *mockBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, time.Duration, error) { return mb.gw.ResolveMutable(ctx, p) } @@ -184,7 +194,7 @@ func (mb *mockBackend) resolvePathNoRootsReturned(ctx context.Context, ip path.P var imPath path.ImmutablePath var err error if ip.Namespace().Mutable() { - imPath, err = mb.ResolveMutable(ctx, ip) + imPath, _, err = mb.ResolveMutable(ctx, ip) if err != nil { return nil, err } diff --git a/namesys/dns_resolver.go b/namesys/dns_resolver.go index 188c98ea9..63289684f 100644 --- a/namesys/dns_resolver.go +++ b/namesys/dns_resolver.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/ipfs/boxo/ipns" path "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" dns "github.com/miekg/dns" @@ -51,6 +52,7 @@ func (r *DNSResolver) resolveOnceAsync(ctx context.Context, name string, options var fqdn string out := make(chan ResolveResult, 1) + name = strings.TrimPrefix(name, ipns.NamespacePrefix) segments := strings.SplitN(name, "/", 2) domain := segments[0] diff --git a/namesys/dns_resolver_test.go b/namesys/dns_resolver_test.go index 177c7a996..5deb9c2e0 100644 --- a/namesys/dns_resolver_test.go +++ b/namesys/dns_resolver_test.go @@ -8,6 +8,43 @@ import ( "github.com/stretchr/testify/assert" ) +func TestDNSParseEntry(t *testing.T) { + t.Parallel() + + t.Run("Valid entries", func(t *testing.T) { + t.Parallel() + + for _, entry := range []string{ + "QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + "dnslink=/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo", + "dnslink=/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/bar", + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo/bar/baz", + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo/bar/baz/", + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + } { + _, err := parseEntry(entry) + assert.NoError(t, err) + } + }) + + t.Run("Invalid entries", func(t *testing.T) { + t.Parallel() + + for _, entry := range []string{ + "QmYhE8xgFCjGcz6PHgnvJz5NOTCORRECT", + "quux=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + "dnslink=", + "dnslink=/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo", + "dnslink=ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/bar", + } { + _, err := parseEntry(entry) + assert.Error(t, err) + } + }) +} + type mockDNS struct { entries map[string][]string } @@ -20,37 +57,6 @@ func (m *mockDNS) lookupTXT(ctx context.Context, name string) (txt []string, err return txt, nil } -func TestDnsEntryParsing(t *testing.T) { - goodEntries := []string{ - "QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - "dnslink=/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo", - "dnslink=/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/bar", - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo/bar/baz", - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo/bar/baz/", - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - } - - badEntries := []string{ - "QmYhE8xgFCjGcz6PHgnvJz5NOTCORRECT", - "quux=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - "dnslink=", - "dnslink=/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo", - "dnslink=ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/bar", - } - - for _, e := range goodEntries { - _, err := parseEntry(e) - assert.NoError(t, err) - } - - for _, e := range badEntries { - _, err := parseEntry(e) - assert.Error(t, err) - } -} - func newMockDNS() *mockDNS { return &mockDNS{ entries: map[string][]string{ @@ -136,42 +142,55 @@ func newMockDNS() *mockDNS { } func TestDNSResolution(t *testing.T) { - mock := newMockDNS() - r := &DNSResolver{lookupTXT: mock.lookupTXT} - testResolution(t, r, "multihash.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) - testResolution(t, r, "ipfs.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) - testResolution(t, r, "dipfs.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) - testResolution(t, r, "dns1.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) - testResolution(t, r, "dns1.example.com", 1, "/ipns/ipfs.example.com", ErrResolveRecursion) - testResolution(t, r, "dns2.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) - testResolution(t, r, "dns2.example.com", 1, "/ipns/dns1.example.com", ErrResolveRecursion) - testResolution(t, r, "dns2.example.com", 2, "/ipns/ipfs.example.com", ErrResolveRecursion) - testResolution(t, r, "multi.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) - testResolution(t, r, "multi.example.com", 1, "/ipns/dns1.example.com", ErrResolveRecursion) - testResolution(t, r, "multi.example.com", 2, "/ipns/ipfs.example.com", ErrResolveRecursion) - testResolution(t, r, "equals.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/=equals", nil) - testResolution(t, r, "loop1.example.com", 1, "/ipns/loop2.example.com", ErrResolveRecursion) - testResolution(t, r, "loop1.example.com", 2, "/ipns/loop1.example.com", ErrResolveRecursion) - testResolution(t, r, "loop1.example.com", 3, "/ipns/loop2.example.com", ErrResolveRecursion) - testResolution(t, r, "loop1.example.com", DefaultDepthLimit, "/ipns/loop1.example.com", ErrResolveRecursion) - testResolution(t, r, "dloop1.example.com", 1, "/ipns/loop2.example.com", ErrResolveRecursion) - testResolution(t, r, "dloop1.example.com", 2, "/ipns/loop1.example.com", ErrResolveRecursion) - testResolution(t, r, "dloop1.example.com", 3, "/ipns/loop2.example.com", ErrResolveRecursion) - testResolution(t, r, "dloop1.example.com", DefaultDepthLimit, "/ipns/loop1.example.com", ErrResolveRecursion) - testResolution(t, r, "bad.example.com", DefaultDepthLimit, "", ErrResolveFailed) - testResolution(t, r, "withsegment.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment", nil) - testResolution(t, r, "withrecsegment.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/subsub", nil) - testResolution(t, r, "withsegment.example.com/test1", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/test1", nil) - testResolution(t, r, "withrecsegment.example.com/test2", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/subsub/test2", nil) - testResolution(t, r, "withrecsegment.example.com/test3/", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/subsub/test3/", nil) - testResolution(t, r, "withtrailingrec.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/", nil) - testResolution(t, r, "double.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) - testResolution(t, r, "conflict.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjE", nil) - testResolution(t, r, "fqdn.example.com.", DefaultDepthLimit, "/ipfs/QmYvMB9yrsSf7RKBghkfwmHJkzJhW2ZgVwq3LxBXXPasFr", nil) - testResolution(t, r, "en.wikipedia-on-ipfs.org", 2, "/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze", nil) - testResolution(t, r, "custom.non-icann.tldextravaganza.", 2, "/ipfs/bafybeieto6mcuvqlechv4iadoqvnffondeiwxc2bcfcewhvpsd2odvbmvm", nil) - testResolution(t, r, "singlednslabelshouldbeok", 2, "/ipfs/bafybeih4a6ylafdki6ailjrdvmr7o4fbbeceeeuty4v3qyyouiz5koqlpi", nil) - testResolution(t, r, "www.wealdtech.eth", 1, "/ipns/ipfs.example.com", ErrResolveRecursion) - testResolution(t, r, "www.wealdtech.eth", 2, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) - testResolution(t, r, "www.wealdtech.eth", 2, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) + t.Parallel() + r := &DNSResolver{lookupTXT: newMockDNS().lookupTXT} + + for _, testCase := range []struct { + name string + depth uint + expectedPath string + expectedError error + }{ + {"multihash.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"/ipns/multihash.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"ipfs.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"dipfs.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"dns1.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"dns1.example.com", 1, "/ipns/ipfs.example.com", ErrResolveRecursion}, + {"dns2.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"dns2.example.com", 1, "/ipns/dns1.example.com", ErrResolveRecursion}, + {"dns2.example.com", 2, "/ipns/ipfs.example.com", ErrResolveRecursion}, + {"multi.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"multi.example.com", 1, "/ipns/dns1.example.com", ErrResolveRecursion}, + {"multi.example.com", 2, "/ipns/ipfs.example.com", ErrResolveRecursion}, + {"equals.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/=equals", nil}, + {"loop1.example.com", 1, "/ipns/loop2.example.com", ErrResolveRecursion}, + {"loop1.example.com", 2, "/ipns/loop1.example.com", ErrResolveRecursion}, + {"loop1.example.com", 3, "/ipns/loop2.example.com", ErrResolveRecursion}, + {"loop1.example.com", DefaultDepthLimit, "/ipns/loop1.example.com", ErrResolveRecursion}, + {"dloop1.example.com", 1, "/ipns/loop2.example.com", ErrResolveRecursion}, + {"dloop1.example.com", 2, "/ipns/loop1.example.com", ErrResolveRecursion}, + {"dloop1.example.com", 3, "/ipns/loop2.example.com", ErrResolveRecursion}, + {"dloop1.example.com", DefaultDepthLimit, "/ipns/loop1.example.com", ErrResolveRecursion}, + {"bad.example.com", DefaultDepthLimit, "", ErrResolveFailed}, + {"/ipns/bad.example.com", DefaultDepthLimit, "", ErrResolveFailed}, + {"withsegment.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment", nil}, + {"withrecsegment.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/subsub", nil}, + {"withsegment.example.com/test1", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/test1", nil}, + {"withrecsegment.example.com/test2", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/subsub/test2", nil}, + {"withrecsegment.example.com/test3/", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/subsub/test3/", nil}, + {"withtrailingrec.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/", nil}, + {"double.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"conflict.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjE", nil}, + {"fqdn.example.com.", DefaultDepthLimit, "/ipfs/QmYvMB9yrsSf7RKBghkfwmHJkzJhW2ZgVwq3LxBXXPasFr", nil}, + {"en.wikipedia-on-ipfs.org", 2, "/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze", nil}, + {"custom.non-icann.tldextravaganza.", 2, "/ipfs/bafybeieto6mcuvqlechv4iadoqvnffondeiwxc2bcfcewhvpsd2odvbmvm", nil}, + {"singlednslabelshouldbeok", 2, "/ipfs/bafybeih4a6ylafdki6ailjrdvmr7o4fbbeceeeuty4v3qyyouiz5koqlpi", nil}, + {"www.wealdtech.eth", 1, "/ipns/ipfs.example.com", ErrResolveRecursion}, + {"www.wealdtech.eth", 2, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"/ipns/www.wealdtech.eth", 2, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"www.wealdtech.eth", 2, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + } { + testResolution(t, r, testCase.name, (testCase.depth), testCase.expectedPath, 0, testCase.expectedError) + } } diff --git a/namesys/interface.go b/namesys/interface.go index 9133cf9c5..f70eee3ed 100644 --- a/namesys/interface.go +++ b/namesys/interface.go @@ -18,6 +18,9 @@ var ( // ErrResolveRecursion signals a recursion-depth limit. ErrResolveRecursion = errors.New("could not resolve name (recursion limit exceeded)") + + // ErrNoNamesys is an explicit error for when no [NameSystem] is provided. + ErrNoNamesys = errors.New("no namesys has been provided") ) const ( @@ -63,8 +66,9 @@ type ResolveResult struct { // Resolver is an object capable of resolving names. type Resolver interface { - // Resolve performs a recursive lookup, returning the dereferenced path. For example, - // if example.com has a DNS TXT record pointing to: + // Resolve performs a recursive lookup, returning the dereferenced path and the TTL. + // If the TTL is unknown, then a TTL of 0 is returned. For example, if example.com + // has a DNS TXT record pointing to: // // /ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy // diff --git a/namesys/ipns_publisher.go b/namesys/ipns_publisher.go index 48ec2edba..7d7b6f205 100644 --- a/namesys/ipns_publisher.go +++ b/namesys/ipns_publisher.go @@ -2,6 +2,7 @@ package namesys import ( "context" + "errors" "strings" "sync" "time" @@ -50,7 +51,7 @@ func (p *IPNSPublisher) Publish(ctx context.Context, priv crypto.PrivKey, value return err } - return PublishRecord(ctx, p.routing, priv.GetPublic(), record) + return PublishIPNSRecord(ctx, p.routing, priv.GetPublic(), record) } // IpnsDsKey returns a datastore key given an IPNS identifier (peer @@ -155,9 +156,9 @@ func (p *IPNSPublisher) updateRecord(ctx context.Context, k crypto.PrivKey, valu return nil, err } - seqno := uint64(0) + seq := uint64(0) if rec != nil { - seqno, err = rec.Sequence() + seq, err = rec.Sequence() if err != nil { return nil, err } @@ -169,14 +170,14 @@ func (p *IPNSPublisher) updateRecord(ctx context.Context, k crypto.PrivKey, valu if value.String() != p.String() { // Don't bother incrementing the sequence number unless the // value changes. - seqno++ + seq++ } } opts := ProcessPublishOptions(options) // Create record - r, err := ipns.NewRecord(k, value, seqno, opts.EOL, opts.TTL, ipns.WithV1Compatibility(opts.CompatibleWithV1)) + r, err := ipns.NewRecord(k, value, seq, opts.EOL, opts.TTL, ipns.WithV1Compatibility(opts.CompatibleWithV1)) if err != nil { return nil, err } @@ -194,13 +195,14 @@ func (p *IPNSPublisher) updateRecord(ctx context.Context, k crypto.PrivKey, valu if err := p.ds.Sync(ctx, key); err != nil { return nil, err } + return r, nil } -// PublishRecord publishes the given entry using the provided ValueStore, -// keyed on the ID associated with the provided public key. The public key is -// also made available to the routing system so that entries can be verified. -func PublishRecord(ctx context.Context, r routing.ValueStore, k crypto.PubKey, rec *ipns.Record) error { +// PublishIPNSRecord publishes the given [ipns.Record] for the provided [crypto.PubKey] in +// the provided [routing.ValueStore]. The public key is also made available to the routing +// system if it cannot be derived from the corresponding [peer.ID]. +func PublishIPNSRecord(ctx context.Context, r routing.ValueStore, pubKey crypto.PubKey, rec *ipns.Record) error { ctx, span := startSpan(ctx, "PutRecordToRouting") defer span.End() @@ -209,25 +211,22 @@ func PublishRecord(ctx context.Context, r routing.ValueStore, k crypto.PubKey, r errs := make(chan error, 2) // At most two errors (IPNS, and public key) - id, err := peer.IDFromPublicKey(k) + pid, err := peer.IDFromPublicKey(pubKey) if err != nil { return err } go func() { - errs <- PublishIPNSRecord(ctx, r, ipns.NameFromPeer(id), rec) + errs <- PutIPNSRecord(ctx, r, ipns.NameFromPeer(pid), rec) }() - // Publish the public key if a public key cannot be extracted from the ID - // TODO: once v0.4.16 is widespread enough, we can stop doing this - // and at that point we can even deprecate the /pk/ namespace in the dht - // - // NOTE: This check actually checks if the public key has been embedded - // in the IPNS entry. This check is sufficient because we embed the - // public key in the IPNS entry if it can't be extracted from the ID. - if _, err := rec.PubKey(); err == nil { + // Publish the public key if the public key cannot be extracted from the peer ID. + // This is most likely not necessary since IPNS Records include, by default, the public + // key in those cases. However, this ensures it's still possible to easily retrieve + // the public key if, for some reason, it is not embedded. + if _, err := pid.ExtractPublicKey(); errors.Is(err, peer.ErrNoPublicKey) { go func() { - errs <- PublishPublicKey(ctx, r, PkKeyForID(id), k) + errs <- PutPublicKey(ctx, r, pid, pubKey) }() if err := waitOnErrChan(ctx, errs); err != nil { @@ -247,39 +246,37 @@ func waitOnErrChan(ctx context.Context, errs chan error) error { } } -// PublishPublicKey stores the given [crypto.PubKey] for the given key in the [routing.ValueStore]. -func PublishPublicKey(ctx context.Context, r routing.ValueStore, key string, pubKey crypto.PubKey) error { - ctx, span := startSpan(ctx, "PublishPublicKey", trace.WithAttributes(attribute.String("Key", key))) +// PutPublicKey puts the given [crypto.PubKey] for the given [peer.ID] in the [routing.ValueStore]. +func PutPublicKey(ctx context.Context, r routing.ValueStore, pid peer.ID, pubKey crypto.PubKey) error { + routingKey := PkRoutingKey(pid) + ctx, span := startSpan(ctx, "PublishPublicKey", trace.WithAttributes(attribute.String("Key", routingKey))) defer span.End() - log.Debugf("Storing pubkey at: %q", key) bytes, err := crypto.MarshalPublicKey(pubKey) if err != nil { return err } - // Store associated public key - return r.PutValue(ctx, key, bytes) + log.Debugf("Storing public key at: %x", routingKey) + return r.PutValue(ctx, routingKey, bytes) } -// PublishIPNSRecord stores the given [ipns.Record] for the given [ipns.Name] in the given [routing.ValueStore]. -func PublishIPNSRecord(ctx context.Context, r routing.ValueStore, name ipns.Name, rec *ipns.Record) error { - routingKey := string(name.RoutingKey()) +// PkRoutingKey returns the public key routing key for the given [peer.ID]. +func PkRoutingKey(id peer.ID) string { + return "/pk/" + string(id) +} +// PutIPNSRecord puts the given [ipns.Record] for the given [ipns.Name] in the [routing.ValueStore]. +func PutIPNSRecord(ctx context.Context, r routing.ValueStore, name ipns.Name, rec *ipns.Record) error { + routingKey := string(name.RoutingKey()) ctx, span := startSpan(ctx, "PublishEntry", trace.WithAttributes(attribute.String("IPNSKey", routingKey))) defer span.End() - data, err := ipns.MarshalRecord(rec) + bytes, err := ipns.MarshalRecord(rec) if err != nil { return err } - log.Debugf("Storing ipns entry at: %x", routingKey) - // Store ipns entry at "/ipns/"+h(pubkey) - return r.PutValue(ctx, routingKey, data) -} - -// PkKeyForID returns the public key routing key for the given [peer.ID]. -func PkKeyForID(id peer.ID) string { - return "/pk/" + string(id) + log.Debugf("Storing ipns record at: %x", routingKey) + return r.PutValue(ctx, routingKey, bytes) } diff --git a/namesys/ipns_publisher_test.go b/namesys/ipns_publisher_test.go index 927d005fa..b2783e31a 100644 --- a/namesys/ipns_publisher_test.go +++ b/namesys/ipns_publisher_test.go @@ -17,79 +17,66 @@ import ( testutil "github.com/libp2p/go-libp2p-testing/net" ci "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" - ma "github.com/multiformats/go-multiaddr" ) -type identity struct { - testutil.PeerNetParams -} - -func (p *identity) ID() peer.ID { - return p.PeerNetParams.ID -} - -func (p *identity) Address() ma.Multiaddr { - return p.Addr -} - -func (p *identity) PrivateKey() ci.PrivKey { - return p.PrivKey -} - -func (p *identity) PublicKey() ci.PubKey { - return p.PubKey -} - -func testNamekeyPublisher(t *testing.T, keyType int, expectedErr error, expectedExistence bool) { - ctx := context.Background() - - privKey, pubKey, err := ci.GenerateKeyPairWithReader(keyType, 2048, rand.Reader) - require.NoError(t, err) - - id, err := peer.IDFromPublicKey(pubKey) - require.NoError(t, err) - - value, err := path.NewPath("/ipfs/bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4") - require.NoError(t, err) - seq := uint64(0) - eol := time.Now().Add(24 * time.Hour) - - // Routing value store - p := testutil.PeerNetParams{ - ID: id, - PrivKey: privKey, - PubKey: pubKey, - Addr: testutil.ZeroLocalTCPAddress, +func TestIPNSPublisher(t *testing.T) { + t.Parallel() + + test := func(t *testing.T, keyType int, expectedErr error, expectedExistence bool) { + ctx := context.Background() + + // Create test identity + privKey, pubKey, err := ci.GenerateKeyPairWithReader(keyType, 2048, rand.Reader) + require.NoError(t, err) + + pid, err := peer.IDFromPublicKey(pubKey) + require.NoError(t, err) + + // Create IPNS Record + value, err := path.NewPath("/ipfs/bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4") + require.NoError(t, err) + rec, err := ipns.NewRecord(privKey, value, 0, time.Now().Add(24*time.Hour), 0) + require.NoError(t, err) + + // Routing value store + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + serv := mockrouting.NewServer() + r := serv.ClientWithDatastore(context.Background(), testutil.NewIdentity(pid, testutil.ZeroLocalTCPAddress, privKey, pubKey), dstore) + + // Publish IPNS Record + err = PublishIPNSRecord(ctx, r, pubKey, rec) + require.NoError(t, err) + + // Check if IPNS Record is stored in value store + _, err = r.GetValue(ctx, string(ipns.NameFromPeer(pid).RoutingKey())) + require.NoError(t, err) + + key := dshelp.NewKeyFromBinary(ipns.NameFromPeer(pid).RoutingKey()) + exists, err := dstore.Has(ctx, key) + require.NoError(t, err) + require.True(t, exists) + + // Check for Public Key is stored in value store + pkRoutingKey := PkRoutingKey(pid) + _, err = r.GetValue(ctx, pkRoutingKey) + require.ErrorIs(t, err, expectedErr) + + // Check if Public Key is in data store for completeness + key = dshelp.NewKeyFromBinary([]byte(pkRoutingKey)) + exists, err = dstore.Has(ctx, key) + require.NoError(t, err) + require.Equal(t, expectedExistence, exists) } - dstore := dssync.MutexWrap(ds.NewMapDatastore()) - serv := mockrouting.NewServer() - r := serv.ClientWithDatastore(context.Background(), &identity{p}, dstore) - - rec, err := ipns.NewRecord(privKey, value, seq, eol, 0) - require.NoError(t, err) - - err = PublishRecord(ctx, r, pubKey, rec) - require.NoError(t, err) - - // Check for namekey existence in value store - namekey := PkKeyForID(id) - _, err = r.GetValue(ctx, namekey) - require.ErrorIs(t, err, expectedErr) - - // Also check datastore for completeness - key := dshelp.NewKeyFromBinary([]byte(namekey)) - exists, err := dstore.Has(ctx, key) - require.NoError(t, err) - require.Equal(t, expectedExistence, exists) -} - -func TestRSAPublisher(t *testing.T) { - testNamekeyPublisher(t, ci.RSA, nil, true) -} + t.Run("RSA", func(t *testing.T) { + t.Parallel() + test(t, ci.RSA, nil, true) + }) -func TestEd22519Publisher(t *testing.T) { - testNamekeyPublisher(t, ci.Ed25519, ds.ErrNotFound, false) + t.Run("Ed22519", func(t *testing.T) { + t.Parallel() + test(t, ci.Ed25519, ds.ErrNotFound, false) + }) } func TestAsyncDS(t *testing.T) { diff --git a/namesys/ipns_resolver.go b/namesys/ipns_resolver.go index e0dc757ee..91c858395 100644 --- a/namesys/ipns_resolver.go +++ b/namesys/ipns_resolver.go @@ -2,14 +2,12 @@ package namesys import ( "context" - "strings" "time" "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/path" dht "github.com/libp2p/go-libp2p-kad-dht" - "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -47,12 +45,12 @@ func (r *IPNSResolver) ResolveAsync(ctx context.Context, name string, options .. return resolveAsync(ctx, r, name, ProcessResolveOptions(options)) } -func (r *IPNSResolver) resolveOnceAsync(ctx context.Context, name string, options ResolveOptions) <-chan ResolveResult { - ctx, span := startSpan(ctx, "IpnsResolver.ResolveOnceAsync", trace.WithAttributes(attribute.String("Name", name))) +func (r *IPNSResolver) resolveOnceAsync(ctx context.Context, nameStr string, options ResolveOptions) <-chan ResolveResult { + ctx, span := startSpan(ctx, "IpnsResolver.ResolveOnceAsync", trace.WithAttributes(attribute.String("Name", nameStr))) defer span.End() out := make(chan ResolveResult, 1) - log.Debugf("RoutingResolver resolving %s", name) + log.Debugf("RoutingResolver resolving %s", nameStr) cancel := func() {} if options.DhtTimeout != 0 { @@ -60,25 +58,18 @@ func (r *IPNSResolver) resolveOnceAsync(ctx context.Context, name string, option ctx, cancel = context.WithTimeout(ctx, options.DhtTimeout) } - name = strings.TrimPrefix(name, "/ipns/") - - pid, err := peer.Decode(name) + name, err := ipns.NameFromString(nameStr) if err != nil { - log.Debugf("RoutingResolver: could not convert public key hash %s to peer ID: %s\n", name, err) + log.Debugf("RoutingResolver: could not convert key %q to IPNS name: %s\n", nameStr, err) out <- ResolveResult{Err: err} close(out) cancel() return out } - // Use the routing system to get the name. - // Note that the DHT will call the ipns validator when retrieving - // the value, which in turn verifies the ipns record signature - ipnsKey := string(ipns.NameFromPeer(pid).RoutingKey()) - - vals, err := r.routing.SearchValue(ctx, ipnsKey, dht.Quorum(int(options.DhtRecordCount))) + vals, err := r.routing.SearchValue(ctx, string(name.RoutingKey()), dht.Quorum(int(options.DhtRecordCount))) if err != nil { - log.Debugf("RoutingResolver: dht get for name %s failed: %s", name, err) + log.Debugf("RoutingResolver: dht get for name %s failed: %s", nameStr, err) out <- ResolveResult{Err: err} close(out) cancel() @@ -100,7 +91,7 @@ func (r *IPNSResolver) resolveOnceAsync(ctx context.Context, name string, option rec, err := ipns.UnmarshalRecord(val) if err != nil { - log.Debugf("RoutingResolver: could not unmarshal value for name %s: %s", name, err) + log.Debugf("RoutingResolver: could not unmarshal value for name %s: %s", nameStr, err) emitOnceResult(ctx, out, ResolveResult{Err: err}) return } @@ -142,3 +133,36 @@ func (r *IPNSResolver) resolveOnceAsync(ctx context.Context, name string, option return out } + +// ResolveIPNS is an utility that takes a [NameSystem] and a [path.Path] and resolves the IPNS Path. +func ResolveIPNS(ctx context.Context, ns NameSystem, p path.Path) (path.Path, time.Duration, error) { + ctx, span := startSpan(ctx, "ResolveIPNS", trace.WithAttributes(attribute.String("Path", p.String()))) + defer span.End() + + if p.Namespace() != path.IPNSNamespace { + return p, 0, nil + } + + if ns == nil { + return nil, 0, ErrNoNamesys + } + + segments := p.Segments() + + resolvablePath, err := path.NewPathFromSegments(segments[0], segments[1]) + if err != nil { + return nil, 0, err + } + + resolvedPath, ttl, err := ns.Resolve(ctx, resolvablePath.String()) + if err != nil { + return nil, 0, err + } + + p, err = path.Join(resolvedPath, segments[2:]...) + if err != nil { + return nil, 0, err + } + + return p, ttl, nil +} diff --git a/namesys/ipns_resolver_test.go b/namesys/ipns_resolver_test.go index c4cee5b00..13984a6a7 100644 --- a/namesys/ipns_resolver_test.go +++ b/namesys/ipns_resolver_test.go @@ -56,7 +56,7 @@ func TestPrexistingExpiredRecord(t *testing.T) { entry, err := ipns.NewRecord(identity.PrivateKey(), h, 0, eol, 0) require.NoError(t, err) - err = PublishRecord(context.Background(), d, identity.PublicKey(), entry) + err = PublishIPNSRecord(context.Background(), d, identity.PublicKey(), entry) require.NoError(t, err) // Now, with an old record in the system already, try and publish a new one @@ -85,7 +85,7 @@ func TestPrexistingRecord(t *testing.T) { entry, err := ipns.NewRecord(identity.PrivateKey(), h, 0, eol, 0) require.NoError(t, err) - err = PublishRecord(context.Background(), d, identity.PublicKey(), entry) + err = PublishIPNSRecord(context.Background(), d, identity.PublicKey(), entry) require.NoError(t, err) // Now, with an old record in the system already, try and publish a new one diff --git a/namesys/namesys.go b/namesys/namesys.go index 7014303e1..de610fc15 100644 --- a/namesys/namesys.go +++ b/namesys/namesys.go @@ -22,7 +22,6 @@ import ( lru "github.com/hashicorp/golang-lru/v2" "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/path" - "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" ci "github.com/libp2p/go-libp2p/core/crypto" @@ -47,7 +46,7 @@ type namesys struct { dnsResolver, ipnsResolver resolver ipnsPublisher Publisher - staticMap map[string]path.Path + staticMap map[string]*cacheEntry cache *lru.Cache[string, any] } @@ -92,14 +91,14 @@ func WithDatastore(ds ds.Datastore) Option { // NewNameSystem constructs an IPFS [NameSystem] based on the given [routing.ValueStore]. func NewNameSystem(r routing.ValueStore, opts ...Option) (NameSystem, error) { - var staticMap map[string]path.Path + var staticMap map[string]*cacheEntry // Prewarm namesys cache with static records for deterministic tests and debugging. // Useful for testing things like DNSLink without real DNS lookup. // Example: // IPFS_NS_MAP="dnslink-test.example.com:/ipfs/bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am" if list := os.Getenv("IPFS_NS_MAP"); list != "" { - staticMap = make(map[string]path.Path) + staticMap = make(map[string]*cacheEntry) for _, pair := range strings.Split(list, ",") { mapping := strings.SplitN(pair, ":", 2) key := mapping[0] @@ -107,7 +106,7 @@ func NewNameSystem(r routing.ValueStore, opts ...Option) (NameSystem, error) { if err != nil { return nil, err } - staticMap[key] = value + staticMap[key] = &cacheEntry{val: value, ttl: 0} } } @@ -141,16 +140,6 @@ func (ns *namesys) Resolve(ctx context.Context, name string, options ...ResolveO ctx, span := startSpan(ctx, "MPNS.Resolve", trace.WithAttributes(attribute.String("Name", name))) defer span.End() - if strings.HasPrefix(name, "/ipfs/") { - p, err := path.NewPath(name) - return p, 0, err - } - - if !strings.HasPrefix(name, "/") { - p, err := path.NewPath("/ipfs/" + name) - return p, 0, err - } - return resolve(ctx, ns, name, ProcessResolveOptions(options)) } @@ -158,22 +147,6 @@ func (ns *namesys) ResolveAsync(ctx context.Context, name string, options ...Res ctx, span := startSpan(ctx, "MPNS.ResolveAsync", trace.WithAttributes(attribute.String("Name", name))) defer span.End() - if strings.HasPrefix(name, "/ipfs/") { - p, err := path.NewPath(name) - res := make(chan ResolveResult, 1) - res <- ResolveResult{Path: p, Err: err} - close(res) - return res - } - - if !strings.HasPrefix(name, "/") { - p, err := path.NewPath("/ipfs/" + name) - res := make(chan ResolveResult, 1) - res <- ResolveResult{Path: p, Err: err} - close(res) - return res - } - return resolveAsync(ctx, ns, name, ProcessResolveOptions(options)) } @@ -184,58 +157,52 @@ func (ns *namesys) resolveOnceAsync(ctx context.Context, name string, options Re out := make(chan ResolveResult, 1) - if !strings.HasPrefix(name, ipns.NamespacePrefix) { - name = ipns.NamespacePrefix + name - } - segments := strings.SplitN(name, "/", 4) - if len(segments) < 3 || segments[0] != "" { - log.Debugf("invalid name syntax for %s", name) - out <- ResolveResult{Err: ErrResolveFailed} + p, err := path.NewPath(name) + if err != nil { + if p, err = path.NewPath("/ipfs/" + name); err == nil { + out <- ResolveResult{Path: p} + } else { + log.Debugf("invalid name syntax for %q", name) + out <- ResolveResult{Err: ErrResolveFailed} + } + + close(out) + return out + } else if !p.Namespace().Mutable() { + out <- ResolveResult{Path: p, Err: err} close(out) return out } - key := segments[2] - - // Resolver selection: - // 1. if it is a PeerID/CID/multihash resolve through "ipns". - // 2. if it is a domain name, resolve through "dns" - - var res resolver - ipnsKey, err := peer.Decode(key) - // CIDs in IPNS are expected to have libp2p-key multicodec - // We ease the transition by returning a more meaningful error with a valid CID - if err != nil { - ipnsCid, cidErr := cid.Decode(key) - if cidErr == nil && ipnsCid.Version() == 1 && ipnsCid.Type() != cid.Libp2pKey { - fixedCid := cid.NewCidV1(cid.Libp2pKey, ipnsCid.Hash()).String() - codecErr := fmt.Errorf("peer ID represented as CIDv1 require libp2p-key multicodec: retry with /ipns/%s", fixedCid) - log.Debugf("RoutingResolver: could not convert public key hash %q to peer ID: %s\n", key, codecErr) - out <- ResolveResult{Err: codecErr} - close(out) - return out - } - } + segments := p.Segments() + key := segments[1] + ipnsKey, err := ipns.NameFromString(key) cacheKey := key if err == nil { - cacheKey = string(ipnsKey) + cacheKey = ipnsKey.String() } - if p, ok := ns.cacheGet(cacheKey); ok { + if p, ttl, ok := ns.cacheGet(cacheKey); ok { var err error - if len(segments) > 3 { - p, err = path.Join(p, segments[3]) + if len(segments) > 2 { + p, err = path.Join(p, segments[2]) } span.SetAttributes(attribute.Bool("CacheHit", true)) span.RecordError(err) - out <- ResolveResult{Path: p, Err: err} + out <- ResolveResult{Path: p, TTL: ttl, Err: err} close(out) return out + } else { + span.SetAttributes(attribute.Bool("CacheHit", false)) } - span.SetAttributes(attribute.Bool("CacheHit", false)) + // Resolver selection: + // 1. If it is an IPNS Name, resolve through IPNS. + // 2. if it is a domain name, resolve through DNSLink. + + var res resolver if err == nil { res = ns.ipnsResolver } else if _, ok := dns.IsDomainName(key); ok { @@ -259,19 +226,18 @@ func (ns *namesys) resolveOnceAsync(ctx context.Context, name string, options Re } return } + if res.Err == nil { best = res } - p := res.Path - err := res.Err - ttl := res.TTL // Attach rest of the path - if len(segments) > 3 { - p, err = path.Join(p, segments[3]) + p := res.Path + if p != nil && len(segments) > 2 { + p, err = path.Join(p, segments[2]) } - emitOnceResult(ctx, out, ResolveResult{Path: p, TTL: ttl, Err: err}) + emitOnceResult(ctx, out, ResolveResult{Path: p, TTL: res.TTL, Err: res.Err}) case <-ctx.Done(): return } diff --git a/namesys/namesys_cache.go b/namesys/namesys_cache.go index d761971ee..34be88955 100644 --- a/namesys/namesys_cache.go +++ b/namesys/namesys_cache.go @@ -6,22 +6,22 @@ import ( "github.com/ipfs/boxo/path" ) -func (ns *namesys) cacheGet(name string) (path.Path, bool) { +func (ns *namesys) cacheGet(name string) (path.Path, time.Duration, bool) { // existence of optional mapping defined via IPFS_NS_MAP is checked first if ns.staticMap != nil { val, ok := ns.staticMap[name] if ok { - return val, true + return val.val, val.ttl, true } } if ns.cache == nil { - return nil, false + return nil, 0, false } ientry, ok := ns.cache.Get(name) if !ok { - return nil, false + return nil, 0, false } entry, ok := ientry.(cacheEntry) @@ -31,12 +31,12 @@ func (ns *namesys) cacheGet(name string) (path.Path, bool) { } if time.Now().Before(entry.eol) { - return entry.val, true + return entry.val, entry.ttl, true } ns.cache.Remove(name) - return nil, false + return nil, 0, false } func (ns *namesys) cacheSet(name string, val path.Path, ttl time.Duration) { @@ -45,6 +45,7 @@ func (ns *namesys) cacheSet(name string, val path.Path, ttl time.Duration) { } ns.cache.Add(name, cacheEntry{ val: val, + ttl: ttl, eol: time.Now().Add(ttl), }) } @@ -58,5 +59,6 @@ func (ns *namesys) cacheInvalidate(name string) { type cacheEntry struct { val path.Path + ttl time.Duration eol time.Time } diff --git a/namesys/namesys_test.go b/namesys/namesys_test.go index a02371a59..cd1db2295 100644 --- a/namesys/namesys_test.go +++ b/namesys/namesys_test.go @@ -21,12 +21,12 @@ type mockResolver struct { entries map[string]string } -func testResolution(t *testing.T, resolver Resolver, name string, depth uint, expected string, expError error) { +func testResolution(t *testing.T, resolver Resolver, name string, depth uint, expected string, expectedTTL time.Duration, expectedError error) { t.Helper() - p, _, err := resolver.Resolve(context.Background(), name, ResolveWithDepth(depth)) - - require.ErrorIs(t, err, expError) + p, ttl, err := resolver.Resolve(context.Background(), name, ResolveWithDepth(depth)) + require.ErrorIs(t, err, expectedError) + require.Equal(t, expectedTTL, ttl) if expected == "" { require.Nil(t, p, "%s with depth %d", name, depth) } else { @@ -69,19 +69,33 @@ func TestNamesysResolution(t *testing.T) { dnsResolver: mockResolverTwo(), } - testResolution(t, r, "Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil) - testResolution(t, r, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil) - testResolution(t, r, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil) - testResolution(t, r, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", 1, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", ErrResolveRecursion) - testResolution(t, r, "/ipns/ipfs.io", DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil) - testResolution(t, r, "/ipns/ipfs.io", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion) - testResolution(t, r, "/ipns/ipfs.io", 2, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", ErrResolveRecursion) - testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil) - testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 1, "/ipns/ipfs.io", ErrResolveRecursion) - testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 2, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion) - testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 3, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", ErrResolveRecursion) - testResolution(t, r, "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion) - testResolution(t, r, "/ipns/bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion) + testResolution(t, r, "Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", 0, nil) + testResolution(t, r, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", 0, nil) + testResolution(t, r, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", 0, nil) + testResolution(t, r, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", 1, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", 0, ErrResolveRecursion) + testResolution(t, r, "/ipns/ipfs.io", DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", 0, nil) + testResolution(t, r, "/ipns/ipfs.io", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", 0, ErrResolveRecursion) + testResolution(t, r, "/ipns/ipfs.io", 2, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", 0, ErrResolveRecursion) + testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", 0, nil) + testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 1, "/ipns/ipfs.io", 0, ErrResolveRecursion) + testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 2, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", 0, ErrResolveRecursion) + testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 3, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", 0, ErrResolveRecursion) + testResolution(t, r, "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", 0, ErrResolveRecursion) + testResolution(t, r, "/ipns/bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", 0, ErrResolveRecursion) +} + +func TestResolveIPNS(t *testing.T) { + ns := &namesys{ + ipnsResolver: mockResolverOne(), + dnsResolver: mockResolverTwo(), + } + + inputPath, err := path.NewPath("/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy/a/b/c") + require.NoError(t, err) + + res, _, err := ResolveIPNS(context.Background(), ns, inputPath) + require.NoError(t, err) + require.Equal(t, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj/a/b/c", res.String()) } func TestPublishWithCache0(t *testing.T) { diff --git a/namesys/republisher/repub.go b/namesys/republisher/repub.go index 34abdde54..2f53cade6 100644 --- a/namesys/republisher/repub.go +++ b/namesys/republisher/repub.go @@ -28,7 +28,7 @@ var ( log = logging.Logger("ipns/repub") ) -var ( +const ( // DefaultRebroadcastInterval is the default interval at which we rebroadcast IPNS records DefaultRebroadcastInterval = time.Hour * 4 @@ -37,10 +37,10 @@ var ( // FailureRetryInterval is the interval at which we retry IPNS records broadcasts (when they fail) FailureRetryInterval = time.Minute * 5 -) -// DefaultRecordLifetime is the default lifetime for IPNS records -const DefaultRecordLifetime = time.Hour * 24 + // DefaultRecordLifetime is the default lifetime for IPNS records + DefaultRecordLifetime = time.Hour * 24 +) // Republisher facilitates the regular publishing of all the IPNS records // associated to keys in a [keystore.Keystore]. diff --git a/namesys/republisher/repub_test.go b/namesys/republisher/repub_test.go index 93e38474f..ff5e50634 100644 --- a/namesys/republisher/repub_test.go +++ b/namesys/republisher/repub_test.go @@ -27,7 +27,7 @@ import ( type mockNode struct { h host.Host - id string + id peer.ID privKey ic.PrivKey store ds.Batching dht *dht.IpfsDHT @@ -51,7 +51,7 @@ func getMockNode(t *testing.T, ctx context.Context) *mockNode { return &mockNode{ h: h, - id: h.ID().Pretty(), + id: h.ID(), privKey: h.Peerstore().PrivKey(h.ID()), store: dstore, dht: idht, @@ -90,7 +90,7 @@ func TestRepublish(t *testing.T) { require.NoError(t, err) rp := namesys.NewIPNSPublisher(publisher.dht, publisher.store) - name := "/ipns/" + publisher.id + name := ipns.NameFromPeer(publisher.id).AsPath().String() // Retry in case the record expires before we can fetch it. This can // happen when running the test on a slow machine. @@ -166,7 +166,7 @@ func TestLongEOLRepublish(t *testing.T) { require.NoError(t, err) rp := namesys.NewIPNSPublisher(publisher.dht, publisher.store) - name := "/ipns/" + publisher.id + name := ipns.NameFromPeer(publisher.id).AsPath().String() expiration := time.Now().Add(time.Hour) err = rp.Publish(ctx, publisher.privKey, p, namesys.PublishWithEOL(expiration)) diff --git a/namesys/resolve/resolve.go b/namesys/resolve/resolve.go deleted file mode 100644 index b2d85e9e8..000000000 --- a/namesys/resolve/resolve.go +++ /dev/null @@ -1,63 +0,0 @@ -package resolve - -import ( - "context" - "errors" - "fmt" - - "github.com/ipfs/boxo/path" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - - "github.com/ipfs/boxo/namesys" -) - -// ErrNoNamesys is an explicit error for when an IPFS node doesn't -// (yet) have a name system -var ErrNoNamesys = errors.New("core/resolve: no Namesys on IpfsNode - can't resolve ipns entry") - -// ResolveIPNS resolves /ipns paths -func ResolveIPNS(ctx context.Context, ns namesys.NameSystem, p path.Path) (path.Path, error) { - ctx, span := startSpan(ctx, "ResolveIPNS", trace.WithAttributes(attribute.String("Path", p.String()))) - defer span.End() - - if p.Namespace() == path.IPNSNamespace { - // TODO(cryptix): we should be able to query the local cache for the path - if ns == nil { - return nil, ErrNoNamesys - } - - seg := p.Segments() - - if len(seg) < 2 || seg[1] == "" { // just "/" without further segments - err := fmt.Errorf("invalid path %q: ipns path missing IPNS ID", p) - return nil, err - } - - extensions := seg[2:] - resolvable, err := path.NewPathFromSegments(seg[0], seg[1]) - if err != nil { - return nil, err - } - - resolvedPath, _, err := ns.Resolve(ctx, resolvable.String()) - if err != nil { - return nil, err - } - - segments := append(resolvedPath.Segments(), extensions...) - p, err = path.NewPathFromSegments(segments...) - if err != nil { - return nil, err - } - } - - return p, nil -} - -var tracer = otel.Tracer("boxo/namesys/resolve") - -func startSpan(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) { - return tracer.Start(ctx, fmt.Sprintf("Namesys.%s", name)) -} diff --git a/namesys/utilities.go b/namesys/utilities.go index 6de61ca3b..a2dc8860d 100644 --- a/namesys/utilities.go +++ b/namesys/utilities.go @@ -3,10 +3,8 @@ package namesys import ( "context" "fmt" - "strings" "time" - "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/path" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/trace" @@ -17,16 +15,11 @@ type resolver interface { } // resolve is a helper for implementing Resolver.ResolveN using resolveOnce. -func resolve(ctx context.Context, r resolver, name string, options ResolveOptions) (path.Path, time.Duration, error) { +func resolve(ctx context.Context, r resolver, name string, options ResolveOptions) (p path.Path, ttl time.Duration, err error) { ctx, cancel := context.WithCancel(ctx) defer cancel() - err := ErrResolveFailed - var ( - p path.Path - ttl time.Duration - ) - + err = ErrResolveFailed resCh := resolveAsync(ctx, r, name, options) for res := range resCh { @@ -69,24 +62,26 @@ func resolveAsync(ctx context.Context, r resolver, name string, options ResolveO } if res.Err != nil { - emitResult(ctx, outCh, ResolveResult{Err: res.Err}) + emitResult(ctx, outCh, res) return } log.Debugf("resolved %s to %s", name, res.Path.String()) - if !strings.HasPrefix(res.Path.String(), ipns.NamespacePrefix) { - emitResult(ctx, outCh, ResolveResult{Path: res.Path, TTL: res.TTL}) + + if !res.Path.Namespace().Mutable() { + emitResult(ctx, outCh, res) break } if depth == 1 { - emitResult(ctx, outCh, ResolveResult{Path: res.Path, TTL: res.TTL, Err: ErrResolveRecursion}) + res.Err = ErrResolveRecursion + emitResult(ctx, outCh, res) break } - subopts := options - if subopts.Depth > 1 { - subopts.Depth-- + subOpts := options + if subOpts.Depth > 1 { + subOpts.Depth-- } var subCtx context.Context @@ -94,11 +89,11 @@ func resolveAsync(ctx context.Context, r resolver, name string, options ResolveO // Cancel previous recursive resolve since it won't be used anyways cancelSub() } + subCtx, cancelSub = context.WithCancel(ctx) _ = cancelSub - p := strings.TrimPrefix(res.Path.String(), ipns.NamespacePrefix) - subCh = resolveAsync(subCtx, r, p, subopts) + subCh = resolveAsync(subCtx, r, res.Path.String(), subOpts) case res, ok := <-subCh: if !ok { subCh = nil