diff --git a/core/commands/dht.go b/core/commands/dht.go index f1d4a7b6c81d..35eb803a3c4c 100644 --- a/core/commands/dht.go +++ b/core/commands/dht.go @@ -361,12 +361,8 @@ func provideKeysRec(ctx context.Context, r routing.IpfsRouting, dserv dag.DAGSer provided := make(map[key.Key]struct{}) for _, c := range cids { kset := key.NewKeySet() - node, err := dserv.Get(ctx, c) - if err != nil { - return err - } - err = dag.EnumerateChildrenAsync(ctx, dserv, node, func(c *cid.Cid) bool { + err := dag.EnumerateChildrenAsync(ctx, dserv, c, func(c *cid.Cid) bool { k := key.Key(c.Hash()) if kset.Has(k) { kset.Add(k) diff --git a/core/commands/pin.go b/core/commands/pin.go index 62df21b63118..1444f1175ff3 100644 --- a/core/commands/pin.go +++ b/core/commands/pin.go @@ -328,11 +328,7 @@ func pinLsAll(typeStr string, ctx context.Context, n *core.IpfsNode) (map[string if typeStr == "indirect" || typeStr == "all" { set := cid.NewSet() for _, k := range n.Pinning.RecursiveKeys() { - links, err := n.DAG.GetLinks(ctx, k) - if err != nil { - return nil, err - } - err = dag.EnumerateChildren(n.Context(), n.DAG, links, set.Visit, false) + err := dag.EnumerateChildren(n.Context(), n.DAG, k, set.Visit, false) if err != nil { return nil, err } diff --git a/core/coreunix/add_test.go b/core/coreunix/add_test.go index d3a6d41cacea..95c07057d3f3 100644 --- a/core/coreunix/add_test.go +++ b/core/coreunix/add_test.go @@ -156,13 +156,9 @@ func TestAddGCLive(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - root, err := node.DAG.Get(ctx, last) - if err != nil { - t.Fatal(err) - } set := cid.NewSet() - err = dag.EnumerateChildren(ctx, node.DAG, root.Links, set.Visit, false) + err = dag.EnumerateChildren(ctx, node.DAG, last, set.Visit, false) if err != nil { t.Fatal(err) } diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go index 3988440f51f0..d7e87a4ed3d3 100644 --- a/merkledag/merkledag.go +++ b/merkledag/merkledag.go @@ -125,8 +125,8 @@ func (n *dagService) Remove(nd *Node) error { } // FetchGraph fetches all nodes that are children of the given node -func FetchGraph(ctx context.Context, root *Node, serv DAGService) error { - return EnumerateChildrenAsync(ctx, serv, root, cid.NewSet().Visit) +func FetchGraph(ctx context.Context, c *cid.Cid, serv DAGService) error { + return EnumerateChildrenAsync(ctx, serv, c, cid.NewSet().Visit) } // FindLinks searches this nodes links for the given key, @@ -393,19 +393,17 @@ func legacyCidFromLink(lnk *Link) *cid.Cid { // EnumerateChildren will walk the dag below the given root node and add all // unseen children to the passed in set. // TODO: parallelize to avoid disk latency perf hits? -func EnumerateChildren(ctx context.Context, ds LinkService, links []*Link, visit func(*cid.Cid) bool, bestEffort bool) error { +func EnumerateChildren(ctx context.Context, ds LinkService, root *cid.Cid, visit func(*cid.Cid) bool, bestEffort bool) error { + links, err := ds.GetLinks(ctx, root) + if bestEffort && err == ErrNotFound { + return nil + } else if err != nil { + return err + } for _, lnk := range links { c := legacyCidFromLink(lnk) if visit(c) { - children, err := ds.GetLinks(ctx, c) - if err != nil { - if bestEffort && err == ErrNotFound { - continue - } else { - return err - } - } - err = EnumerateChildren(ctx, ds, children, visit, bestEffort) + err = EnumerateChildren(ctx, ds, c, visit, bestEffort) if err != nil { return err } @@ -414,7 +412,7 @@ func EnumerateChildren(ctx context.Context, ds LinkService, links []*Link, visit return nil } -func EnumerateChildrenAsync(ctx context.Context, ds DAGService, root *Node, visit func(*cid.Cid) bool) error { +func EnumerateChildrenAsync(ctx context.Context, ds DAGService, c *cid.Cid, visit func(*cid.Cid) bool) error { toprocess := make(chan []*cid.Cid, 8) nodes := make(chan *NodeOption, 8) @@ -424,6 +422,11 @@ func EnumerateChildrenAsync(ctx context.Context, ds DAGService, root *Node, visi go fetchNodes(ctx, ds, toprocess, nodes) + root, err := ds.Get(ctx, c) + if err != nil { + return err + } + nodes <- &NodeOption{Node: root} live := 1 diff --git a/merkledag/merkledag_test.go b/merkledag/merkledag_test.go index 91283bbbbd51..d6c36367e85d 100644 --- a/merkledag/merkledag_test.go +++ b/merkledag/merkledag_test.go @@ -231,7 +231,7 @@ func TestFetchGraph(t *testing.T) { t.Fatal(err) } - err = FetchGraph(context.TODO(), root, dservs[1]) + err = FetchGraph(context.TODO(), root.Cid(), dservs[1]) if err != nil { t.Fatal(err) } @@ -241,7 +241,7 @@ func TestFetchGraph(t *testing.T) { offline_ds := NewDAGService(bs) - err = EnumerateChildren(context.Background(), offline_ds, root.Links, func(_ *cid.Cid) bool { return true }, false) + err = EnumerateChildren(context.Background(), offline_ds, root.Cid(), func(_ *cid.Cid) bool { return true }, false) if err != nil { t.Fatal(err) } @@ -258,7 +258,7 @@ func TestEnumerateChildren(t *testing.T) { } set := cid.NewSet() - err = EnumerateChildren(context.Background(), ds, root.Links, set.Visit, false) + err = EnumerateChildren(context.Background(), ds, root.Cid(), set.Visit, false) if err != nil { t.Fatal(err) } @@ -269,7 +269,7 @@ func TestEnumerateChildren(t *testing.T) { for _, lnk := range n.Links { c := cid.NewCidV0(lnk.Hash) if !set.Has(c) { - t.Fatal("missing key in set!") + t.Fatal("missing key in set! ", lnk.Hash.B58String()) } child, err := ds.Get(context.Background(), c) if err != nil { diff --git a/pin/gc/gc.go b/pin/gc/gc.go index a6973942725c..6f909e564aa7 100644 --- a/pin/gc/gc.go +++ b/pin/gc/gc.go @@ -71,13 +71,9 @@ func GC(ctx context.Context, bs bstore.GCBlockstore, ls dag.LinkService, pn pin. func Descendants(ctx context.Context, ls dag.LinkService, set key.KeySet, roots []*cid.Cid, bestEffort bool) error { for _, c := range roots { set.Add(key.Key(c.Hash())) - links, err := ls.GetLinks(ctx, c) - if err != nil { - return err - } // EnumerateChildren recursively walks the dag and adds the keys to the given set - err = dag.EnumerateChildren(ctx, ls, links, func(c *cid.Cid) bool { + err := dag.EnumerateChildren(ctx, ls, c, func(c *cid.Cid) bool { k := key.Key(c.Hash()) seen := set.Has(k) if seen { diff --git a/pin/pin.go b/pin/pin.go index b0d8302ed1cc..1c42fa04fb89 100644 --- a/pin/pin.go +++ b/pin/pin.go @@ -178,7 +178,7 @@ func (p *pinner) Pin(ctx context.Context, node *mdag.Node, recurse bool) error { } // fetch entire graph - err := mdag.FetchGraph(ctx, node, p.dserv) + err := mdag.FetchGraph(ctx, c, p.dserv) if err != nil { return err } @@ -279,12 +279,7 @@ func (p *pinner) isPinnedWithType(c *cid.Cid, mode PinMode) (string, bool, error // Default is Indirect for _, rc := range p.recursePin.Keys() { - links, err := p.dserv.GetLinks(context.Background(), rc) - if err != nil { - return "", false, err - } - - has, err := hasChild(p.dserv, links, k) + has, err := hasChild(p.dserv, rc, k) if err != nil { return "", false, err } @@ -521,19 +516,18 @@ func (p *pinner) PinWithMode(c *cid.Cid, mode PinMode) { } } -func hasChild(ds mdag.LinkService, links []*mdag.Link, child key.Key) (bool, error) { +func hasChild(ds mdag.LinkService, root *cid.Cid, child key.Key) (bool, error) { + links, err := ds.GetLinks(context.Background(), root) + if err != nil { + return false, err + } for _, lnk := range links { c := cid.NewCidV0(lnk.Hash) if key.Key(c.Hash()) == child { return true, nil } - children, err := ds.GetLinks(context.Background(), c) - if err != nil { - return false, err - } - - has, err := hasChild(ds, children, child) + has, err := hasChild(ds, c, child) if err != nil { return false, err } diff --git a/pin/pin_test.go b/pin/pin_test.go index af3aa08dabd1..0d4e7591f92a 100644 --- a/pin/pin_test.go +++ b/pin/pin_test.go @@ -225,6 +225,11 @@ func TestPinRecursiveFail(t *testing.T) { t.Fatal(err) } + _, err = dserv.Add(a) + if err != nil { + t.Fatal(err) + } + // this one is time based... but shouldnt cause any issues mctx, _ = context.WithTimeout(ctx, time.Second) err = p.Pin(mctx, a, true)