diff --git a/coding.go b/coding.go index f0d6d69..a9a1189 100644 --- a/coding.go +++ b/coding.go @@ -6,10 +6,13 @@ import ( "strings" blocks "github.com/ipfs/go-block-format" - pb "github.com/ipfs/go-merkledag/pb" - cid "github.com/ipfs/go-cid" - ipld "github.com/ipfs/go-ipld-format" + format "github.com/ipfs/go-ipld-format" + pb "github.com/ipfs/go-merkledag/pb" + dagpb "github.com/ipld/go-codec-dagpb" + ipld "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/fluent/qp" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" ) // Make sure the user doesn't upgrade this file. @@ -22,38 +25,84 @@ const _ = pb.DoNotUpgradeFileEverItWillChangeYourHashes // unmarshal decodes raw data into a *Node instance. // The conversion uses an intermediate PBNode. -func (n *ProtoNode) unmarshal(encoded []byte) error { - var pbn pb.PBNode - if err := pbn.Unmarshal(encoded); err != nil { - return fmt.Errorf("unmarshal failed. %v", err) +func unmarshal(encodedBytes []byte) (*ProtoNode, error) { + nb := dagpb.Type.PBNode.NewBuilder() + if err := dagpb.DecodeBytes(nb, encodedBytes); err != nil { + return nil, err } + nd := nb.Build() + return fromImmutableNode(&immutableProtoNode{encodedBytes, nd.(dagpb.PBNode)}), nil +} - pbnl := pbn.GetLinks() - n.links = make([]*ipld.Link, len(pbnl)) - for i, l := range pbnl { - n.links[i] = &ipld.Link{Name: l.GetName(), Size: l.GetTsize()} - c, err := cid.Cast(l.GetHash()) - if err != nil { - return fmt.Errorf("link hash #%d is not valid multihash. %v", i, err) +func fromImmutableNode(encoded *immutableProtoNode) *ProtoNode { + n := new(ProtoNode) + n.encoded = encoded + if n.encoded.PBNode.Data.Exists() { + n.data = n.encoded.PBNode.Data.Must().Bytes() + } + numLinks := n.encoded.PBNode.Links.Length() + n.links = make([]*format.Link, numLinks) + linkAllocs := make([]format.Link, numLinks) + for i := int64(0); i < numLinks; i++ { + next := n.encoded.PBNode.Links.Lookup(i) + name := "" + if next.FieldName().Exists() { + name = next.FieldName().Must().String() } - n.links[i].Cid = c + c := cid.Undef + c = next.FieldHash().Link().(cidlink.Link).Cid + size := uint64(0) + if next.FieldTsize().Exists() { + size = uint64(next.FieldTsize().Must().Int()) + } + link := &linkAllocs[i] + link.Name = name + link.Size = size + link.Cid = c + n.links[i] = link + } + return n +} +func (n *ProtoNode) marshalImmutable() (*immutableProtoNode, error) { + nd, err := qp.BuildMap(dagpb.Type.PBNode, 2, func(ma ipld.MapAssembler) { + qp.MapEntry(ma, "Links", qp.List(int64(len(n.links)), func(la ipld.ListAssembler) { + for _, link := range n.links { + qp.ListEntry(la, qp.Map(3, func(ma ipld.MapAssembler) { + if link.Cid.Defined() { + qp.MapEntry(ma, "Hash", qp.Link(cidlink.Link{Cid: link.Cid})) + } + qp.MapEntry(ma, "Name", qp.String(link.Name)) + qp.MapEntry(ma, "Tsize", qp.Int(int64(link.Size))) + })) + } + })) + if n.data != nil { + qp.MapEntry(ma, "Data", qp.Bytes(n.data)) + } + }) + if err != nil { + return nil, err } - sort.Stable(LinkSlice(n.links)) // keep links sorted - n.data = pbn.GetData() - n.encoded = encoded - return nil + // 1KiB can be allocated on the stack, and covers most small nodes + // without having to grow the buffer and cause allocations. + enc := make([]byte, 0, 1024) + + enc, err = dagpb.AppendEncode(enc, nd) + if err != nil { + return nil, err + } + return &immutableProtoNode{enc, nd.(dagpb.PBNode)}, nil } // Marshal encodes a *Node instance into a new byte slice. // The conversion uses an intermediate PBNode. func (n *ProtoNode) Marshal() ([]byte, error) { - pbn := n.GetPBNode() - data, err := pbn.Marshal() + enc, err := n.marshalImmutable() if err != nil { - return data, fmt.Errorf("marshal failed. %v", err) + return nil, err } - return data, nil + return enc.encoded, nil } // GetPBNode converts *ProtoNode into it's protocol buffer variant. @@ -88,14 +137,14 @@ func (n *ProtoNode) EncodeProtobuf(force bool) ([]byte, error) { if n.encoded == nil || force { n.cached = cid.Undef var err error - n.encoded, err = n.Marshal() + n.encoded, err = n.marshalImmutable() if err != nil { return nil, err } } if !n.cached.Defined() { - c, err := n.CidBuilder().Sum(n.encoded) + c, err := n.CidBuilder().Sum(n.encoded.encoded) if err != nil { return nil, err } @@ -103,13 +152,12 @@ func (n *ProtoNode) EncodeProtobuf(force bool) ([]byte, error) { n.cached = c } - return n.encoded, nil + return n.encoded.encoded, nil } // DecodeProtobuf decodes raw data and returns a new Node instance. func DecodeProtobuf(encoded []byte) (*ProtoNode, error) { - n := new(ProtoNode) - err := n.unmarshal(encoded) + n, err := unmarshal(encoded) if err != nil { return nil, fmt.Errorf("incorrectly formatted merkledag node: %s", err) } @@ -118,7 +166,7 @@ func DecodeProtobuf(encoded []byte) (*ProtoNode, error) { // DecodeProtobufBlock is a block decoder for protobuf IPLD nodes conforming to // node.DecodeBlockFunc -func DecodeProtobufBlock(b blocks.Block) (ipld.Node, error) { +func DecodeProtobufBlock(b blocks.Block) (format.Node, error) { c := b.Cid() if c.Type() != cid.DagProtobuf { return nil, fmt.Errorf("this function can only decode protobuf nodes") @@ -138,4 +186,4 @@ func DecodeProtobufBlock(b blocks.Block) (ipld.Node, error) { } // Type assertion -var _ ipld.DecodeBlockFunc = DecodeProtobufBlock +var _ format.DecodeBlockFunc = DecodeProtobufBlock diff --git a/coding_test.go b/coding_test.go new file mode 100644 index 0000000..2ebe743 --- /dev/null +++ b/coding_test.go @@ -0,0 +1,52 @@ +package merkledag_test + +import ( + "bytes" + "fmt" + "testing" + + cid "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" +) + +var benchInput []byte + +func init() { + someData := bytes.Repeat([]byte("some plaintext data\n"), 10) + // make a test CID -- doesn't matter just to add as a link + someCid, _ := cid.Cast([]byte{1, 85, 0, 5, 0, 1, 2, 3, 4}) + + node := &merkledag.ProtoNode{} + node.SetData(someData) + for i := 0; i < 10; i++ { + node.AddRawLink(fmt.Sprintf("%d", i), &ipld.Link{ + Size: 10, + Cid: someCid, + }) + } + + enc, err := node.EncodeProtobuf(true) + if err != nil { + panic(err) + } + benchInput = enc +} + +func BenchmarkRoundtrip(b *testing.B) { + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + node, err := merkledag.DecodeProtobuf(benchInput) + if err != nil { + b.Fatal(err) + } + + enc, err := node.EncodeProtobuf(true) + if err != nil { + b.Fatal(err) + } + _ = enc + } + }) +} diff --git a/go.mod b/go.mod index 0bfaa92..adcafdd 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,9 @@ require ( github.com/ipfs/go-ipfs-util v0.0.2 github.com/ipfs/go-ipld-cbor v0.0.5 github.com/ipfs/go-ipld-format v0.2.0 + github.com/ipfs/go-ipld-legacy v0.1.0 + github.com/ipld/go-codec-dagpb v1.3.0 + github.com/ipld/go-ipld-prime v0.11.0 github.com/multiformats/go-multihash v0.0.15 ) diff --git a/go.sum b/go.sum index b30c930..8b54865 100644 --- a/go.sum +++ b/go.sum @@ -77,6 +77,8 @@ github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= +github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -113,8 +115,9 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -122,8 +125,10 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3 h1:x95R7cp+rSeeqAMI2knLtQ0DKlaBhv2NrtrOvafPHRo= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= @@ -220,6 +225,8 @@ github.com/ipfs/go-ipld-cbor v0.0.5/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9 github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= github.com/ipfs/go-ipld-format v0.2.0 h1:xGlJKkArkmBvowr+GMCX0FEZtkro71K1AwiKnL37mwA= github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs= +github.com/ipfs/go-ipld-legacy v0.1.0 h1:wxkkc4k8cnvIGIjPO0waJCe7SHEyFgl+yQdafdjGrpA= +github.com/ipfs/go-ipld-legacy v0.1.0/go.mod h1:86f5P/srAmh9GcIcWQR9lfFLZPrIyyXQeVlOWeeWEuI= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk= github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A= @@ -238,6 +245,11 @@ github.com/ipfs/go-peertaskqueue v0.2.0 h1:2cSr7exUGKYyDeUyQ7P/nHPs9P7Ht/B+ROrpN github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUnr+P7jivavs9lY= github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= +github.com/ipld/go-codec-dagpb v1.3.0 h1:czTcaoAuNNyIYWs6Qe01DJ+sEX7B+1Z0LcXjSatMGe8= +github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= +github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= +github.com/ipld/go-ipld-prime v0.11.0 h1:jD/b/22R7CSL+F9xNffcexs+wO0Ji/TfwXO/TWck+70= +github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -258,8 +270,9 @@ github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJS github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= @@ -603,8 +616,10 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992 h1:bzMe+2coZJYHnhGgVlcQKuRy4FSny4ds8dLQjw5P1XE= github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e h1:ZOcivgkkFRnjfoTcGsDq3UQYiBmekwLA+qg0OjyB/ls= +github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -638,8 +653,9 @@ github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5k github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa h1:E+gaaifzi2xF65PbDmuKI3PhLWY6G5opMLniFq8vmXA= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= @@ -669,8 +685,9 @@ github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cb github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= -github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436 h1:qOpVTI+BrstcjTZLm2Yz/3sOnqkzj3FQoh0g+E5s3Gc= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w= +github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158 h1:WXhVOwj2USAXB5oMDwRl3piOux2XMV9TANaYxXHdkoE= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= @@ -840,6 +857,7 @@ golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -893,8 +911,10 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/merkledag.go b/merkledag.go index 76f402b..5d318e3 100644 --- a/merkledag.go +++ b/merkledag.go @@ -10,16 +10,25 @@ import ( bserv "github.com/ipfs/go-blockservice" cid "github.com/ipfs/go-cid" ipldcbor "github.com/ipfs/go-ipld-cbor" - ipld "github.com/ipfs/go-ipld-format" + format "github.com/ipfs/go-ipld-format" + legacy "github.com/ipfs/go-ipld-legacy" + dagpb "github.com/ipld/go-codec-dagpb" + + // blank import is used to register the IPLD raw codec + _ "github.com/ipld/go-ipld-prime/codec/raw" + basicnode "github.com/ipld/go-ipld-prime/node/basic" ) // TODO: We should move these registrations elsewhere. Really, most of the IPLD // functionality should go in a `go-ipld` repo but that will take a lot of work // and design. func init() { - ipld.Register(cid.DagProtobuf, DecodeProtobufBlock) - ipld.Register(cid.Raw, DecodeRawBlock) - ipld.Register(cid.DagCBOR, ipldcbor.DecodeBlock) + format.Register(cid.DagProtobuf, DecodeProtobufBlock) + format.Register(cid.Raw, DecodeRawBlock) + format.Register(cid.DagCBOR, ipldcbor.DecodeBlock) + + legacy.RegisterCodec(cid.DagProtobuf, dagpb.Type.PBNode, ProtoNodeConverter) + legacy.RegisterCodec(cid.Raw, basicnode.Prototype.Bytes, RawNodeConverter) } // contextKey is a type to use as value for the ProgressTracker contexts. @@ -43,7 +52,7 @@ type dagService struct { } // Add adds a node to the dagService, storing the block in the BlockService -func (n *dagService) Add(ctx context.Context, nd ipld.Node) error { +func (n *dagService) Add(ctx context.Context, nd format.Node) error { if n == nil { // FIXME remove this assertion. protect with constructor invariant return fmt.Errorf("dagService is nil") } @@ -51,7 +60,7 @@ func (n *dagService) Add(ctx context.Context, nd ipld.Node) error { return n.Blocks.AddBlock(nd) } -func (n *dagService) AddMany(ctx context.Context, nds []ipld.Node) error { +func (n *dagService) AddMany(ctx context.Context, nds []format.Node) error { blks := make([]blocks.Block, len(nds)) for i, nd := range nds { blks[i] = nd @@ -60,7 +69,7 @@ func (n *dagService) AddMany(ctx context.Context, nds []ipld.Node) error { } // Get retrieves a node from the dagService, fetching the block in the BlockService -func (n *dagService) Get(ctx context.Context, c cid.Cid) (ipld.Node, error) { +func (n *dagService) Get(ctx context.Context, c cid.Cid) (format.Node, error) { if n == nil { return nil, fmt.Errorf("dagService is nil") } @@ -71,17 +80,17 @@ func (n *dagService) Get(ctx context.Context, c cid.Cid) (ipld.Node, error) { b, err := n.Blocks.GetBlock(ctx, c) if err != nil { if err == bserv.ErrNotFound { - return nil, ipld.ErrNotFound + return nil, format.ErrNotFound } return nil, fmt.Errorf("failed to get block for %s: %v", c, err) } - return ipld.Decode(b) + return legacy.DecodeNode(ctx, b) } // GetLinks return the links for the node, the node doesn't necessarily have // to exist locally. -func (n *dagService) GetLinks(ctx context.Context, c cid.Cid) ([]*ipld.Link, error) { +func (n *dagService) GetLinks(ctx context.Context, c cid.Cid) ([]*format.Link, error) { if c.Type() == cid.Raw { return nil, nil } @@ -114,12 +123,12 @@ func (n *dagService) RemoveMany(ctx context.Context, cids []cid.Cid) error { // GetLinksDirect creates a function to get the links for a node, from // the node, bypassing the LinkService. If the node does not exist // locally (and can not be retrieved) an error will be returned. -func GetLinksDirect(serv ipld.NodeGetter) GetLinks { - return func(ctx context.Context, c cid.Cid) ([]*ipld.Link, error) { +func GetLinksDirect(serv format.NodeGetter) GetLinks { + return func(ctx context.Context, c cid.Cid) ([]*format.Link, error) { nd, err := serv.Get(ctx, c) if err != nil { if err == bserv.ErrNotFound { - err = ipld.ErrNotFound + err = format.ErrNotFound } return nil, err } @@ -132,32 +141,32 @@ type sesGetter struct { } // Get gets a single node from the DAG. -func (sg *sesGetter) Get(ctx context.Context, c cid.Cid) (ipld.Node, error) { +func (sg *sesGetter) Get(ctx context.Context, c cid.Cid) (format.Node, error) { blk, err := sg.bs.GetBlock(ctx, c) switch err { case bserv.ErrNotFound: - return nil, ipld.ErrNotFound + return nil, format.ErrNotFound case nil: // noop default: return nil, err } - return ipld.Decode(blk) + return legacy.DecodeNode(ctx, blk) } // GetMany gets many nodes at once, batching the request if possible. -func (sg *sesGetter) GetMany(ctx context.Context, keys []cid.Cid) <-chan *ipld.NodeOption { +func (sg *sesGetter) GetMany(ctx context.Context, keys []cid.Cid) <-chan *format.NodeOption { return getNodesFromBG(ctx, sg.bs, keys) } // Session returns a NodeGetter using a new session for block fetches. -func (n *dagService) Session(ctx context.Context) ipld.NodeGetter { +func (n *dagService) Session(ctx context.Context) format.NodeGetter { return &sesGetter{bserv.NewSession(ctx, n.Blocks)} } // FetchGraph fetches all nodes that are children of the given node -func FetchGraph(ctx context.Context, root cid.Cid, serv ipld.DAGService) error { +func FetchGraph(ctx context.Context, root cid.Cid, serv format.DAGService) error { return FetchGraphWithDepthLimit(ctx, root, -1, serv) } @@ -165,8 +174,8 @@ func FetchGraph(ctx context.Context, root cid.Cid, serv ipld.DAGService) error { // node down to the given depth. maxDepth=0 means "only fetch root", // maxDepth=1 means "fetch root and its direct children" and so on... // maxDepth=-1 means unlimited. -func FetchGraphWithDepthLimit(ctx context.Context, root cid.Cid, depthLim int, serv ipld.DAGService) error { - var ng ipld.NodeGetter = NewSession(ctx, serv) +func FetchGraphWithDepthLimit(ctx context.Context, root cid.Cid, depthLim int, serv format.DAGService) error { + var ng format.NodeGetter = NewSession(ctx, serv) set := make(map[cid.Cid]int) @@ -212,7 +221,7 @@ func FetchGraphWithDepthLimit(ctx context.Context, root cid.Cid, depthLim int, s // This method may not return all requested nodes (and may or may not return an // error indicating that it failed to do so. It is up to the caller to verify // that it received all nodes. -func (n *dagService) GetMany(ctx context.Context, keys []cid.Cid) <-chan *ipld.NodeOption { +func (n *dagService) GetMany(ctx context.Context, keys []cid.Cid) <-chan *format.NodeOption { return getNodesFromBG(ctx, n.Blocks, keys) } @@ -227,10 +236,10 @@ func dedupKeys(keys []cid.Cid) []cid.Cid { return set.Keys() } -func getNodesFromBG(ctx context.Context, bs bserv.BlockGetter, keys []cid.Cid) <-chan *ipld.NodeOption { +func getNodesFromBG(ctx context.Context, bs bserv.BlockGetter, keys []cid.Cid) <-chan *format.NodeOption { keys = dedupKeys(keys) - out := make(chan *ipld.NodeOption, len(keys)) + out := make(chan *format.NodeOption, len(keys)) blocks := bs.GetBlocks(ctx, keys) var count int @@ -241,22 +250,22 @@ func getNodesFromBG(ctx context.Context, bs bserv.BlockGetter, keys []cid.Cid) < case b, ok := <-blocks: if !ok { if count != len(keys) { - out <- &ipld.NodeOption{Err: fmt.Errorf("failed to fetch all nodes")} + out <- &format.NodeOption{Err: fmt.Errorf("failed to fetch all nodes")} } return } - nd, err := ipld.Decode(b) + nd, err := legacy.DecodeNode(ctx, b) if err != nil { - out <- &ipld.NodeOption{Err: err} + out <- &format.NodeOption{Err: err} return } - out <- &ipld.NodeOption{Node: nd} + out <- &format.NodeOption{Node: nd} count++ case <-ctx.Done(): - out <- &ipld.NodeOption{Err: ctx.Err()} + out <- &format.NodeOption{Err: ctx.Err()} return } } @@ -266,15 +275,15 @@ func getNodesFromBG(ctx context.Context, bs bserv.BlockGetter, keys []cid.Cid) < // GetLinks is the type of function passed to the EnumerateChildren function(s) // for getting the children of an IPLD node. -type GetLinks func(context.Context, cid.Cid) ([]*ipld.Link, error) +type GetLinks func(context.Context, cid.Cid) ([]*format.Link, error) // GetLinksWithDAG returns a GetLinks function that tries to use the given // NodeGetter as a LinkGetter to get the children of a given IPLD node. This may // allow us to traverse the DAG without actually loading and parsing the node in // question (if we already have the links cached). -func GetLinksWithDAG(ng ipld.NodeGetter) GetLinks { - return func(ctx context.Context, c cid.Cid) ([]*ipld.Link, error) { - return ipld.GetLinks(ctx, ng, c) +func GetLinksWithDAG(ng format.NodeGetter) GetLinks { + return func(ctx context.Context, c cid.Cid) ([]*format.Link, error) { + return format.GetLinks(ctx, ng, c) } } @@ -344,7 +353,7 @@ func IgnoreErrors() WalkOption { func IgnoreMissing() WalkOption { return func(walkOptions *walkOptions) { walkOptions.addHandler(func(c cid.Cid, err error) error { - if err == ipld.ErrNotFound { + if err == format.ErrNotFound { return nil } return err @@ -357,7 +366,7 @@ func IgnoreMissing() WalkOption { func OnMissing(callback func(c cid.Cid)) WalkOption { return func(walkOptions *walkOptions) { walkOptions.addHandler(func(c cid.Cid, err error) error { - if err == ipld.ErrNotFound { + if err == format.ErrNotFound { callback(c) } return err @@ -454,7 +463,7 @@ func parallelWalkDepth(ctx context.Context, getLinks GetLinks, root cid.Cid, vis } type linksDepth struct { - links []*ipld.Link + links []*format.Link depth int } @@ -569,7 +578,7 @@ func parallelWalkDepth(ctx context.Context, getLinks GetLinks, root cid.Cid, vis } } -var _ ipld.LinkGetter = &dagService{} -var _ ipld.NodeGetter = &dagService{} -var _ ipld.NodeGetter = &sesGetter{} -var _ ipld.DAGService = &dagService{} +var _ format.LinkGetter = &dagService{} +var _ format.NodeGetter = &dagService{} +var _ format.NodeGetter = &sesGetter{} +var _ format.DAGService = &dagService{} diff --git a/node.go b/node.go index 3478a02..cafd9c3 100644 --- a/node.go +++ b/node.go @@ -5,27 +5,48 @@ import ( "encoding/json" "fmt" + blocks "github.com/ipfs/go-block-format" cid "github.com/ipfs/go-cid" - ipld "github.com/ipfs/go-ipld-format" + format "github.com/ipfs/go-ipld-format" + legacy "github.com/ipfs/go-ipld-legacy" + dagpb "github.com/ipld/go-codec-dagpb" + ipld "github.com/ipld/go-ipld-prime" mh "github.com/multiformats/go-multihash" ) // Common errors var ( ErrNotProtobuf = fmt.Errorf("expected protobuf dag node") + ErrNotRawNode = fmt.Errorf("expected raw bytes node") ErrLinkNotFound = fmt.Errorf("no link by that name") ) +type immutableProtoNode struct { + encoded []byte + dagpb.PBNode +} + // ProtoNode represents a node in the IPFS Merkle DAG. // nodes have opaque data and a set of navigable links. +// ProtoNode is a go-ipld-legacy.UniversalNode, meaning it is both +// a go-ipld-prime node and a go-ipld-format node. +// ProtoNode maintains compatibility with it's original implementation +// as a go-ipld-format only node, which included some mutability, namely the +// the ability to add/remove links in place +// +// TODO: We should be able to eventually replace this implementation with +// * go-codec-dagpb for basic DagPB encode/decode to go-ipld-prime +// * go-unixfsnode ADLs for higher level DAGPB functionality +// For the time being however, go-unixfsnode is read only and +// this mutable protonode implementation is needed to support go-unixfs, +// the only library that implements both read and write for UnixFS v1. type ProtoNode struct { - links []*ipld.Link + links []*format.Link data []byte // cache encoded/marshaled value - encoded []byte - - cached cid.Cid + encoded *immutableProtoNode + cached cid.Cid // builder specifies cid version and hashing function builder cid.Builder @@ -82,8 +103,8 @@ func (n *ProtoNode) SetCidBuilder(builder cid.Builder) { } } -// LinkSlice is a slice of ipld.Links -type LinkSlice []*ipld.Link +// LinkSlice is a slice of format.Links +type LinkSlice []*format.Link func (ls LinkSlice) Len() int { return len(ls) } func (ls LinkSlice) Swap(a, b int) { ls[a], ls[b] = ls[b], ls[a] } @@ -95,10 +116,8 @@ func NodeWithData(d []byte) *ProtoNode { } // AddNodeLink adds a link to another node. -func (n *ProtoNode) AddNodeLink(name string, that ipld.Node) error { - n.encoded = nil - - lnk, err := ipld.MakeLink(that) +func (n *ProtoNode) AddNodeLink(name string, that format.Node) error { + lnk, err := format.MakeLink(that) if err != nil { return err } @@ -111,9 +130,9 @@ func (n *ProtoNode) AddNodeLink(name string, that ipld.Node) error { } // AddRawLink adds a copy of a link to this node -func (n *ProtoNode) AddRawLink(name string, l *ipld.Link) error { +func (n *ProtoNode) AddRawLink(name string, l *format.Link) error { n.encoded = nil - n.links = append(n.links, &ipld.Link{ + n.links = append(n.links, &format.Link{ Name: name, Size: l.Size, Cid: l.Cid, @@ -147,10 +166,10 @@ func (n *ProtoNode) RemoveNodeLink(name string) error { } // GetNodeLink returns a copy of the link with the given name. -func (n *ProtoNode) GetNodeLink(name string) (*ipld.Link, error) { +func (n *ProtoNode) GetNodeLink(name string) (*format.Link, error) { for _, l := range n.links { if l.Name == name { - return &ipld.Link{ + return &format.Link{ Name: l.Name, Size: l.Size, Cid: l.Cid, @@ -161,7 +180,7 @@ func (n *ProtoNode) GetNodeLink(name string) (*ipld.Link, error) { } // GetLinkedProtoNode returns a copy of the ProtoNode with the given name. -func (n *ProtoNode) GetLinkedProtoNode(ctx context.Context, ds ipld.DAGService, name string) (*ProtoNode, error) { +func (n *ProtoNode) GetLinkedProtoNode(ctx context.Context, ds format.DAGService, name string) (*ProtoNode, error) { nd, err := n.GetLinkedNode(ctx, ds, name) if err != nil { return nil, err @@ -176,7 +195,7 @@ func (n *ProtoNode) GetLinkedProtoNode(ctx context.Context, ds ipld.DAGService, } // GetLinkedNode returns a copy of the IPLD Node with the given name. -func (n *ProtoNode) GetLinkedNode(ctx context.Context, ds ipld.DAGService, name string) (ipld.Node, error) { +func (n *ProtoNode) GetLinkedNode(ctx context.Context, ds format.DAGService, name string) (format.Node, error) { lnk, err := n.GetNodeLink(name) if err != nil { return nil, err @@ -187,7 +206,7 @@ func (n *ProtoNode) GetLinkedNode(ctx context.Context, ds ipld.DAGService, name // Copy returns a copy of the node. // NOTE: Does not make copies of Node objects in the links. -func (n *ProtoNode) Copy() ipld.Node { +func (n *ProtoNode) Copy() format.Node { nnode := new(ProtoNode) if len(n.data) > 0 { nnode.data = make([]byte, len(n.data)) @@ -195,7 +214,7 @@ func (n *ProtoNode) Copy() ipld.Node { } if len(n.links) > 0 { - nnode.links = make([]*ipld.Link, len(n.links)) + nnode.links = make([]*format.Link, len(n.links)) copy(nnode.links, n.links) } @@ -204,9 +223,11 @@ func (n *ProtoNode) Copy() ipld.Node { return nnode } -// RawData returns the protobuf-encoded version of the node. func (n *ProtoNode) RawData() []byte { - out, _ := n.EncodeProtobuf(false) + out, err := n.EncodeProtobuf(false) + if err != nil { + panic(err) + } return out } @@ -247,7 +268,7 @@ func (n *ProtoNode) Size() (uint64, error) { } // Stat returns statistics on the node. -func (n *ProtoNode) Stat() (*ipld.NodeStat, error) { +func (n *ProtoNode) Stat() (*format.NodeStat, error) { enc, err := n.EncodeProtobuf(false) if err != nil { return nil, err @@ -258,7 +279,7 @@ func (n *ProtoNode) Stat() (*ipld.NodeStat, error) { return nil, err } - return &ipld.NodeStat{ + return &format.NodeStat{ Hash: n.Cid().String(), NumLinks: len(n.links), BlockSize: len(enc), @@ -278,8 +299,8 @@ func (n *ProtoNode) Loggable() map[string]interface{} { // UnmarshalJSON reads the node fields from a JSON-encoded byte slice. func (n *ProtoNode) UnmarshalJSON(b []byte) error { s := struct { - Data []byte `json:"data"` - Links []*ipld.Link `json:"links"` + Data []byte `json:"data"` + Links []*format.Link `json:"links"` }{} err := json.Unmarshal(b, &s) @@ -309,7 +330,7 @@ func (n *ProtoNode) Cid() cid.Cid { return n.cached } - c, err := n.builder.Sum(n.RawData()) + c, err := n.CidBuilder().Sum(n.RawData()) if err != nil { // programmer error err = fmt.Errorf("invalid CID of length %d: %x: %v", len(n.RawData()), n.RawData(), err) @@ -338,12 +359,12 @@ func (n *ProtoNode) Multihash() mh.Multihash { } // Links returns the node links. -func (n *ProtoNode) Links() []*ipld.Link { +func (n *ProtoNode) Links() []*format.Link { return n.links } // SetLinks replaces the node links with the given ones. -func (n *ProtoNode) SetLinks(links []*ipld.Link) { +func (n *ProtoNode) SetLinks(links []*format.Link) { n.links = links } @@ -355,7 +376,7 @@ func (n *ProtoNode) Resolve(path []string) (interface{}, []string, error) { // ResolveLink consumes the first element of the path and obtains the link // corresponding to it from the node. It returns the link // and the path without the consumed element. -func (n *ProtoNode) ResolveLink(path []string) (*ipld.Link, []string, error) { +func (n *ProtoNode) ResolveLink(path []string) (*format.Link, []string, error) { if len(path) == 0 { return nil, nil, fmt.Errorf("end of path, no more links to resolve") } @@ -382,3 +403,17 @@ func (n *ProtoNode) Tree(p string, depth int) []string { } return out } + +func ProtoNodeConverter(b blocks.Block, nd ipld.Node) (legacy.UniversalNode, error) { + pbNode, ok := nd.(dagpb.PBNode) + if !ok { + return nil, ErrNotProtobuf + } + encoded := &immutableProtoNode{b.RawData(), pbNode} + pn := fromImmutableNode(encoded) + pn.cached = b.Cid() + pn.builder = b.Cid().Prefix() + return pn, nil +} + +var _ legacy.UniversalNode = &ProtoNode{} diff --git a/node_test.go b/node_test.go index 2ae75e7..d12e088 100644 --- a/node_test.go +++ b/node_test.go @@ -12,15 +12,26 @@ import ( ipld "github.com/ipfs/go-ipld-format" ) +var sampleCid cid.Cid + +func init() { + var err error + // make a test CID -- doesn't matter just to add as a link + sampleCid, err = cid.Cast([]byte{1, 85, 0, 5, 0, 1, 2, 3, 4}) + if err != nil { + panic(err) + } +} + func TestStableCID(t *testing.T) { nd := &ProtoNode{} nd.SetData([]byte("foobar")) nd.SetLinks([]*ipld.Link{ - {Name: "a"}, - {Name: "b"}, - {Name: "c"}, + {Name: "a", Cid: sampleCid}, + {Name: "b", Cid: sampleCid}, + {Name: "c", Cid: sampleCid}, }) - expected, err := cid.Decode("QmSN3WED2xPLbYvBbfvew2ZLtui8EbFYYcbfkpKH5jwG9C") + expected, err := cid.Decode("QmciCHWD9Q47VPX6naY3XsPZGnqVqbedAniGCcaHjBaCri") if err != nil { t.Fatal(err) } @@ -32,12 +43,12 @@ func TestStableCID(t *testing.T) { func TestRemoveLink(t *testing.T) { nd := &ProtoNode{} nd.SetLinks([]*ipld.Link{ - {Name: "a"}, - {Name: "b"}, - {Name: "a"}, - {Name: "a"}, - {Name: "c"}, - {Name: "a"}, + {Name: "a", Cid: sampleCid}, + {Name: "b", Cid: sampleCid}, + {Name: "a", Cid: sampleCid}, + {Name: "a", Cid: sampleCid}, + {Name: "c", Cid: sampleCid}, + {Name: "a", Cid: sampleCid}, }) err := nd.RemoveNodeLink("a") @@ -138,9 +149,9 @@ func TestFindLink(t *testing.T) { func TestNodeCopy(t *testing.T) { nd := &ProtoNode{} nd.SetLinks([]*ipld.Link{ - {Name: "a"}, - {Name: "c"}, - {Name: "b"}, + {Name: "a", Cid: sampleCid}, + {Name: "c", Cid: sampleCid}, + {Name: "b", Cid: sampleCid}, }) nd.SetData([]byte("testing")) @@ -156,9 +167,9 @@ func TestNodeCopy(t *testing.T) { func TestJsonRoundtrip(t *testing.T) { nd := new(ProtoNode) nd.SetLinks([]*ipld.Link{ - {Name: "a"}, - {Name: "c"}, - {Name: "b"}, + {Name: "a", Cid: sampleCid}, + {Name: "c", Cid: sampleCid}, + {Name: "b", Cid: sampleCid}, }) nd.SetData([]byte("testing")) diff --git a/prime.go b/prime.go new file mode 100644 index 0000000..5f9bbc6 --- /dev/null +++ b/prime.go @@ -0,0 +1,206 @@ +package merkledag + +import ( + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" +) + +// Protonode was originally implemented as a go-ipld-format node, and included +// functionality that does not fit well into the model for go-ipld-prime, namely +// the ability ot modify the node in place. + +// In order to support the go-ipld-prime interface, all of these prime methods +// serialize and rebuild the go-ipld-prime node as needed, so that it remains up +// to date with mutations made via the add/remove link methods + +// Kind returns a value from the Kind enum describing what the +// essential serializable kind of this node is (map, list, integer, etc). +// Most other handling of a node requires first switching upon the kind. +func (n *ProtoNode) Kind() ipld.Kind { + _, _ = n.EncodeProtobuf(false) + return n.encoded.Kind() +} + +// LookupByString looks up a child object in this node and returns it. +// The returned Node may be any of the Kind: +// a primitive (string, int64, etc), a map, a list, or a link. +// +// If the Kind of this Node is not Kind_Map, a nil node and an error +// will be returned. +// +// If the key does not exist, a nil node and an error will be returned. +func (n *ProtoNode) LookupByString(key string) (ipld.Node, error) { + _, err := n.EncodeProtobuf(false) + if err != nil { + return nil, err + } + return n.encoded.LookupByString(key) +} + +// LookupByNode is the equivalent of LookupByString, but takes a reified Node +// as a parameter instead of a plain string. +// This mechanism is useful if working with typed maps (if the key types +// have constraints, and you already have a reified `schema.TypedNode` value, +// using that value can save parsing and validation costs); +// and may simply be convenient if you already have a Node value in hand. +// +// (When writing generic functions over Node, a good rule of thumb is: +// when handling a map, check for `schema.TypedNode`, and in this case prefer +// the LookupByNode(Node) method; otherwise, favor LookupByString; typically +// implementations will have their fastest paths thusly.) +func (n *ProtoNode) LookupByNode(key ipld.Node) (ipld.Node, error) { + _, err := n.EncodeProtobuf(false) + if err != nil { + return nil, err + } + return n.encoded.LookupByNode(key) +} + +// LookupByIndex is the equivalent of LookupByString but for indexing into a list. +// As with LookupByString, the returned Node may be any of the Kind: +// a primitive (string, int64, etc), a map, a list, or a link. +// +// If the Kind of this Node is not Kind_List, a nil node and an error +// will be returned. +// +// If idx is out of range, a nil node and an error will be returned. +func (n *ProtoNode) LookupByIndex(idx int64) (ipld.Node, error) { + _, err := n.EncodeProtobuf(false) + if err != nil { + return nil, err + } + return n.encoded.LookupByIndex(idx) +} + +// LookupBySegment is will act as either LookupByString or LookupByIndex, +// whichever is contextually appropriate. +// +// Using LookupBySegment may imply an "atoi" conversion if used on a list node, +// or an "itoa" conversion if used on a map node. If an "itoa" conversion +// takes place, it may error, and this method may return that error. +func (n *ProtoNode) LookupBySegment(seg ipld.PathSegment) (ipld.Node, error) { + _, err := n.EncodeProtobuf(false) + if err != nil { + return nil, err + } + return n.encoded.LookupBySegment(seg) +} + +// Note that when using codegenerated types, there may be a fifth variant +// of lookup method on maps: `Get($GeneratedTypeKey) $GeneratedTypeValue`! +// MapIterator returns an iterator which yields key-value pairs +// traversing the node. +// If the node kind is anything other than a map, nil will be returned. +// +// The iterator will yield every entry in the map; that is, it +// can be expected that itr.Next will be called node.Length times +// before itr.Done becomes true. +func (n *ProtoNode) MapIterator() ipld.MapIterator { + _, _ = n.EncodeProtobuf(false) + return n.encoded.MapIterator() +} + +// ListIterator returns an iterator which yields key-value pairs +// traversing the node. +// If the node kind is anything other than a list, nil will be returned. +// +// The iterator will yield every entry in the list; that is, it +// can be expected that itr.Next will be called node.Length times +// before itr.Done becomes true. +func (n *ProtoNode) ListIterator() ipld.ListIterator { + _, _ = n.EncodeProtobuf(false) + return n.encoded.ListIterator() +} + +// Length returns the length of a list, or the number of entries in a map, +// or -1 if the node is not of list nor map kind. +func (n *ProtoNode) Length() int64 { + _, _ = n.EncodeProtobuf(false) + return n.encoded.Length() +} + +// Absent nodes are returned when traversing a struct field that is +// defined by a schema but unset in the data. (Absent nodes are not +// possible otherwise; you'll only see them from `schema.TypedNode`.) +// The absent flag is necessary so iterating over structs can +// unambiguously make the distinction between values that are +// present-and-null versus values that are absent. +// +// Absent nodes respond to `Kind()` as `ipld.Kind_Null`, +// for lack of any better descriptive value; you should therefore +// always check IsAbsent rather than just a switch on kind +// when it may be important to handle absent values distinctly. +func (n *ProtoNode) IsAbsent() bool { + _, _ = n.EncodeProtobuf(false) + return n.encoded.IsAbsent() +} + +func (n *ProtoNode) IsNull() bool { + _, _ = n.EncodeProtobuf(false) + return n.encoded.IsNull() +} + +func (n *ProtoNode) AsBool() (bool, error) { + _, err := n.EncodeProtobuf(false) + if err != nil { + return false, err + } + return n.encoded.AsBool() +} + +func (n *ProtoNode) AsInt() (int64, error) { + _, err := n.EncodeProtobuf(false) + if err != nil { + return 0, err + } + return n.encoded.AsInt() +} + +func (n *ProtoNode) AsFloat() (float64, error) { + _, err := n.EncodeProtobuf(false) + if err != nil { + return 0, err + } + return n.encoded.AsFloat() +} + +func (n *ProtoNode) AsString() (string, error) { + _, err := n.EncodeProtobuf(false) + if err != nil { + return "", err + } + return n.encoded.AsString() +} + +func (n *ProtoNode) AsBytes() ([]byte, error) { + _, err := n.EncodeProtobuf(false) + if err != nil { + return nil, err + } + return n.encoded.AsBytes() +} + +func (n *ProtoNode) AsLink() (ipld.Link, error) { + _, err := n.EncodeProtobuf(false) + if err != nil { + return nil, err + } + return n.encoded.AsLink() +} + +// Prototype returns a NodePrototype which can describe some properties of this node's implementation, +// and also be used to get a NodeBuilder, +// which can be use to create new nodes with the same implementation as this one. +// +// For typed nodes, the NodePrototype will also implement schema.Type. +// +// For Advanced Data Layouts, the NodePrototype will encapsulate any additional +// parameters and configuration of the ADL, and will also (usually) +// implement NodePrototypeSupportingAmend. +// +// Calling this method should not cause an allocation. +func (n *ProtoNode) Prototype() ipld.NodePrototype { + return dagpb.Type.PBNode +} + +var _ ipld.Node = &ProtoNode{} diff --git a/raw.go b/raw.go index a0adb4a..f4e9e53 100644 --- a/raw.go +++ b/raw.go @@ -3,37 +3,48 @@ package merkledag import ( "encoding/json" "fmt" - "github.com/ipfs/go-block-format" - cid "github.com/ipfs/go-cid" + blocks "github.com/ipfs/go-block-format" u "github.com/ipfs/go-ipfs-util" - ipld "github.com/ipfs/go-ipld-format" + legacy "github.com/ipfs/go-ipld-legacy" + ipld "github.com/ipld/go-ipld-prime" + basicnode "github.com/ipld/go-ipld-prime/node/basic" + + cid "github.com/ipfs/go-cid" + format "github.com/ipfs/go-ipld-format" ) // RawNode represents a node which only contains data. type RawNode struct { blocks.Block + + // Always a node/basic Bytes. + // We can't reference a specific type, as it's not exposed there. + // If we find that the interface indirection really matters, + // then we could possibly use dagpb.Bytes. + ipld.Node } +var _ legacy.UniversalNode = &RawNode{} + // NewRawNode creates a RawNode using the default sha2-256 hash function. func NewRawNode(data []byte) *RawNode { h := u.Hash(data) c := cid.NewCidV1(cid.Raw, h) blk, _ := blocks.NewBlockWithCid(data, c) - - return &RawNode{blk} + return &RawNode{blk, basicnode.NewBytes(data)} } // DecodeRawBlock is a block decoder for raw IPLD nodes conforming to `node.DecodeBlockFunc`. -func DecodeRawBlock(block blocks.Block) (ipld.Node, error) { +func DecodeRawBlock(block blocks.Block) (format.Node, error) { if block.Cid().Type() != cid.Raw { return nil, fmt.Errorf("raw nodes cannot be decoded from non-raw blocks: %d", block.Cid().Type()) } // Once you "share" a block, it should be immutable. Therefore, we can just use this block as-is. - return &RawNode{block}, nil + return &RawNode{block, basicnode.NewBytes(block.RawData())}, nil } -var _ ipld.DecodeBlockFunc = DecodeRawBlock +var _ format.DecodeBlockFunc = DecodeRawBlock // NewRawNodeWPrefix creates a RawNode using the provided cid builder func NewRawNodeWPrefix(data []byte, builder cid.Builder) (*RawNode, error) { @@ -46,16 +57,17 @@ func NewRawNodeWPrefix(data []byte, builder cid.Builder) (*RawNode, error) { if err != nil { return nil, err } - return &RawNode{blk}, nil + // Once you "share" a block, it should be immutable. Therefore, we can just use this block as-is. + return &RawNode{blk, basicnode.NewBytes(data)}, nil } // Links returns nil. -func (rn *RawNode) Links() []*ipld.Link { +func (rn *RawNode) Links() []*format.Link { return nil } // ResolveLink returns an error. -func (rn *RawNode) ResolveLink(path []string) (*ipld.Link, []string, error) { +func (rn *RawNode) ResolveLink(path []string) (*format.Link, []string, error) { return nil, nil, ErrLinkNotFound } @@ -69,8 +81,8 @@ func (rn *RawNode) Tree(p string, depth int) []string { return nil } -// Copy performs a deep copy of this node and returns it as an ipld.Node -func (rn *RawNode) Copy() ipld.Node { +// Copy performs a deep copy of this node and returns it as an format.Node +func (rn *RawNode) Copy() format.Node { copybuf := make([]byte, len(rn.RawData())) copy(copybuf, rn.RawData()) nblk, err := blocks.NewBlockWithCid(rn.RawData(), rn.Cid()) @@ -78,8 +90,8 @@ func (rn *RawNode) Copy() ipld.Node { // programmer error panic("failure attempting to clone raw block: " + err.Error()) } - - return &RawNode{nblk} + // Once you "share" a block, it should be immutable. Therefore, we can just use this block as-is. + return &RawNode{nblk, basicnode.NewBytes(nblk.RawData())} } // Size returns the size of this node @@ -88,8 +100,8 @@ func (rn *RawNode) Size() (uint64, error) { } // Stat returns some Stats about this node. -func (rn *RawNode) Stat() (*ipld.NodeStat, error) { - return &ipld.NodeStat{ +func (rn *RawNode) Stat() (*format.NodeStat, error) { + return &format.NodeStat{ CumulativeSize: len(rn.RawData()), DataSize: len(rn.RawData()), }, nil @@ -100,4 +112,11 @@ func (rn *RawNode) MarshalJSON() ([]byte, error) { return json.Marshal(string(rn.RawData())) } -var _ ipld.Node = (*RawNode)(nil) +func RawNodeConverter(b blocks.Block, nd ipld.Node) (legacy.UniversalNode, error) { + if nd.Kind() != ipld.Kind_Bytes { + return nil, ErrNotRawNode + } + return &RawNode{b, nd}, nil +} + +var _ legacy.UniversalNode = (*RawNode)(nil)