From f7d408d502eb77f133c4106025ba1389e90fdb3e Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Tue, 18 Sep 2018 19:39:54 -0400 Subject: [PATCH] Enable --cid-base for more commands. Move APICid and Encoder to go-cidutil. License: MIT Signed-off-by: Kevin Atkinson --- core/apicid.go | 131 ----------------------------- core/commands/add.go | 4 +- core/commands/dns.go | 2 +- core/commands/files.go | 19 ++++- core/commands/filestore.go | 20 ++++- core/commands/ls.go | 7 +- core/commands/pin.go | 22 +++-- core/commands/refs.go | 30 ++++++- core/commands/resolve.go | 9 +- core/commands/root.go | 70 ++++++++++----- core/commands/tar.go | 5 +- core/coreapi/dht.go | 2 +- core/coreunix/add.go | 7 +- core/coreunix/add_test.go | 5 +- exchange/reprovide/providers.go | 2 +- filestore/util.go | 7 +- package.json | 4 +- test/sharness/t0040-add-and-cat.sh | 29 +++++++ test/sharness/t0045-ls.sh | 10 ++- test/sharness/t0085-pins.sh | 7 +- test/sharness/t0160-resolve.sh | 32 ++++++- thirdparty/cidv0v1/blockstore.go | 22 +---- 22 files changed, 232 insertions(+), 214 deletions(-) delete mode 100644 core/apicid.go diff --git a/core/apicid.go b/core/apicid.go deleted file mode 100644 index 7ff823134947..000000000000 --- a/core/apicid.go +++ /dev/null @@ -1,131 +0,0 @@ -package core - -import ( - "encoding/json" - - cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid" - mbase "gx/ipfs/QmekxXDhCxCJRNuzmHreuaT3BsuJcsjcXWNrtV9C8DRHtd/go-multibase" - //path "gx/ipfs/QmX7uSbkNz76yNwBhuwYwRbhihLnJqM73VTCjS3UMJud9A/go-path" -) - -// CidEncoder is a type used to encode or recode Cid as the user -// specifies -type CidEncoder interface { - Encode(c cid.Cid) string - Recode(v string) (string, error) -} - -// BasicCidEncoder is a basic CidEncoder that will encode Cid's using -// a specifed base, optionally upgrading a Cid if is Version 0 -type BasicCidEncoder struct { - Base mbase.Encoder - Upgrade bool -} - -var DefaultCidEncoder = BasicCidEncoder{ - Base: mbase.MustNewEncoder(mbase.Base58BTC), - Upgrade: false, -} - -// CidJSONBase is the base to use when Encoding into JSON. -//var CidJSONBase mbase.Encoder = mbase.MustNewEncoder(mbase.Base58BTC) -var CidJSONBase mbase.Encoder = mbase.MustNewEncoder(mbase.Base32) - -// APICid is a type to respesnt CID in the API -type APICid struct { - str string // always in CidJSONBase -} - -// FromCid created an APICid from a Cid -func FromCid(c cid.Cid) APICid { - return APICid{c.Encode(CidJSONBase)} -} - -// Cid converts an APICid to a CID -func (c APICid) Cid() (cid.Cid, error) { - return cid.Decode(c.str) -} - -func (c APICid) String() string { - return c.Encode(DefaultCidEncoder) -} - -func (c APICid) Encode(enc CidEncoder) string { - if c.str == "" { - return "" - } - str, err := enc.Recode(c.str) - if err != nil { - return c.str - } - return str -} - -func (c *APICid) UnmarshalJSON(b []byte) error { - return json.Unmarshal(b, &c.str) -} - -func (c APICid) MarshalJSON() ([]byte, error) { - return json.Marshal(c.str) -} - -func (enc BasicCidEncoder) Encode(c cid.Cid) string { - if c.Version() == 0 { - c = cid.NewCidV1(c.Type(), c.Hash()) - } - return c.Encode(enc.Base) -} - -func (enc BasicCidEncoder) Recode(v string) (string, error) { - skip, err := enc.NoopRecode(v) - if skip || err != nil { - return v, err - } - - c, err := cid.Decode(v) - if err != nil { - return v, err - } - - return enc.Encode(c), nil -} - -func (enc BasicCidEncoder) NoopRecode(v string) (bool, error) { - if len(v) < 2 { - return false, cid.ErrCidTooShort - } - ver := cidVer(v) - skip := ver == 0 && !enc.Upgrade || ver == 1 && v[0] == byte(enc.Base.Encoding()) - return skip, nil -} - -func cidVer(v string) int { - if len(v) == 46 && v[:2] == "Qm" { - return 0 - } else { - return 1 - } -} - -// func (enc *CidEncoder) Scan(cids ...string) { -// if enc.Override == nil { -// enc.Override = map[cid.Cid]string{} -// } -// for _, p := range cids { -// //segs := path.FromString(p).Segments() -// //v := segs[0] -// //if v == "ipfs" && len(segs) > 0 { -// // v = segs[1] -// //} -// v := p -// skip, err := enc.noopRecode(v) -// if skip || err != nil { -// continue -// } -// c, err := cid.Decode(v) -// if err != nil { -// continue -// } -// enc.Override[c] = v -// } -// } diff --git a/core/commands/add.go b/core/commands/add.go index 17b76ea81c46..0f62e4d964d7 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -19,10 +19,10 @@ import ( cmds "gx/ipfs/QmPTfgFTo9PFr1PvPKyKoeMgBvYPh6cX3aDP7DHKVbnCbi/go-ipfs-cmds" mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" pb "gx/ipfs/QmPtj12fdwuAqj9sBSTNUxBNu8kCGNp8b3o8yUzMm5GHpq/pb" - cidutil "gx/ipfs/QmQJSeE3CX4zos9qeaG8EhecEK9zvrTEfTG84J8C5NVRwt/go-cidutil" mfs "gx/ipfs/QmRkrpnhZqDxTxwGCsDbuZMr7uCFZHH6SGfrcjgEQwxF3t/go-mfs" cmdkit "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit" files "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files" + cidutil "gx/ipfs/QmSwrzRygK8yHBzd8gJJYedttQVQTZJrbEbpEBYvQLaRy8/go-cidutil" offline "gx/ipfs/QmcRC35JF2pJQneAxa5LdQBQRumWggccWErogSrCkS1h8T/go-ipfs-exchange-offline" bstore "gx/ipfs/QmegPGspn3RpTMQ23Fd3GVVMopo1zsEMurudbFMZ5UXBLH/go-ipfs-blockstore" ) @@ -390,7 +390,7 @@ You can now check what blocks have been created by: log.Warning("cannot determine size of input file") } - err := HandleCidBase(req) + _, err := NewCidBaseHandler(req).UseGlobal().Proc() if err != nil { re.SetError(err, cmdkit.ErrNormal) return re diff --git a/core/commands/dns.go b/core/commands/dns.go index 2353ee7ec653..2f2e7800e8d0 100644 --- a/core/commands/dns.go +++ b/core/commands/dns.go @@ -54,7 +54,6 @@ The resolver can recursively resolve: cmdkit.BoolOption("recursive", "r", "Resolve until the result is not a DNS link."), }, Run: func(req cmds.Request, res cmds.Response) { - recursive, _, _ := req.Option("recursive").Bool() name := req.Arguments()[0] resolver := namesys.NewDNSResolver() @@ -77,6 +76,7 @@ The resolver can recursively resolve: }, Marshalers: cmds.MarshalerMap{ cmds.Text: func(res cmds.Response) (io.Reader, error) { + v, err := unwrapOutput(res.Output()) if err != nil { return nil, err diff --git a/core/commands/files.go b/core/commands/files.go index a855d09925bf..aaf0b0700ca8 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -30,6 +30,7 @@ import ( logging "gx/ipfs/QmRREK2CAZ5Re2Bd9zZFG6FeYDppUWt5cMgsoUEp3ktgSr/go-log" mfs "gx/ipfs/QmRkrpnhZqDxTxwGCsDbuZMr7uCFZHH6SGfrcjgEQwxF3t/go-mfs" cmdkit "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit" + apicid "gx/ipfs/QmSwrzRygK8yHBzd8gJJYedttQVQTZJrbEbpEBYvQLaRy8/go-cidutil/apicid" offline "gx/ipfs/QmcRC35JF2pJQneAxa5LdQBQRumWggccWErogSrCkS1h8T/go-ipfs-exchange-offline" ipld "gx/ipfs/QmdDXJs4axxefSPgK6Y1QhpJWKuDPnGJiqgq4uncb4rFHL/go-ipld-format" ) @@ -77,7 +78,7 @@ var hashOption = cmdkit.StringOption("hash", "Hash function to use. Will set Cid var errFormat = errors.New("format was set by multiple options. Only one format option is allowed") type statOutput struct { - Hash string + Hash apicid.Hash Size uint64 CumulativeSize uint64 Blocks int @@ -167,13 +168,18 @@ var filesStatCmd = &cmds.Command{ }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error { + _, err := NewCidBaseHandler(req).UseGlobal().Proc() + if err != nil { + return err + } + out, ok := v.(*statOutput) if !ok { return e.TypeErr(out, v) } s, _ := statGetFormatOptions(req) - s = strings.Replace(s, "", out.Hash, -1) + s = strings.Replace(s, "", out.Hash.String(), -1) s = strings.Replace(s, "", fmt.Sprintf("%d", out.Size), -1) s = strings.Replace(s, "", fmt.Sprintf("%d", out.CumulativeSize), -1) s = strings.Replace(s, "", fmt.Sprintf("%d", out.Blocks), -1) @@ -244,7 +250,7 @@ func statNode(nd ipld.Node) (*statOutput, error) { } return &statOutput{ - Hash: c.String(), + Hash: apicid.FromCid(c), Blocks: len(nd.Links()), Size: d.GetFilesize(), CumulativeSize: cumulsize, @@ -252,7 +258,7 @@ func statNode(nd ipld.Node) (*statOutput, error) { }, nil case *dag.RawNode: return &statOutput{ - Hash: c.String(), + Hash: apicid.FromCid(c), Blocks: 0, Size: cumulsize, CumulativeSize: cumulsize, @@ -491,6 +497,11 @@ Examples: }, Marshalers: oldcmds.MarshalerMap{ oldcmds.Text: func(res oldcmds.Response) (io.Reader, error) { + _, err := NewCidBaseHandlerLegacy(res.Request()).UseGlobal().Proc() + if err != nil { + return nil, err + } + v, err := unwrapOutput(res.Output()) if err != nil { return nil, err diff --git a/core/commands/filestore.go b/core/commands/filestore.go index 3fa5c9a37bcd..e4357352766d 100644 --- a/core/commands/filestore.go +++ b/core/commands/filestore.go @@ -84,6 +84,12 @@ The output is: cmds.CLI: func(req *cmds.Request, re cmds.ResponseEmitter) cmds.ResponseEmitter { reNext, res := cmds.NewChanResponsePair(req) + _, err := NewCidBaseHandler(req).UseGlobal().Proc() + if err != nil { + re.SetError(err, cmdkit.ErrNormal) + return reNext + } + go func() { defer re.Close() @@ -174,6 +180,11 @@ For ERROR entries the error will also be printed to stderr. }, Marshalers: oldCmds.MarshalerMap{ oldCmds.Text: func(res oldCmds.Response) (io.Reader, error) { + _, err := NewCidBaseHandlerLegacy(res.Request()).UseGlobal().Proc() + if err != nil { + return nil, err + } + v, err := unwrapOutput(res.Output()) if err != nil { return nil, err @@ -210,6 +221,13 @@ var dupsFileStore = &oldCmds.Command{ return } + h, err := NewCidBaseHandlerLegacy(req).Proc() + if err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } + enc := h.Encoder() + out := make(chan interface{}, 128) res.SetOutput((<-chan interface{})(out)) @@ -226,7 +244,7 @@ var dupsFileStore = &oldCmds.Command{ } if have { select { - case out <- &RefWrapper{Ref: cid.String()}: + case out <- &RefWrapper{Ref: enc.Encode(cid)}: case <-req.Context().Done(): return } diff --git a/core/commands/ls.go b/core/commands/ls.go index 201253546a24..c36d4c2ede66 100644 --- a/core/commands/ls.go +++ b/core/commands/ls.go @@ -19,13 +19,14 @@ import ( cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid" "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit" + apicid "gx/ipfs/QmSwrzRygK8yHBzd8gJJYedttQVQTZJrbEbpEBYvQLaRy8/go-cidutil/apicid" offline "gx/ipfs/QmcRC35JF2pJQneAxa5LdQBQRumWggccWErogSrCkS1h8T/go-ipfs-exchange-offline" ipld "gx/ipfs/QmdDXJs4axxefSPgK6Y1QhpJWKuDPnGJiqgq4uncb4rFHL/go-ipld-format" ) type LsLink struct { Name string - Hash core.APICid + Hash apicid.Hash Size uint64 Type unixfspb.Data_DataType } @@ -161,7 +162,7 @@ The JSON output contains type information. } output[i].Links[j] = LsLink{ Name: link.Name, - Hash: core.FromCid(link.Cid), + Hash: apicid.FromCid(link.Cid), Size: link.Size, Type: t, } @@ -172,7 +173,7 @@ The JSON output contains type information. }, Marshalers: cmds.MarshalerMap{ cmds.Text: func(res cmds.Response) (io.Reader, error) { - err := HandleCidBaseLegacy(res.Request()) + _, err := NewCidBaseHandlerLegacy(res.Request()).UseGlobal().Proc() if err != nil { return nil, err } diff --git a/core/commands/pin.go b/core/commands/pin.go index 5449c7c9d782..dd43a9681b65 100644 --- a/core/commands/pin.go +++ b/core/commands/pin.go @@ -16,6 +16,7 @@ import ( cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid" "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit" + apicid "gx/ipfs/QmSwrzRygK8yHBzd8gJJYedttQVQTZJrbEbpEBYvQLaRy8/go-cidutil/apicid" "gx/ipfs/QmVkMRSkXrpjqrroEXWuYBvDBnXCdMMY6gsKicBGVGUqKT/go-verifcid" path "gx/ipfs/QmX7uSbkNz76yNwBhuwYwRbhihLnJqM73VTCjS3UMJud9A/go-path" resolver "gx/ipfs/QmX7uSbkNz76yNwBhuwYwRbhihLnJqM73VTCjS3UMJud9A/go-path/resolver" @@ -39,11 +40,11 @@ var PinCmd = &cmds.Command{ } type PinOutput struct { - Pins []core.APICid + Pins []apicid.Hash } type AddPinOutput struct { - Pins []core.APICid + Pins []apicid.Hash Progress int `json:",omitempty"` } @@ -130,7 +131,7 @@ var addPinCmd = &cmds.Command{ }, Marshalers: cmds.MarshalerMap{ cmds.Text: func(res cmds.Response) (io.Reader, error) { - err := HandleCidBaseLegacy(res.Request()) + _, err := NewCidBaseHandlerLegacy(res.Request()).UseGlobal().Proc() if err != nil { return nil, err } @@ -139,7 +140,7 @@ var addPinCmd = &cmds.Command{ return nil, err } - var added []core.APICid + var added []apicid.Hash switch out := v.(type) { case *AddPinOutput: @@ -214,10 +215,11 @@ collected if needed. (By default, recursively. Use -r=false for direct pins.) }, Marshalers: cmds.MarshalerMap{ cmds.Text: func(res cmds.Response) (io.Reader, error) { - err := HandleCidBaseLegacy(res.Request()) + _, err := NewCidBaseHandlerLegacy(res.Request()).UseGlobal().Proc() if err != nil { return nil, err } + v, err := unwrapOutput(res.Output()) if err != nil { return nil, err @@ -326,10 +328,6 @@ Example: Type: RefKeyList{}, Marshalers: cmds.MarshalerMap{ cmds.Text: func(res cmds.Response) (io.Reader, error) { - err := HandleCidBaseLegacy(res.Request()) - if err != nil { - return nil, err - } v, err := unwrapOutput(res.Output()) if err != nil { return nil, err @@ -696,10 +694,10 @@ func (r PinVerifyRes) Format(out io.Writer) { } } -func toAPICids(cs []cid.Cid) []core.APICid { - out := make([]core.APICid, 0, len(cs)) +func toAPICids(cs []cid.Cid) []apicid.Hash { + out := make([]apicid.Hash, 0, len(cs)) for _, c := range cs { - out = append(out, core.FromCid(c)) + out = append(out, apicid.FromCid(c)) } return out } diff --git a/core/commands/refs.go b/core/commands/refs.go index 05386f6a705d..ed51ffd76158 100644 --- a/core/commands/refs.go +++ b/core/commands/refs.go @@ -10,6 +10,7 @@ import ( cmds "github.com/ipfs/go-ipfs/commands" "github.com/ipfs/go-ipfs/core" e "github.com/ipfs/go-ipfs/core/commands/e" + cidenc "gx/ipfs/QmSwrzRygK8yHBzd8gJJYedttQVQTZJrbEbpEBYvQLaRy8/go-cidutil/cidenc" cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid" cmdkit "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit" @@ -117,6 +118,13 @@ NOTE: List all references recursively by using the flag '-r'. format = " -> " } + h, err := NewCidBaseHandlerLegacy(req).Proc() + if err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } + enc := h.Encoder() + objs, err := objectsForPaths(ctx, n, req.Arguments()) if err != nil { res.SetError(err, cmdkit.ErrNormal) @@ -136,6 +144,7 @@ NOTE: List all references recursively by using the flag '-r'. Unique: unique, PrintFmt: format, MaxDepth: maxDepth, + Encoder: enc, } for _, o := range objs { @@ -169,6 +178,13 @@ Displays the hashes of all local objects. return } + h, err := NewCidBaseHandlerLegacy(req).Proc() + if err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } + enc := h.Encoder() + // todo: make async allKeys, err := n.Blockstore.AllKeysChan(ctx) if err != nil { @@ -184,7 +200,7 @@ Displays the hashes of all local objects. for k := range allKeys { select { - case out <- &RefWrapper{Ref: k.String()}: + case out <- &RefWrapper{Ref: enc.Encode(k)}: case <-req.Context().Done(): return } @@ -197,6 +213,11 @@ Displays the hashes of all local objects. var refsMarshallerMap = cmds.MarshalerMap{ cmds.Text: func(res cmds.Response) (io.Reader, error) { + _, err := NewCidBaseHandlerLegacy(res.Request()).UseGlobal().Proc() + if err != nil { + return nil, err + } + v, err := unwrapOutput(res.Output()) if err != nil { return nil, err @@ -245,6 +266,7 @@ type RefWriter struct { Unique bool MaxDepth int PrintFmt string + Encoder cidenc.Interface seen map[string]int } @@ -379,11 +401,11 @@ func (rw *RefWriter) WriteEdge(from, to cid.Cid, linkname string) error { switch { case rw.PrintFmt != "": s = rw.PrintFmt - s = strings.Replace(s, "", from.String(), -1) - s = strings.Replace(s, "", to.String(), -1) + s = strings.Replace(s, "", rw.Encoder.Encode(from), -1) + s = strings.Replace(s, "", rw.Encoder.Encode(to), -1) s = strings.Replace(s, "", linkname, -1) default: - s += to.String() + s += rw.Encoder.Encode(to) } rw.out <- &RefWrapper{Ref: s} diff --git a/core/commands/resolve.go b/core/commands/resolve.go index 75258464f843..adad6a24f09c 100644 --- a/core/commands/resolve.go +++ b/core/commands/resolve.go @@ -84,6 +84,13 @@ Resolve the value of an IPFS DAG path: name := req.Arguments()[0] recursive, _, _ := req.Option("recursive").Bool() + h, err := NewCidBaseHandlerLegacy(req).Proc() + if err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } + enc := h.EncoderFromPath(name) + // the case when ipns is resolved step by step if strings.HasPrefix(name, "/ipns/") && !recursive { rc, rcok, _ := req.Option("dht-record-count").Int() @@ -129,7 +136,7 @@ Resolve the value of an IPFS DAG path: c := node.Cid() - res.SetOutput(&ncmd.ResolvedPath{Path: path.FromCid(c)}) + res.SetOutput(&ncmd.ResolvedPath{Path: path.FromString("/ipfs/" + enc.Encode(c))}) }, Marshalers: cmds.MarshalerMap{ cmds.Text: func(res cmds.Response) (io.Reader, error) { diff --git a/core/commands/root.go b/core/commands/root.go index 55aebe25367a..ed0b0104df32 100644 --- a/core/commands/root.go +++ b/core/commands/root.go @@ -7,12 +7,12 @@ import ( oldcmds "github.com/ipfs/go-ipfs/commands" lgc "github.com/ipfs/go-ipfs/commands/legacy" - core "github.com/ipfs/go-ipfs/core" dag "github.com/ipfs/go-ipfs/core/commands/dag" e "github.com/ipfs/go-ipfs/core/commands/e" name "github.com/ipfs/go-ipfs/core/commands/name" ocmd "github.com/ipfs/go-ipfs/core/commands/object" unixfs "github.com/ipfs/go-ipfs/core/commands/unixfs" + cidenc "gx/ipfs/QmSwrzRygK8yHBzd8gJJYedttQVQTZJrbEbpEBYvQLaRy8/go-cidutil/cidenc" "gx/ipfs/QmPTfgFTo9PFr1PvPKyKoeMgBvYPh6cX3aDP7DHKVbnCbi/go-ipfs-cmds" logging "gx/ipfs/QmRREK2CAZ5Re2Bd9zZFG6FeYDppUWt5cMgsoUEp3ktgSr/go-log" @@ -229,34 +229,66 @@ func MessageTextMarshaler(res oldcmds.Response) (io.Reader, error) { return strings.NewReader(out.Message), nil } -func handleCidBase(base string, upgrade bool, upgradeDefined bool) error { - enc := core.DefaultCidEncoder +type CidBaseHandler struct { + base string + upgrade bool + upgradeDefined bool + args []string + enc *cidenc.Encoder +} + +func NewCidBaseHandler(req *cmds.Request) *CidBaseHandler { + h := &CidBaseHandler{} + h.base, _ = req.Options[CidBaseOption].(string) + h.upgrade, h.upgradeDefined = req.Options[UpgradeCidV0Option].(bool) + h.args = req.Arguments + return h +} + +func NewCidBaseHandlerLegacy(req oldcmds.Request) *CidBaseHandler { + h := &CidBaseHandler{} + h.base, _, _ = req.Option(CidBaseOption).String() + h.upgrade, h.upgradeDefined, _ = req.Option(UpgradeCidV0Option).Bool() + h.args = req.Arguments() + return h +} - if base != "" { +func (h *CidBaseHandler) UseGlobal() *CidBaseHandler { + h.enc = &cidenc.Default + return h +} + +func (h *CidBaseHandler) Proc() (*CidBaseHandler, error) { + var e cidenc.Encoder = cidenc.Default + if h.base != "" { var err error - enc.Base, err = mbase.EncoderByName(base) + e.Base, err = mbase.EncoderByName(h.base) if err != nil { - return err + return h, err } } - enc.Upgrade = upgrade - if base != "" && !upgradeDefined { - enc.Upgrade = true + e.Upgrade = h.upgrade + if h.base != "" && !h.upgradeDefined { + e.Upgrade = true } - core.DefaultCidEncoder = enc - return nil + if h.enc == nil { + h.enc = &cidenc.Encoder{} + } + *h.enc = e + return h, nil } -func HandleCidBase(req *cmds.Request) error { - base, _ := req.Options[CidBaseOption].(string) - upgrade, defined := req.Options[UpgradeCidV0Option].(bool) - return handleCidBase(base, upgrade, defined) +func (h *CidBaseHandler) Encoder() cidenc.Encoder { + return *h.enc } -func HandleCidBaseLegacy(req oldcmds.Request) error { - base, _, _ := req.Option(CidBaseOption).String() - upgrade, defined, _ := req.Option(UpgradeCidV0Option).Bool() - return handleCidBase(base, upgrade, defined) +func (h *CidBaseHandler) EncoderFromPath(p string) cidenc.Encoder { + if h.base == "" { + enc, _ := h.enc.FromPath(p) + return enc + } else { + return *h.enc + } } diff --git a/core/commands/tar.go b/core/commands/tar.go index 40b979e0cf07..0e059d469d36 100644 --- a/core/commands/tar.go +++ b/core/commands/tar.go @@ -13,6 +13,7 @@ import ( dag "gx/ipfs/QmXv5mwmQ74r4aiHcNeQ4GAmfB3aWJuqaE4WyDfDfvkgLM/go-merkledag" "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit" + apicid "gx/ipfs/QmSwrzRygK8yHBzd8gJJYedttQVQTZJrbEbpEBYvQLaRy8/go-cidutil/apicid" ) var TarCmd = &cmds.Command{ @@ -62,13 +63,13 @@ represent it. fi.FileName() res.SetOutput(&coreunix.AddedObject{ Name: fi.FileName(), - Hash: core.FromCid(c), + Hash: apicid.FromCid(c), }) }, Type: coreunix.AddedObject{}, Marshalers: cmds.MarshalerMap{ cmds.Text: func(res cmds.Response) (io.Reader, error) { - err := HandleCidBaseLegacy(res.Request()) + _, err := NewCidBaseHandlerLegacy(res.Request()).UseGlobal().Proc() if err != nil { return nil, err } diff --git a/core/coreapi/dht.go b/core/coreapi/dht.go index e2fc741de71d..27d1bcb1bb41 100644 --- a/core/coreapi/dht.go +++ b/core/coreapi/dht.go @@ -9,8 +9,8 @@ import ( caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid" - cidutil "gx/ipfs/QmQJSeE3CX4zos9qeaG8EhecEK9zvrTEfTG84J8C5NVRwt/go-cidutil" peer "gx/ipfs/QmQsErDt8Qgw1XrsXf2BpEzDgGWtB1YLsTAARBup5b6B9W/go-libp2p-peer" + cidutil "gx/ipfs/QmSwrzRygK8yHBzd8gJJYedttQVQTZJrbEbpEBYvQLaRy8/go-cidutil" dag "gx/ipfs/QmXv5mwmQ74r4aiHcNeQ4GAmfB3aWJuqaE4WyDfDfvkgLM/go-merkledag" blockservice "gx/ipfs/Qma2KhbQarYTkmSJAeaMGRAg8HAXAhEWK8ge4SReG7ZSD3/go-blockservice" offline "gx/ipfs/QmcRC35JF2pJQneAxa5LdQBQRumWggccWErogSrCkS1h8T/go-ipfs-exchange-offline" diff --git a/core/coreunix/add.go b/core/coreunix/add.go index e48f6d4df0c3..6a192ac230e1 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -23,6 +23,7 @@ import ( logging "gx/ipfs/QmRREK2CAZ5Re2Bd9zZFG6FeYDppUWt5cMgsoUEp3ktgSr/go-log" mfs "gx/ipfs/QmRkrpnhZqDxTxwGCsDbuZMr7uCFZHH6SGfrcjgEQwxF3t/go-mfs" files "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files" + apicid "gx/ipfs/QmSwrzRygK8yHBzd8gJJYedttQVQTZJrbEbpEBYvQLaRy8/go-cidutil/apicid" ipld "gx/ipfs/QmdDXJs4axxefSPgK6Y1QhpJWKuDPnGJiqgq4uncb4rFHL/go-ipld-format" chunker "gx/ipfs/QmdSeG9s4EQ9TGruJJS9Us38TQDZtMmFGwzTYUDVqNTURm/go-ipfs-chunker" bstore "gx/ipfs/QmegPGspn3RpTMQ23Fd3GVVMopo1zsEMurudbFMZ5UXBLH/go-ipfs-blockstore" @@ -41,14 +42,14 @@ type Link struct { } type Object struct { - Hash core.APICid + Hash apicid.Hash Links []Link Size string } type AddedObject struct { Name string - Hash core.APICid `json:",omitempty"` + Hash apicid.Hash `json:",omitempty"` Bytes int64 `json:",omitempty"` Size string `json:",omitempty"` } @@ -564,7 +565,7 @@ func getOutput(dagnode ipld.Node) (*Object, error) { } output := &Object{ - Hash: core.FromCid(c), + Hash: apicid.FromCid(c), Size: strconv.FormatUint(s, 10), Links: make([]Link, len(dagnode.Links())), } diff --git a/core/coreunix/add_test.go b/core/coreunix/add_test.go index cb5ef45a42c2..d903fd911e96 100644 --- a/core/coreunix/add_test.go +++ b/core/coreunix/add_test.go @@ -20,6 +20,7 @@ import ( files "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files" datastore "gx/ipfs/QmSpg1CvpXQQow5ernt1gNBXaXV6yxyNqi7XoeerWfzB5w/go-datastore" syncds "gx/ipfs/QmSpg1CvpXQQow5ernt1gNBXaXV6yxyNqi7XoeerWfzB5w/go-datastore/sync" + apicid "gx/ipfs/QmSwrzRygK8yHBzd8gJJYedttQVQTZJrbEbpEBYvQLaRy8/go-cidutil/apicid" dag "gx/ipfs/QmXv5mwmQ74r4aiHcNeQ4GAmfB3aWJuqaE4WyDfDfvkgLM/go-merkledag" config "gx/ipfs/QmYVqYJTVjetcf1guieEgWpK1PZtHPytP624vKzTF1P3r2/go-ipfs-config" "gx/ipfs/Qma2KhbQarYTkmSJAeaMGRAg8HAXAhEWK8ge4SReG7ZSD3/go-blockservice" @@ -93,7 +94,7 @@ func TestAddGCLive(t *testing.T) { }() - addedHashes := make(map[core.APICid]struct{}) + addedHashes := make(map[apicid.Hash]struct{}) select { case o := <-out: addedHashes[o.(*AddedObject).Hash] = struct{}{} @@ -132,7 +133,7 @@ func TestAddGCLive(t *testing.T) { if r.Error != nil { t.Fatal(err) } - if _, ok := addedHashes[core.FromCid(r.KeyRemoved)]; ok { + if _, ok := addedHashes[apicid.FromCid(r.KeyRemoved)]; ok { t.Fatal("gc'ed a hash we just added") } } diff --git a/exchange/reprovide/providers.go b/exchange/reprovide/providers.go index 57a675d47911..e7d32e07144b 100644 --- a/exchange/reprovide/providers.go +++ b/exchange/reprovide/providers.go @@ -6,7 +6,7 @@ import ( pin "github.com/ipfs/go-ipfs/pin" cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid" - cidutil "gx/ipfs/QmQJSeE3CX4zos9qeaG8EhecEK9zvrTEfTG84J8C5NVRwt/go-cidutil" + cidutil "gx/ipfs/QmSwrzRygK8yHBzd8gJJYedttQVQTZJrbEbpEBYvQLaRy8/go-cidutil" merkledag "gx/ipfs/QmXv5mwmQ74r4aiHcNeQ4GAmfB3aWJuqaE4WyDfDfvkgLM/go-merkledag" ipld "gx/ipfs/QmdDXJs4axxefSPgK6Y1QhpJWKuDPnGJiqgq4uncb4rFHL/go-ipld-format" blocks "gx/ipfs/QmegPGspn3RpTMQ23Fd3GVVMopo1zsEMurudbFMZ5UXBLH/go-ipfs-blockstore" diff --git a/filestore/util.go b/filestore/util.go index 60a2a72d76f6..8c8b29a31c31 100644 --- a/filestore/util.go +++ b/filestore/util.go @@ -10,6 +10,7 @@ import ( cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid" ds "gx/ipfs/QmSpg1CvpXQQow5ernt1gNBXaXV6yxyNqi7XoeerWfzB5w/go-datastore" dsq "gx/ipfs/QmSpg1CvpXQQow5ernt1gNBXaXV6yxyNqi7XoeerWfzB5w/go-datastore/query" + apicid "gx/ipfs/QmSwrzRygK8yHBzd8gJJYedttQVQTZJrbEbpEBYvQLaRy8/go-cidutil/apicid" blockstore "gx/ipfs/QmegPGspn3RpTMQ23Fd3GVVMopo1zsEMurudbFMZ5UXBLH/go-ipfs-blockstore" ) @@ -60,7 +61,7 @@ func (s Status) Format() string { type ListRes struct { Status Status ErrorMsg string - Key cid.Cid + Key apicid.Cid FilePath string Offset uint64 Size uint64 @@ -269,14 +270,14 @@ func mkListRes(c cid.Cid, d *pb.DataObj, err error) *ListRes { return &ListRes{ Status: status, ErrorMsg: errorMsg, - Key: c, + Key: apicid.Cid{Cid: c}, } } return &ListRes{ Status: status, ErrorMsg: errorMsg, - Key: c, + Key: apicid.Cid{Cid: c}, FilePath: d.FilePath, Size: d.Size_, Offset: d.Offset, diff --git a/package.json b/package.json index 9922db2e6722..4bdbd4d73a3a 100644 --- a/package.json +++ b/package.json @@ -542,9 +542,9 @@ }, { "author": "kevina", - "hash": "QmQJSeE3CX4zos9qeaG8EhecEK9zvrTEfTG84J8C5NVRwt", + "hash": "QmSwrzRygK8yHBzd8gJJYedttQVQTZJrbEbpEBYvQLaRy8", "name": "go-cidutil", - "version": "0.1.1" + "version": "0.1.2" }, { "author": "lgierth", diff --git a/test/sharness/t0040-add-and-cat.sh b/test/sharness/t0040-add-and-cat.sh index ebee485bd3d9..c02116803714 100755 --- a/test/sharness/t0040-add-and-cat.sh +++ b/test/sharness/t0040-add-and-cat.sh @@ -214,6 +214,35 @@ test_add_cat_file() { echo "IPFS" > actual && test_cmp expected actual ' + # --cid-base=base32 + + test_expect_success "ipfs add --cid-base=base32 succeeds" ' + echo "Hello Worlds!" >mountdir/hello.txt && + ipfs add --cid-base=base32 mountdir/hello.txt >actual + ' + + test_expect_success "ipfs add output looks good" ' + HASH="bafybeidpq7lcjx4w5c6yr4vuthzvlav54hgxsremwk73to5ferdc2rxhai" && + echo "added $HASH hello.txt" >expected && + test_cmp expected actual + ' + + test_expect_success "ipfs add --cid-base=base32 --only-hash succeeds" ' + ipfs add --cid-base=base32 --only-hash mountdir/hello.txt > oh_actual + ' + + test_expect_success "ipfs add --only-hash output looks good" ' + test_cmp expected oh_actual + ' + + test_expect_success "ipfs cat succeeds" ' + ipfs cat "$HASH" >actual + ' + + test_expect_success "ipfs cat output looks good" ' + echo "Hello Worlds!" >expected && + test_cmp expected actual + ' } test_add_cat_5MB() { diff --git a/test/sharness/t0045-ls.sh b/test/sharness/t0045-ls.sh index 2cb46a7dde9a..b9d4c0e814aa 100755 --- a/test/sharness/t0045-ls.sh +++ b/test/sharness/t0045-ls.sh @@ -11,7 +11,6 @@ test_description="Test ls command" test_init_ipfs test_ls_cmd() { - test_expect_success "'ipfs add -r testData' succeeds" ' mkdir -p testData testData/d1 testData/d2 && echo "test" >testData/f1 && @@ -88,6 +87,15 @@ QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN 14 a EOF test_cmp expected_ls_headers actual_ls_headers ' + + test_expect_success "'ipfs ls --cid-base=base32 ' succeeds" ' + ipfs ls --cid-base=base32 $(cid-fmt -v 1 -b base32 %s QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss) >actual_ls_base32 + ' + + test_expect_success "'ipfs ls --cid-base=base32 ' output looks good" ' + cid-fmt -b base32 -v 1 --filter %s < expected_ls > expected_ls_base32 + test_cmp expected_ls_base32 actual_ls_base32 + ' } test_ls_cmd_raw_leaves() { diff --git a/test/sharness/t0085-pins.sh b/test/sharness/t0085-pins.sh index 9fac885e887e..c388386287cd 100755 --- a/test/sharness/t0085-pins.sh +++ b/test/sharness/t0085-pins.sh @@ -36,6 +36,11 @@ test_pins() { cat hashes | ipfs pin add $EXTRA_ARGS ' + #test_expect_success "'ipfs pin add $EXTRA_ARGS' output looks good" ' + # sed -e "s/^/pinned /; s/$/ recursively/" hashes > expected && + # test_cmp expected actual + #' + test_expect_success "see if verify works" ' ipfs pin verify ' @@ -51,7 +56,7 @@ test_pins() { test_expect_success "test pin update" ' ipfs pin add "$HASH_A" && - ipfs pin ls > before_update && + ipfs pin ls | tee before_update && test_should_contain "$HASH_A" before_update && test_must_fail grep -q "$HASH_B" before_update && ipfs pin update --unpin=true "$HASH_A" "$HASH_B" && diff --git a/test/sharness/t0160-resolve.sh b/test/sharness/t0160-resolve.sh index d64c0eac5b7b..f9757dcad64c 100755 --- a/test/sharness/t0160-resolve.sh +++ b/test/sharness/t0160-resolve.sh @@ -12,6 +12,9 @@ test_expect_success "resolve: prepare files" ' a_hash=$(ipfs add -q -r a | tail -n1) && b_hash=$(ipfs add -q -r a/b | tail -n1) && c_hash=$(ipfs add -q -r a/b/c | tail -n1) + a_hash_b32=$(cid-fmt -v 1 -b b %s $a_hash) + b_hash_b32=$(cid-fmt -v 1 -b b %s $b_hash) + c_hash_b32=$(cid-fmt -v 1 -b b %s $c_hash) ' test_resolve_setup_name() { @@ -41,9 +44,10 @@ test_resolve_setup_name_fail() { test_resolve() { src=$1 dst=$2 + extra=$3 test_expect_success "resolve succeeds: $src" ' - ipfs resolve -r "$src" >actual + ipfs resolve $extra -r "$src" >actual ' test_expect_success "resolved correctly: $src -> $dst" ' @@ -53,7 +57,6 @@ test_resolve() { } test_resolve_cmd() { - test_resolve "/ipfs/$a_hash" "/ipfs/$a_hash" test_resolve "/ipfs/$a_hash/b" "/ipfs/$b_hash" test_resolve "/ipfs/$a_hash/b/c" "/ipfs/$c_hash" @@ -72,6 +75,30 @@ test_resolve_cmd() { test_resolve "/ipns/$id_hash" "/ipfs/$c_hash" } +test_resolve_cmd_b32() { + # no flags needed, base should be preserved + + test_resolve "/ipfs/$a_hash_b32" "/ipfs/$a_hash_b32" + test_resolve "/ipfs/$a_hash_b32/b" "/ipfs/$b_hash_b32" + test_resolve "/ipfs/$a_hash_b32/b/c" "/ipfs/$c_hash_b32" + test_resolve "/ipfs/$b_hash_b32/c" "/ipfs/$c_hash_b32" + + # flags needed passed in path does not contain cid to derive base + + test_resolve_setup_name "/ipfs/$a_hash_b32" + test_resolve "/ipns/$id_hash" "/ipfs/$a_hash_b32" --cid-base=base32 + test_resolve "/ipns/$id_hash/b" "/ipfs/$b_hash_b32" --cid-base=base32 + test_resolve "/ipns/$id_hash/b/c" "/ipfs/$c_hash_b32" --cid-base=base32 + + test_resolve_setup_name "/ipfs/$b_hash_b32" --cid-base=base32 + test_resolve "/ipns/$id_hash" "/ipfs/$b_hash_b32" --cid-base=base32 + test_resolve "/ipns/$id_hash/c" "/ipfs/$c_hash_b32" --cid-base=base32 + + test_resolve_setup_name "/ipfs/$c_hash_b32" + test_resolve "/ipns/$id_hash" "/ipfs/$c_hash_b32" --cid-base=base32 +} + + #todo remove this once the online resolve is fixed test_resolve_fail() { src=$1 @@ -108,6 +135,7 @@ test_resolve_cmd_fail() { # should work offline test_resolve_cmd +test_resolve_cmd_b32 # should work online test_launch_ipfs_daemon diff --git a/thirdparty/cidv0v1/blockstore.go b/thirdparty/cidv0v1/blockstore.go index 3f0f968cf0ae..f094ce65c2ba 100644 --- a/thirdparty/cidv0v1/blockstore.go +++ b/thirdparty/cidv0v1/blockstore.go @@ -2,8 +2,8 @@ package cidv0v1 import ( cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid" - mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" blocks "gx/ipfs/QmRcHuYzAyswytBuMF78rj3LTChYszomRFXNg4685ZN1WM/go-block-format" + cidutil "gx/ipfs/QmSwrzRygK8yHBzd8gJJYedttQVQTZJrbEbpEBYvQLaRy8/go-cidutil" bs "gx/ipfs/QmegPGspn3RpTMQ23Fd3GVVMopo1zsEMurudbFMZ5UXBLH/go-ipfs-blockstore" ) @@ -20,7 +20,7 @@ func (b *blockstore) Has(c cid.Cid) (bool, error) { if have || err != nil { return have, err } - c1 := tryOtherCidVersion(c) + c1 := cidutil.TryOtherCidVersion(c) if !c1.Defined() { return false, nil } @@ -35,7 +35,7 @@ func (b *blockstore) Get(c cid.Cid) (blocks.Block, error) { if err != bs.ErrNotFound { return nil, err } - c1 := tryOtherCidVersion(c) + c1 := cidutil.TryOtherCidVersion(c) if !c1.Defined() { return nil, bs.ErrNotFound } @@ -65,23 +65,9 @@ func (b *blockstore) GetSize(c cid.Cid) (int, error) { if err != bs.ErrNotFound { return -1, err } - c1 := tryOtherCidVersion(c) + c1 := cidutil.TryOtherCidVersion(c) if !c1.Defined() { return -1, bs.ErrNotFound } return b.Blockstore.GetSize(c1) } - -func tryOtherCidVersion(c cid.Cid) cid.Cid { - prefix := c.Prefix() - if prefix.Codec != cid.DagProtobuf || prefix.MhType != mh.SHA2_256 || prefix.MhLength != 32 { - return cid.Undef - } - var c1 cid.Cid - if prefix.Version == 0 { - c1 = cid.NewCidV1(cid.DagProtobuf, c.Hash()) - } else { - c1 = cid.NewCidV0(c.Hash()) - } - return c1 -}