feat: bring file interface into package

I've elected to bring the file interface into this package itself to decouple from the ipfs codebase. I'm still thinking we'll track pretty closely with the IPFS definition of a file, but reduces the number of imports & provides greater islolation between the two codebases. This commit also brings a bunch of documentation fixes & updates in preparation for open sourcing.

.github/CONTRIBUTING.md

@@ -0,0 +1,5 @@
+# Contribute
+
+cafs is MIT licensed open source software. If you want to contribute, we're here to help you succeed.
+
+Please make sure to check out the [issues](https://github.com/qri-io/cafs/issues). Search the closed ones before reporting things, and (if you can!) help us with open ones.

castore.go

@@ -9,7 +9,6 @@ import (
 	"fmt"
 
 	"github.com/ipfs/go-datastore"
-	"github.com/ipfs/go-ipfs/commands/files"
 )
 
 var (
@@ -22,14 +21,14 @@ var (
 // toward compatibility with git (git-scm.com), then maybe other stuff, who knows.
 type Filestore interface {
 	// put places a raw slice of bytes. Expect this to change to something like:
-	// Put(file files.File, options map[string]interface{}) (key datastore.Key, err error)
+	// Put(file File, options map[string]interface{}) (key datastore.Key, err error)
 	// The most notable difference from a standard file store is the store itself determines
 	// the resulting key (google "content addressing" for more info ;)
-	Put(file files.File, pin bool) (key datastore.Key, err error)
+	Put(file File, pin bool) (key datastore.Key, err error)
 
 	// Get retrieves the object `value` named by `key`.
 	// Get will return ErrNotFound if the key is not mapped to a value.
-	Get(key datastore.Key) (file files.File, err error)
+	Get(key datastore.Key) (file File, err error)
 
 	// Has returns whether the `key` is mapped to a `value`.
 	// In some contexts, it may be much cheaper only to check for existence of
@@ -54,7 +53,7 @@ type Filestore interface {
 // filestores can opt into the fetcher interface
 type Fetcher interface {
 	// Fetch gets a file from a source
-	Fetch(source Source, key datastore.Key) (files.SizeFile, error)
+	Fetch(source Source, key datastore.Key) (SizeFile, error)
 }
 
 // Source identifies where a file should come from.
@@ -90,7 +89,7 @@ type Adder interface {
 	// AddFile adds a file or directory of files to the store
 	// this function will return immideately, consumers should read
 	// from the Added() channel to see the results of file addition.
-	AddFile(files.File) error
+	AddFile(File) error
 	// Added gives a channel to read added files from.
 	Added() chan AddedFile
 	// In IPFS land close calls adder.Finalize() and adder.PinRoot()
@@ -110,7 +109,7 @@ type AddedFile struct {
 }
 
 // Walk traverses a file tree calling visit on each node
-func Walk(root files.File, depth int, visit func(f files.File, depth int) error) (err error) {
+func Walk(root File, depth int, visit func(f File, depth int) error) (err error) {
 	if err := visit(root, depth); err != nil {
 		return err
 	}

file.go

@@ -0,0 +1,64 @@
+// This file is pulled from github.com/ipfs/go-ipfs/commands/files
+// It's hoisted here to generalize for all cafs implementations.
+package cafs
+
+import (
+	"errors"
+	"io"
+	"os"
+)
+
+var (
+	ErrNotDirectory = errors.New("Couldn't call NextFile(), this isn't a directory")
+	ErrNotReader    = errors.New("This file is a directory, can't use Reader functions")
+)
+
+// File is an interface that provides functionality for handling
+// files/directories as values that can be supplied to commands. For
+// directories, child files are accessed serially by calling `NextFile()`.
+type File interface {
+	// Files implement ReadCloser, but can only be read from or closed if
+	// they are not directories
+	io.ReadCloser
+
+	// FileName returns a filename associated with this file
+	FileName() string
+
+	// FullPath returns the full path used when adding this file
+	FullPath() string
+
+	// IsDirectory returns true if the File is a directory (and therefore
+	// supports calling `NextFile`) and false if the File is a normal file
+	// (and therefor supports calling `Read` and `Close`)
+	IsDirectory() bool
+
+	// NextFile returns the next child file available (if the File is a
+	// directory). It will return (nil, io.EOF) if no more files are
+	// available. If the file is a regular file (not a directory), NextFile
+	// will return a non-nil error.
+	NextFile() (File, error)
+}
+
+type StatFile interface {
+	File
+
+	Stat() os.FileInfo
+}
+
+type PeekFile interface {
+	SizeFile
+
+	Peek(n int) File
+	Length() int
+}
+
+type SizeFile interface {
+	File
+
+	Size() (int64, error)
+}
+
+type FileInfo interface {
+	AbsPath() string
+	Stat() os.FileInfo
+}

memfs/mem_filestore.go

@@ -8,7 +8,6 @@ import (
 	"strings"
 
 	"github.com/ipfs/go-datastore"
-	"github.com/ipfs/go-ipfs/commands/files"
 	"github.com/jbenet/go-base58"
 	"github.com/multiformats/go-multihash"
 	"github.com/qri-io/cafs"
@@ -28,15 +27,15 @@ func (m MapStore) Tree() string {
 	for path, file := range m {
 		buf.WriteString(path.String() + "\n")
 		// fmt.Println(path, file)
-		cafs.Walk(file.File(), 0, func(f files.File, depth int) error {
+		cafs.Walk(file.File(), 0, func(f cafs.File, depth int) error {
 			buf.WriteString(strings.Repeat("  ", depth+1) + f.FileName() + "\n")
 			return nil
 		})
 	}
 	return buf.String()
 }
 
-func (m MapStore) Put(file files.File, pin bool) (key datastore.Key, err error) {
+func (m MapStore) Put(file cafs.File, pin bool) (key datastore.Key, err error) {
 	if file.IsDirectory() {
 		buf := bytes.NewBuffer(nil)
 		dir := fsDir{
@@ -96,7 +95,7 @@ func (m MapStore) Put(file files.File, pin bool) (key datastore.Key, err error)
 	return
 }
 
-func (m MapStore) Get(key datastore.Key) (files.File, error) {
+func (m MapStore) Get(key datastore.Key) (cafs.File, error) {
 	if m[key] == nil {
 		return nil, datastore.ErrNotFound
 	}
@@ -130,7 +129,7 @@ type adder struct {
 	out      chan cafs.AddedFile
 }
 
-func (a *adder) AddFile(f files.File) error {
+func (a *adder) AddFile(f cafs.File) error {
 	path, err := a.mapstore.Put(f, a.pin)
 	if err != nil {
 		fmt.Errorf("error putting file in mapstore: %s", err.Error())
@@ -174,7 +173,7 @@ type fsFile struct {
 	data []byte
 }
 
-func (f fsFile) File() files.File {
+func (f fsFile) File() cafs.File {
 	return &Memfile{
 		name: f.name,
 		path: f.path,
@@ -188,8 +187,8 @@ type fsDir struct {
 	files []datastore.Key
 }
 
-func (f fsDir) File() files.File {
-	files := make([]files.File, len(f.files))
+func (f fsDir) File() cafs.File {
+	files := make([]cafs.File, len(f.files))
 	for i, path := range f.files {
 		file, err := f.store.Get(path)
 		if err != nil {
@@ -205,5 +204,5 @@ func (f fsDir) File() files.File {
 }
 
 type filer interface {
-	File() files.File
+	File() cafs.File
 }

memfs/mem_filestore_test.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"fmt"
 	"github.com/ipfs/go-datastore"
-	"github.com/ipfs/go-ipfs/commands/files"
 	"github.com/qri-io/cafs"
 	"io/ioutil"
 	"testing"
@@ -103,7 +102,7 @@ func Directory(f cafs.Filestore) error {
 	}
 
 	paths := []string{}
-	cafs.Walk(outf, 0, func(f files.File, depth int) error {
+	cafs.Walk(outf, 0, func(f cafs.File, depth int) error {
 		paths = append(paths, f.FullPath())
 		return nil
 	})

memfs/memfile.go

@@ -1,6 +1,6 @@
-// memfs satsfies the ipfs files.File interface in memory
-// An example pulled from tests will create a tree of "files"
-// with directories & files, with paths properly set:
+// memfs satsfies the ipfs cafs.File interface in memory
+// An example pulled from tests will create a tree of "cafs"
+// with directories & cafs, with paths properly set:
 // NewMemdir("/a",
 // 	NewMemfileBytes("a.txt", []byte("foo")),
 // 	NewMemfileBytes("b.txt", []byte("bar")),
@@ -12,13 +12,13 @@
 // 	),
 // )
 // File is an interface that provides functionality for handling
-// files/directories as values that can be supplied to commands.
+// cafs/directories as values that can be supplied to commands.
 //
 // This is pretty close to things that already exist in ipfs
 // and might not be necessary in most situations, but provides a sensible
 // degree of modularity for our purposes:
-// * memdir: github.com/ipfs/go-ipfs/commands/files.SerialFile
-// * memfs: github.com/ipfs/go-ipfs/commands/files.ReaderFile
+// * memdir: github.com/ipfs/go-ipfs/commands/cafs.SerialFile
+// * memfs: github.com/ipfs/go-ipfs/commands/cafs.ReaderFile
 package memfs
 
 import (
@@ -27,7 +27,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/ipfs/go-ipfs/commands/files"
+	"github.com/qri-io/cafs"
 )
 
 // PathSetter adds the capacity to modify a path property
@@ -42,8 +42,8 @@ type Memfile struct {
 	path string
 }
 
-// Confirm that Memfile satisfies the files.File interface
-var _ = (files.File)(&Memfile{})
+// Confirm that Memfile satisfies the cafs.File interface
+var _ = (cafs.File)(&Memfile{})
 
 // NewMemfileBytes creates a file from an io.Reader
 func NewMemfileReader(name string, r io.Reader) *Memfile {
@@ -88,37 +88,37 @@ func (Memfile) IsDirectory() bool {
 	return false
 }
 
-func (Memfile) NextFile() (files.File, error) {
-	return nil, files.ErrNotDirectory
+func (Memfile) NextFile() (cafs.File, error) {
+	return nil, cafs.ErrNotDirectory
 }
 
 // Memdir is an in-memory directory
 // Currently it only supports either Memfile & Memdir as children
 type Memdir struct {
 	path     string
 	fi       int // file index for reading
-	children []files.File
+	children []cafs.File
 }
 
-// Confirm that Memdir satisfies the files.File interface
-var _ = (files.File)(&Memdir{})
+// Confirm that Memdir satisfies the cafs.File interface
+var _ = (cafs.File)(&Memdir{})
 
 // NewMemdir creates a new Memdir, supplying zero or more children
-func NewMemdir(path string, children ...files.File) *Memdir {
+func NewMemdir(path string, children ...cafs.File) *Memdir {
 	m := &Memdir{
 		path:     path,
-		children: []files.File{},
+		children: []cafs.File{},
 	}
 	m.AddChildren(children...)
 	return m
 }
 
 func (Memdir) Close() error {
-	return files.ErrNotReader
+	return cafs.ErrNotReader
 }
 
 func (Memdir) Read([]byte) (int, error) {
-	return 0, files.ErrNotReader
+	return 0, cafs.ErrNotReader
 }
 
 func (m Memdir) FileName() string {
@@ -133,7 +133,7 @@ func (Memdir) IsDirectory() bool {
 	return true
 }
 
-func (d *Memdir) NextFile() (files.File, error) {
+func (d *Memdir) NextFile() (cafs.File, error) {
 	if d.fi >= len(d.children) {
 		d.fi = 0
 		return nil, io.EOF
@@ -154,7 +154,7 @@ func (d *Memdir) SetPath(path string) {
 // AddChildren allows any sort of file to be added, but only
 // implementations that implement the PathSetter interface will have
 // properly configured paths.
-func (d *Memdir) AddChildren(fs ...files.File) {
+func (d *Memdir) AddChildren(fs ...cafs.File) {
 	for _, f := range fs {
 		if fps, ok := f.(PathSetter); ok {
 			fps.SetPath(filepath.Join(d.FullPath(), f.FileName()))
@@ -178,7 +178,7 @@ func (d *Memdir) ChildDir(dirname string) *Memdir {
 	return nil
 }
 
-func (d *Memdir) MakeDirP(f files.File) *Memdir {
+func (d *Memdir) MakeDirP(f cafs.File) *Memdir {
 	dirpath, _ := filepath.Split(f.FileName())
 	if dirpath == "" {
 		return d

memfs/memfile_test.go

@@ -1,7 +1,6 @@
 package memfs
 
 import (
-	"github.com/ipfs/go-ipfs/commands/files"
 	"github.com/qri-io/cafs"
 	"testing"
 )
@@ -36,7 +35,7 @@ func TestMemfile(t *testing.T) {
 	}
 
 	paths := []string{}
-	err := cafs.Walk(a, 0, func(f files.File, depth int) error {
+	err := cafs.Walk(a, 0, func(f cafs.File, depth int) error {
 		paths = append(paths, f.FullPath())
 		return nil
 	})
@@ -71,7 +70,7 @@ func TestMemdirMakeDirP(t *testing.T) {
 	}
 
 	paths := []string{}
-	err := cafs.Walk(dir, 0, func(f files.File, depth int) error {
+	err := cafs.Walk(dir, 0, func(f cafs.File, depth int) error {
 		paths = append(paths, f.FullPath())
 		return nil
 	})