From db9037a4f865356f7465e7fd18e58d7eb935271c Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Tue, 6 Aug 2024 14:19:32 -0700 Subject: [PATCH 01/18] feat: support UnixFS 1.5 file mode and modification times Replaces PR #34 written by @kstuart Adds support for storing and retrieving file mode and last modification time. Support added to: - [X] Files - [X] LinkFiles - [X] Webfiles - [X] Directories - [X] Tar Archives When the TAR archive (headers) include a file mode or modification time, the extractor will restore that metadata when supported for the underlying filesystem. The Golang runtime currently does not support changing the times on a symlink, for Linux and some BSDs a custom solution has been implemented, for Darwin this is not the case so when copying a symlink to the filesystem the last modification time is not updated. Since for concrete files and directories stored modes and modification times are faithfully restored to the filesystem this should not be a breaking issue, a similar solution to that implemented for Linux/BSDs is likely implementable by a developer with access to a Darwin platform. Replaces PRs: - https://github.com/ipfs/go-ipfs-files/pull/31 - https://github.com/ipfs/tar-utils/pull/11 - #34 Relates to https://github.com/ipfs/kubo/issues/6920 --- files/file.go | 8 + files/file_test.go | 141 +++++++++++++++ files/filter_test.go | 18 +- files/helpers_test.go | 12 ++ files/linkfile.go | 24 ++- files/multifilereader.go | 44 ++++- files/multifilereader_test.go | 53 +++++- files/multipartfile.go | 85 ++++++++- files/readerfile.go | 20 ++- files/serialfile.go | 9 + files/slicedirectory.go | 48 +++++- files/tarwriter.go | 65 +++---- files/tarwriter_test.go | 49 ++++-- files/util.go | 37 ++++ files/util_test.go | 24 +++ files/webfile.go | 34 ++++ files/webfile_test.go | 13 ++ gateway/backend_blocks.go | 6 +- gateway/backend_car_files.go | 18 ++ gateway/backend_car_test.go | 10 +- gateway/handler_tar.go | 2 +- ipld/unixfs/file/unixfile.go | 43 ++++- tar/extractor.go | 130 ++++++++++++-- tar/extractor_test.go | 316 ++++++++++++++++++++++++++++------ tar/util_other.go | 25 +++ tar/util_posix.go | 41 +++++ tar/util_windows.go | 31 ++++ 27 files changed, 1150 insertions(+), 156 deletions(-) create mode 100644 files/util_test.go create mode 100644 tar/util_other.go create mode 100644 tar/util_posix.go create mode 100644 tar/util_windows.go diff --git a/files/file.go b/files/file.go index 7ac1fc98a..2fd907931 100644 --- a/files/file.go +++ b/files/file.go @@ -4,6 +4,7 @@ import ( "errors" "io" "os" + "time" ) var ( @@ -17,6 +18,13 @@ var ( type Node interface { io.Closer + // Mode returns the file's mode. + Mode() os.FileMode + + // ModTime returns the file's last modification time. If the last + // modification time is unknown/unspecified ModTime returns zero. + ModTime() (mtime time.Time) + // Size returns size of this file (if this file is a directory, total size of // all files stored in the tree should be returned). Some implementations may // choose not to implement this diff --git a/files/file_test.go b/files/file_test.go index 3edecf107..07d9f04db 100644 --- a/files/file_test.go +++ b/files/file_test.go @@ -3,8 +3,10 @@ package files import ( "io" "mime/multipart" + "os" "strings" "testing" + "time" ) func TestSliceFiles(t *testing.T) { @@ -49,6 +51,21 @@ func TestReaderFiles(t *testing.T) { } } +func TestReaderFileStat(t *testing.T) { + reader := strings.NewReader("beep boop") + mode := os.FileMode(0754) + mtime := time.Date(2020, 11, 2, 12, 27, 35, 55555, time.UTC) + stat := &mockFileInfo{name: "test", mode: mode, mtime: mtime} + + rf := NewReaderStatFile(reader, stat) + if rf.Mode() != mode { + t.Fatalf("Expected file mode to be [%v] but got [%v]", mode, rf.Mode()) + } + if rf.ModTime() != mtime { + t.Fatalf("Expected file modified time to be [%v] but got [%v]", mtime, rf.ModTime()) + } +} + func TestMultipartFiles(t *testing.T) { data := ` --Boundary! @@ -141,3 +158,127 @@ implicit file2 }, }) } + +func TestMultipartFilesWithMode(t *testing.T) { + data := ` +--Boundary! +Content-Type: text/plain +Content-Disposition: form-data; name="file-0?mode=0754&mtime=1604320500&mtime-nsecs=55555"; filename="%C2%A3%E1%BA%9E%C7%91%C7%93%C3%86+%C3%A6+%E2%99%AB%E2%99%AC" +Some-Header: beep + +beep +--Boundary! +Content-Type: application/x-directory +Content-Disposition: form-data; name="dir-0?mode=755&mtime=1604320500"; ans=42; filename="dir1" + +--Boundary! +Content-Type: text/plain +Content-Disposition: form-data; name="file"; filename="dir1/nested" + +some content +--Boundary! +Content-Type: text/plain +Content-Disposition: form-data; name="file?mode=600"; filename="dir1/nested2"; ans=42 + +some content +--Boundary! +Content-Type: application/symlink +Content-Disposition: form-data; name="file-5"; filename="dir1/simlynk" + +anotherfile +--Boundary! +Content-Type: application/symlink +Content-Disposition: form-data; name="file?mtime=1604320500"; filename="dir1/simlynk2" + +anotherfile +--Boundary! +Content-Type: text/plain +Content-Disposition: form-data; name="dir?mode=0644"; filename="implicit1/implicit2/deep_implicit" + +implicit file1 +--Boundary! +Content-Type: text/plain +Content-Disposition: form-data; name="dir?mode=755&mtime=1604320500"; filename="implicit1/shallow_implicit" + +implicit file2 +--Boundary!-- + +` + + reader := strings.NewReader(data) + mpReader := multipart.NewReader(reader, "Boundary!") + dir, err := NewFileFromPartReader(mpReader, multipartFormdataType) + if err != nil { + t.Fatal(err) + } + + CheckDir(t, dir, []Event{ + { + kind: TFile, + name: "£ẞǑǓÆ æ ♫♬", + value: "beep", + mode: 0754, + mtime: time.Unix(1604320500, 55555), + }, + { + kind: TDirStart, + name: "dir1", + mode: 0755, + mtime: time.Unix(1604320500, 0), + }, + { + kind: TFile, + name: "nested", + value: "some content", + }, + { + kind: TFile, + name: "nested2", + value: "some content", + mode: 0600, + }, + { + kind: TSymlink, + name: "simlynk", + value: "anotherfile", + mode: 0777, + }, + { + kind: TSymlink, + name: "simlynk2", + value: "anotherfile", + mode: 0777, + mtime: time.Unix(1604320500, 0), + }, + { + kind: TDirEnd, + }, + { + kind: TDirStart, + name: "implicit1", + }, + { + kind: TDirStart, + name: "implicit2", + }, + { + kind: TFile, + name: "deep_implicit", + value: "implicit file1", + mode: 0644, + }, + { + kind: TDirEnd, + }, + { + kind: TFile, + name: "shallow_implicit", + value: "implicit file2", + mode: 0755, + mtime: time.Unix(1604320500, 0), + }, + { + kind: TDirEnd, + }, + }) +} diff --git a/files/filter_test.go b/files/filter_test.go index 00b2e8baf..f2de61168 100644 --- a/files/filter_test.go +++ b/files/filter_test.go @@ -4,17 +4,33 @@ import ( "os" "path/filepath" "testing" + "time" ) type mockFileInfo struct { os.FileInfo - name string + name string + mode os.FileMode + mtime time.Time + size int64 } func (m *mockFileInfo) Name() string { return m.name } +func (m *mockFileInfo) Mode() os.FileMode { + return m.mode +} + +func (m *mockFileInfo) ModTime() time.Time { + return m.mtime +} + +func (m *mockFileInfo) Size() int64 { + return m.size +} + func (m *mockFileInfo) Sys() interface{} { return nil } diff --git a/files/helpers_test.go b/files/helpers_test.go index 0180b8f27..32e54544e 100644 --- a/files/helpers_test.go +++ b/files/helpers_test.go @@ -2,7 +2,9 @@ package files import ( "io" + "os" "testing" + "time" ) type Kind int @@ -18,6 +20,8 @@ type Event struct { kind Kind name string value string + mode os.FileMode + mtime time.Time } func CheckDir(t *testing.T, dir Directory, expected []Event) { @@ -50,6 +54,14 @@ func CheckDir(t *testing.T, dir Directory, expected []Event) { t.Fatalf("[%d] expected filename to be %q", i, next.name) } + if next.mode != 0 && it.Node().Mode()&os.ModePerm != next.mode { + t.Fatalf("[%d] expected mode for '%s' to be %O, got %O", i, it.Name(), next.mode, it.Node().Mode()) + } + + if !next.mtime.IsZero() && !it.Node().ModTime().Equal(next.mtime) { + t.Fatalf("[%d] expected modification time for '%s' to be %q", i, it.Name(), next.mtime) + } + switch next.kind { case TFile: mf, ok := it.Node().(File) diff --git a/files/linkfile.go b/files/linkfile.go index 526998652..6881068f7 100644 --- a/files/linkfile.go +++ b/files/linkfile.go @@ -3,21 +3,41 @@ package files import ( "os" "strings" + "time" ) type Symlink struct { Target string - stat os.FileInfo + mtime time.Time reader strings.Reader } func NewLinkFile(target string, stat os.FileInfo) File { - lf := &Symlink{Target: target, stat: stat} + mtime := time.Time{} + if stat != nil { + mtime = stat.ModTime() + } + return NewSymlinkFile(target, mtime) +} + +func NewSymlinkFile(target string, mtime time.Time) File { + lf := &Symlink{ + Target: target, + mtime: mtime, + } lf.reader.Reset(lf.Target) return lf } +func (lf *Symlink) Mode() os.FileMode { + return os.ModeSymlink | os.ModePerm +} + +func (lf *Symlink) ModTime() time.Time { + return lf.mtime +} + func (lf *Symlink) Close() error { return nil } diff --git a/files/multifilereader.go b/files/multifilereader.go index 1a5d4ac1a..33fde2889 100644 --- a/files/multifilereader.go +++ b/files/multifilereader.go @@ -8,6 +8,8 @@ import ( "net/textproto" "net/url" "path" + "strconv" + "strings" "sync" ) @@ -89,18 +91,12 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { // handle starting a new file part if !mfr.closed { - mfr.currentFile = entry.Node() // write the boundary and headers header := make(textproto.MIMEHeader) - filename := url.QueryEscape(path.Join(path.Join(mfr.path...), entry.Name())) - dispositionPrefix := "attachment" - if mfr.form { - dispositionPrefix = "form-data; name=\"file\"" - } - - header.Set("Content-Disposition", fmt.Sprintf("%s; filename=\"%s\"", dispositionPrefix, filename)) + filename := path.Join(path.Join(mfr.path...), entry.Name()) + mfr.addContentDisposition(header, filename) var contentType string @@ -119,7 +115,7 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { return 0, ErrNotSupported } - header.Set("Content-Type", contentType) + header.Set(contentTypeHeader, contentType) if rf, ok := entry.Node().(FileInfo); ok { if mfr.rawAbsPath { // Legacy compatibility with old servers. @@ -157,6 +153,36 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { return written, nil } +func (mfr *MultiFileReader) addContentDisposition(header textproto.MIMEHeader, filename string) { + sb := &strings.Builder{} + params := url.Values{} + + if mode := mfr.currentFile.Mode(); mode != 0 { + params.Add("mode", "0"+strconv.FormatUint(uint64(mode), 8)) + } + if mtime := mfr.currentFile.ModTime(); !mtime.IsZero() { + params.Add("mtime", strconv.FormatInt(mtime.Unix(), 10)) + if n := mtime.Nanosecond(); n > 0 { + params.Add("mtime-nsecs", strconv.FormatInt(int64(n), 10)) + } + } + + sb.Grow(120) + if mfr.form { + sb.WriteString("form-data; name=\"file") + if len(params) > 0 { + fmt.Fprintf(sb, "?%s", params.Encode()) + } + sb.WriteString("\"") + } else { + sb.WriteString("attachment") + } + + fmt.Fprintf(sb, "; filename=\"%s\"", url.QueryEscape(filename)) + + header.Set(contentDispositionHeader, sb.String()) +} + // Boundary returns the boundary string to be used to separate files in the multipart data func (mfr *MultiFileReader) Boundary() string { return mfr.mpWriter.Boundary() diff --git a/files/multifilereader_test.go b/files/multifilereader_test.go index b39217037..623c5404a 100644 --- a/files/multifilereader_test.go +++ b/files/multifilereader_test.go @@ -3,8 +3,13 @@ package files import ( "bytes" "io" + "io/fs" "mime/multipart" + "net/textproto" + "path" + "strings" "testing" + "time" "github.com/stretchr/testify/require" ) @@ -12,7 +17,12 @@ import ( var text = "Some text! :)" func newBytesFileWithPath(abspath string, b []byte) File { - return &ReaderFile{abspath, bytesReaderCloser{bytes.NewReader(b)}, nil, int64(len(b))} + return &ReaderFile{ + abspath: abspath, + reader: bytesReaderCloser{bytes.NewReader(b)}, + stat: &mockFileInfo{name: path.Base(abspath), mode: 0754, mtime: time.Unix(1604320500, 55555)}, + fsize: int64(len(b)), + } } func makeMultiFileReader(t *testing.T, binaryFileName, rawAbsPath bool) (string, *MultiFileReader) { @@ -53,6 +63,9 @@ func runMultiFileReaderToMultiFileTest(t *testing.T, binaryFileName, rawAbsPath, require.True(t, it.Next()) require.Equal(t, "beep.txt", it.Name()) + n := it.Node() + require.Equal(t, fs.FileMode(0754), n.Mode(), "unexpected file mode") + require.Equal(t, time.Unix(1604320500, 55555), n.ModTime(), "unexpected last modification time") require.True(t, it.Next()) require.Equal(t, "boop", it.Name()) require.NotNil(t, DirFromEntry(it)) @@ -103,12 +116,20 @@ func TestMultiFileReaderToMultiFile(t *testing.T) { func getTestMultiFileReader(t *testing.T) *MultiFileReader { sf := NewMapDirectory(map[string]Node{ - "file.txt": NewBytesFile([]byte(text)), + "file.txt": NewReaderStatFile( + strings.NewReader(text), + &mockFileInfo{name: "file.txt", mode: 0, mtime: time.Time{}}), "boop": NewMapDirectory(map[string]Node{ - "a.txt": NewBytesFile([]byte("bleep")), - "b.txt": NewBytesFile([]byte("bloop")), + "a.txt": NewReaderStatFile( + strings.NewReader("bleep"), + &mockFileInfo{name: "a.txt", mode: 0744, mtime: time.Time{}}), + "b.txt": NewReaderStatFile( + strings.NewReader("bloop"), + &mockFileInfo{name: "b.txt", mode: 0666, mtime: time.Unix(1604320500, 0)}), }), - "beep.txt": NewBytesFile([]byte("beep")), + "beep.txt": NewReaderStatFile( + strings.NewReader("beep"), + &mockFileInfo{name: "beep.txt", mode: 0754, mtime: time.Unix(1604320500, 55555)}), }) // testing output by reading it with the go stdlib "mime/multipart" Reader @@ -242,3 +263,25 @@ func TestCommonPrefix(t *testing.T) { }, }) } + +func TestContentDispositonEncoding(t *testing.T) { + testContentDispositionEncoding(t, false, "£ẞǑǓÆ æ ♫♬", + "attachment; filename=\"%C2%A3%E1%BA%9E%C7%91%C7%93%C3%86+%C3%A6+%E2%99%AB%E2%99%AC\"") + testContentDispositionEncoding(t, true, "£ẞǑǓÆ æ ♫♬", + "form-data; name=\"file\"; filename=\"%C2%A3%E1%BA%9E%C7%91%C7%93%C3%86+%C3%A6+%E2%99%AB%E2%99%AC\"") +} + +func testContentDispositionEncoding(t *testing.T, form bool, filename string, expected string) { + sf := NewMapDirectory(map[string]Node{"": NewBytesFile([]byte(""))}) + mfr := NewMultiFileReader(sf, form, false) + if _, err := mfr.Read(nil); err != nil { + t.Fatal("MultiFileReader.Read failed") + } + + header := make(textproto.MIMEHeader) + mfr.addContentDisposition(header, filename) + v := header.Get(contentDispositionHeader) + if v != expected { + t.Fatalf("content-disposition did not match:\nExpected: %s\nActual : %s", expected, v) + } +} diff --git a/files/multipartfile.go b/files/multipartfile.go index b5aab9620..3f0a5b3fa 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -5,8 +5,12 @@ import ( "mime" "mime/multipart" "net/url" + "os" "path" + "path/filepath" + "strconv" "strings" + "time" ) const ( @@ -17,17 +21,48 @@ const ( applicationSymlink = "application/symlink" applicationFile = "application/octet-stream" - contentTypeHeader = "Content-Type" + contentTypeHeader = "Content-Type" + contentDispositionHeader = "Content-Disposition" ) +// multiPartFileInfo implements the `fs.FileInfo` interface for a file or +// directory received in a `multipart.part`. +type multiPartFileInfo struct { + name string + mode os.FileMode + mtime time.Time +} + +func (fi *multiPartFileInfo) Name() string { return fi.name } +func (fi *multiPartFileInfo) Mode() os.FileMode { return fi.mode } +func (fi *multiPartFileInfo) ModTime() time.Time { return fi.mtime } +func (fi *multiPartFileInfo) IsDir() bool { return fi.mode.IsDir() } +func (fi *multiPartFileInfo) Sys() interface{} { return nil } +func (fi *multiPartFileInfo) Size() int64 { panic("size for multipart file info is not supported") } + type multipartDirectory struct { path string walker *multipartWalker + stat os.FileInfo // part is the part describing the directory. It's nil when implicit. part *multipart.Part } +func (f *multipartDirectory) Mode() os.FileMode { + if f.stat == nil { + return 0 + } + return f.stat.Mode() +} + +func (f *multipartDirectory) ModTime() time.Time { + if f.stat == nil { + return time.Time{} + } + return f.stat.ModTime() +} + type multipartWalker struct { part *multipart.Part reader *multipart.Reader @@ -85,12 +120,15 @@ func (w *multipartWalker) nextFile() (Node, error) { } } + name := fileName(part) + switch contentType { case multipartFormdataType, applicationDirectory: return &multipartDirectory{ part: part, - path: fileName(part), + path: name, walker: w, + stat: fileInfo(name, part), }, nil case applicationSymlink: out, err := io.ReadAll(part) @@ -98,7 +136,7 @@ func (w *multipartWalker) nextFile() (Node, error) { return nil, err } - return NewLinkFile(string(out), nil), nil + return NewLinkFile(string(out), fileInfo(name, part)), nil default: var absPath string if absPathEncoded := part.Header.Get("abspath-encoded"); absPathEncoded != "" { @@ -113,6 +151,7 @@ func (w *multipartWalker) nextFile() (Node, error) { return &ReaderFile{ reader: part, abspath: absPath, + stat: fileInfo(name, part), }, nil } } @@ -169,6 +208,44 @@ func (it *multipartIterator) Node() Node { return it.curFile } +// fileInfo constructs an `os.FileInfo` from a `multipart.part` serving +// a file or directory. +func fileInfo(name string, part *multipart.Part) os.FileInfo { + fi := multiPartFileInfo{name: filepath.Base(name)} + formName := part.FormName() + + i := strings.IndexByte(formName, '?') + if i == -1 { + return &fi + } + + params, err := url.ParseQuery(formName[i+1:]) + if err != nil { + return nil + } + + if v := params["mode"]; v != nil { + mode, err := strconv.ParseUint(v[0], 8, 32) + if err == nil { + fi.mode = os.FileMode(mode) + } + } + + var secs, nsecs int64 + if v := params["mtime"]; v != nil { + secs, err = strconv.ParseInt(v[0], 10, 64) + if err != nil { + return &fi + } + } + if v := params["mtime-nsecs"]; v != nil { + nsecs, _ = strconv.ParseInt(v[0], 10, 64) + } + fi.mtime = time.Unix(secs, nsecs) + + return &fi +} + func (it *multipartIterator) Next() bool { if it.f.walker.reader == nil || it.err != nil { return false @@ -206,9 +283,9 @@ func (it *multipartIterator) Next() bool { } return true } - it.curName = name // Finally, advance to the next file. + it.curName = name it.curFile, it.err = it.f.walker.nextFile() return it.err == nil diff --git a/files/readerfile.go b/files/readerfile.go index bf3fa1c9e..8b9e4069c 100644 --- a/files/readerfile.go +++ b/files/readerfile.go @@ -5,6 +5,7 @@ import ( "io" "os" "path/filepath" + "time" ) // ReaderFile is a implementation of File created from an `io.Reader`. @@ -13,8 +14,21 @@ type ReaderFile struct { abspath string reader io.ReadCloser stat os.FileInfo + fsize int64 +} - fsize int64 +func (f *ReaderFile) Mode() os.FileMode { + if f.stat == nil { + return 0 + } + return f.stat.Mode() +} + +func (f *ReaderFile) ModTime() time.Time { + if f.stat == nil { + return time.Time{} + } + return f.stat.ModTime() } func NewBytesFile(b []byte) File { @@ -32,6 +46,10 @@ func (b bytesReaderCloser) Close() error { return nil } +func NewBytesStatFile(b []byte, stat os.FileInfo) File { + return NewReaderStatFile(bytes.NewReader(b), stat) +} + func NewReaderFile(reader io.Reader) File { return NewReaderStatFile(reader, nil) } diff --git a/files/serialfile.go b/files/serialfile.go index bd25bd1bc..cf4d44be3 100644 --- a/files/serialfile.go +++ b/files/serialfile.go @@ -6,6 +6,7 @@ import ( "io/fs" "os" "path/filepath" + "time" ) // serialFile implements Node, and reads from a path on the OS filesystem. @@ -164,6 +165,14 @@ func (f *serialFile) Size() (int64, error) { return du, err } +func (f *serialFile) Mode() os.FileMode { + return f.stat.Mode() +} + +func (f *serialFile) ModTime() time.Time { + return f.stat.ModTime() +} + var ( _ Directory = &serialFile{} _ DirIterator = &serialIterator{} diff --git a/files/slicedirectory.go b/files/slicedirectory.go index 9cf910c6a..7a444b65a 100644 --- a/files/slicedirectory.go +++ b/files/slicedirectory.go @@ -1,6 +1,11 @@ package files -import "sort" +import ( + "cmp" + "os" + "slices" + "time" +) type fileEntry struct { name string @@ -49,22 +54,51 @@ func (it *sliceIterator) Err() error { // SliceFiles are always directories, and can't be read from or closed. type SliceFile struct { files []DirEntry + stat os.FileInfo +} + +func (f *SliceFile) Mode() os.FileMode { + if f.stat != nil { + return f.stat.Mode() + } + return 0 +} + +func (f *SliceFile) ModTime() time.Time { + if f.stat != nil { + return f.stat.ModTime() + } + return time.Time{} } func NewMapDirectory(f map[string]Node) Directory { - ents := make([]DirEntry, 0, len(f)) + return NewSliceDirectory(sortDirEntries(f)) +} + +func NewMapStatDirectory(f map[string]Node, stat os.FileInfo) Directory { + return NewSliceStatDirectory(sortDirEntries(f), stat) +} + +func sortDirEntries(f map[string]Node) []DirEntry { + ents := make([]DirEntry, len(f)) + var i int for name, nd := range f { - ents = append(ents, FileEntry(name, nd)) + ents[i] = FileEntry(name, nd) + i++ } - sort.Slice(ents, func(i, j int) bool { - return ents[i].Name() < ents[j].Name() + slices.SortFunc(ents, func(a, b DirEntry) int { + return cmp.Compare(a.Name(), b.Name()) }) - return NewSliceDirectory(ents) + return ents } func NewSliceDirectory(files []DirEntry) Directory { - return &SliceFile{files} + return &SliceFile{files: files} +} + +func NewSliceStatDirectory(files []DirEntry, stat os.FileInfo) Directory { + return &SliceFile{files: files, stat: stat} } func (f *SliceFile) Entries() DirIterator { diff --git a/files/tarwriter.go b/files/tarwriter.go index e5d857116..f8b1d7791 100644 --- a/files/tarwriter.go +++ b/files/tarwriter.go @@ -16,23 +16,25 @@ type TarWriter struct { TarW *tar.Writer baseDirSet bool baseDir string + format tar.Format } // NewTarWriter wraps given io.Writer into a new tar writer func NewTarWriter(w io.Writer) (*TarWriter, error) { return &TarWriter{ - TarW: tar.NewWriter(w), + TarW: tar.NewWriter(w), + format: tar.FormatUnknown, }, nil } func (w *TarWriter) writeDir(f Directory, fpath string) error { - if err := writeDirHeader(w.TarW, fpath); err != nil { + if err := w.writeHeader(f, fpath, 0); err != nil { return err } it := f.Entries() for it.Next() { - if err := w.WriteFile(it.Node(), path.Join(fpath, it.Name())); err != nil { + if err := w.WriteNode(it.Node(), path.Join(fpath, it.Name())); err != nil { return err } } @@ -45,7 +47,7 @@ func (w *TarWriter) writeFile(f File, fpath string) error { return err } - if err := writeFileHeader(w.TarW, fpath, uint64(size)); err != nil { + if err = w.writeHeader(f, fpath, size); err != nil { return err } @@ -77,7 +79,7 @@ func validateTarFilePath(baseDir, fpath string) bool { } // WriteNode adds a node to the archive. -func (w *TarWriter) WriteFile(nd Node, fpath string) error { +func (w *TarWriter) WriteNode(nd Node, fpath string) error { if !w.baseDirSet { w.baseDirSet = true // Use a variable for this as baseDir may be an empty string. w.baseDir = fpath @@ -89,7 +91,7 @@ func (w *TarWriter) WriteFile(nd Node, fpath string) error { switch nd := nd.(type) { case *Symlink: - return writeSymlinkHeader(w.TarW, nd.Target, fpath) + return w.writeHeader(nd, fpath, 0) case File: return w.writeFile(nd, fpath) case Directory: @@ -104,32 +106,33 @@ func (w *TarWriter) Close() error { return w.TarW.Close() } -func writeDirHeader(w *tar.Writer, fpath string) error { - return w.WriteHeader(&tar.Header{ - Name: fpath, - Typeflag: tar.TypeDir, - Mode: 0o777, - ModTime: time.Now().Truncate(time.Second), - // TODO: set mode, dates, etc. when added to unixFS - }) -} +func (w *TarWriter) writeHeader(n Node, fpath string, size int64) error { + hdr := &tar.Header{ + Format: w.format, + Name: fpath, + Size: size, + Mode: int64(UnixPermsOrDefault(n)), + } + + switch nd := n.(type) { + case *Symlink: + hdr.Typeflag = tar.TypeSymlink + hdr.Linkname = nd.Target + case Directory: + hdr.Typeflag = tar.TypeDir + default: + hdr.Typeflag = tar.TypeReg + } + + if m := n.ModTime(); m.IsZero() { + hdr.ModTime = time.Now() + } else { + hdr.ModTime = m + } -func writeFileHeader(w *tar.Writer, fpath string, size uint64) error { - return w.WriteHeader(&tar.Header{ - Name: fpath, - Size: int64(size), - Typeflag: tar.TypeReg, - Mode: 0o644, - ModTime: time.Now().Truncate(time.Second), - // TODO: set mode, dates, etc. when added to unixFS - }) + return w.TarW.WriteHeader(hdr) } -func writeSymlinkHeader(w *tar.Writer, target, fpath string) error { - return w.WriteHeader(&tar.Header{ - Name: fpath, - Linkname: target, - Mode: 0o777, - Typeflag: tar.TypeSymlink, - }) +func (w *TarWriter) SetFormat(format tar.Format) { + w.format = format } diff --git a/files/tarwriter_test.go b/files/tarwriter_test.go index 0e1488e7f..3b71ebc4a 100644 --- a/files/tarwriter_test.go +++ b/files/tarwriter_test.go @@ -11,11 +11,13 @@ import ( func TestTarWriter(t *testing.T) { tf := NewMapDirectory(map[string]Node{ "file.txt": NewBytesFile([]byte(text)), - "boop": NewMapDirectory(map[string]Node{ + "boop": NewMapStatDirectory(map[string]Node{ "a.txt": NewBytesFile([]byte("bleep")), "b.txt": NewBytesFile([]byte("bloop")), - }), - "beep.txt": NewBytesFile([]byte("beep")), + }, &mockFileInfo{name: "", mode: 0750, mtime: time.Unix(6600000000, 0)}), + "beep.txt": NewBytesStatFile([]byte("beep"), + &mockFileInfo{name: "beep.txt", size: 4, mode: 0766, mtime: time.Unix(1604320500, 54321)}), + "boop-sl": NewSymlinkFile("boop", time.Unix(6600050000, 0)), }) pr, pw := io.Pipe() @@ -23,18 +25,20 @@ func TestTarWriter(t *testing.T) { if err != nil { t.Fatal(err) } + tw.SetFormat(tar.FormatPAX) tr := tar.NewReader(pr) go func() { defer tw.Close() - if err := tw.WriteFile(tf, ""); err != nil { + if err := tw.WriteNode(tf, ""); err != nil { t.Error(err) } }() var cur *tar.Header + const delta = 4 * time.Second - checkHeader := func(name string, typ byte, size int64) { + checkHeader := func(name string, typ byte, size int64, mode int64, mtime time.Time) { if cur.Name != name { t.Errorf("got wrong name: %s != %s", cur.Name, name) } @@ -44,41 +48,52 @@ func TestTarWriter(t *testing.T) { if cur.Size != size { t.Errorf("got wrong size: %d != %d", cur.Size, size) } - now := time.Now() - if cur.ModTime.After(now) { - t.Errorf("wrote timestamp in the future: %s (now) < %s", now, cur.ModTime) + if cur.Mode != mode { + t.Errorf("got wrong mode: %d != %d", cur.Mode, mode) + } + if mtime.IsZero() { + interval := time.Since(cur.ModTime) + if interval < -delta || interval > delta { + t.Errorf("expected timestamp to be current: %s", cur.ModTime) + } + } else if cur.ModTime.UnixNano() != mtime.UnixNano() { + t.Errorf("got wrong timestamp: %s != %s", cur.ModTime, mtime) } } if cur, err = tr.Next(); err != nil { t.Fatal(err) } - checkHeader("", tar.TypeDir, 0) + checkHeader("", tar.TypeDir, 0, 0755, time.Time{}) if cur, err = tr.Next(); err != nil { t.Fatal(err) } - checkHeader("beep.txt", tar.TypeReg, 4) + checkHeader("beep.txt", tar.TypeReg, 4, 0766, time.Unix(1604320500, 54321)) if cur, err = tr.Next(); err != nil { t.Fatal(err) } - checkHeader("boop", tar.TypeDir, 0) + checkHeader("boop", tar.TypeDir, 0, 0750, time.Unix(6600000000, 0)) if cur, err = tr.Next(); err != nil { t.Fatal(err) } - checkHeader("boop/a.txt", tar.TypeReg, 5) + checkHeader("boop/a.txt", tar.TypeReg, 5, 0644, time.Time{}) if cur, err = tr.Next(); err != nil { t.Fatal(err) } - checkHeader("boop/b.txt", tar.TypeReg, 5) + checkHeader("boop/b.txt", tar.TypeReg, 5, 0644, time.Time{}) if cur, err = tr.Next(); err != nil { t.Fatal(err) } - checkHeader("file.txt", tar.TypeReg, 13) + checkHeader("boop-sl", tar.TypeSymlink, 0, 0777, time.Unix(6600050000, 0)) + if cur, err = tr.Next(); err != nil { + t.Fatal(err) + } + checkHeader("file.txt", tar.TypeReg, 13, 0644, time.Time{}) if cur, err = tr.Next(); err != io.EOF { t.Fatal(err) @@ -101,7 +116,7 @@ func TestTarWriterRelativePathInsideRoot(t *testing.T) { } defer tw.Close() - if err := tw.WriteFile(tf, ""); err != nil { + if err = tw.WriteNode(tf, ""); err != nil { t.Error(err) } } @@ -122,7 +137,7 @@ func TestTarWriterFailsFileOutsideRoot(t *testing.T) { } defer tw.Close() - if err := tw.WriteFile(tf, ""); !errors.Is(err, ErrUnixFSPathOutsideRoot) { + if err = tw.WriteNode(tf, ""); !errors.Is(err, ErrUnixFSPathOutsideRoot) { t.Errorf("unexpected error, wanted: %v; got: %v", ErrUnixFSPathOutsideRoot, err) } } @@ -143,7 +158,7 @@ func TestTarWriterFailsFileOutsideRootWithBaseDir(t *testing.T) { } defer tw.Close() - if err := tw.WriteFile(tf, "test.tar"); !errors.Is(err, ErrUnixFSPathOutsideRoot) { + if err = tw.WriteNode(tf, "test.tar"); !errors.Is(err, ErrUnixFSPathOutsideRoot) { t.Errorf("unexpected error, wanted: %v; got: %v", ErrUnixFSPathOutsideRoot, err) } } diff --git a/files/util.go b/files/util.go index e727e7ae6..1ac673b0e 100644 --- a/files/util.go +++ b/files/util.go @@ -1,5 +1,7 @@ package files +import "os" + // ToFile is an alias for n.(File). If the file isn't a regular file, nil value // will be returned func ToFile(n Node) File { @@ -23,3 +25,38 @@ func FileFromEntry(e DirEntry) File { func DirFromEntry(e DirEntry) Directory { return ToDir(e.Node()) } + +// UnixPermsOrDefault returns the unix style permissions stored for the given +// Node, or default unix permissions for the Node type. +func UnixPermsOrDefault(n Node) uint32 { + perms := ModePermsToUnixPerms(n.Mode()) + if perms != 0 { + return perms + } + + switch n.(type) { + case *Symlink: + return 0777 + case Directory: + return 0755 + default: + return 0644 + } +} + +// ModePermsToUnixPerms converts the permission bits of an os.FileMode to unix +// style mode permissions. +func ModePermsToUnixPerms(fileMode os.FileMode) uint32 { + return uint32((fileMode & 0xC00000 >> 12) | (fileMode & os.ModeSticky >> 11) | (fileMode & 0x1FF)) +} + +// UnixPermsToModePerms converts unix style mode permissions to os.FileMode +// permissions, as it only operates on permission bits it does not set the +// underlying type (fs.ModeDir, fs.ModeSymlink, etc.) in the returned +// os.FileMode. +func UnixPermsToModePerms(unixPerms uint32) os.FileMode { + if unixPerms == 0 { + return 0 + } + return os.FileMode((unixPerms & 0x1FF) | (unixPerms & 0xC00 << 12) | (unixPerms & 0x200 << 11)) +} diff --git a/files/util_test.go b/files/util_test.go new file mode 100644 index 000000000..396abec99 --- /dev/null +++ b/files/util_test.go @@ -0,0 +1,24 @@ +package files + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestModePermsToUnixPerms(t *testing.T) { + assert.Equal(t, uint32(0777), ModePermsToUnixPerms(os.FileMode(0777))) + assert.Equal(t, uint32(04755), ModePermsToUnixPerms(0755|os.ModeSetuid)) + assert.Equal(t, uint32(02777), ModePermsToUnixPerms(0777|os.ModeSetgid)) + assert.Equal(t, uint32(01377), ModePermsToUnixPerms(0377|os.ModeSticky)) + assert.Equal(t, uint32(05300), ModePermsToUnixPerms(0300|os.ModeSetuid|os.ModeSticky)) +} + +func TestUnixPermsToModePerms(t *testing.T) { + assert.Equal(t, os.FileMode(0777), UnixPermsToModePerms(0777)) + assert.Equal(t, 0755|os.ModeSetuid, UnixPermsToModePerms(04755)) + assert.Equal(t, 0777|os.ModeSetgid, UnixPermsToModePerms(02777)) + assert.Equal(t, 0377|os.ModeSticky, UnixPermsToModePerms(01377)) + assert.Equal(t, 0300|os.ModeSetuid|os.ModeSticky, UnixPermsToModePerms(05300)) +} diff --git a/files/webfile.go b/files/webfile.go index 4586eab63..8791fce8a 100644 --- a/files/webfile.go +++ b/files/webfile.go @@ -7,8 +7,16 @@ import ( "net/http" "net/url" "os" + "strconv" + "time" ) +// the HTTP Response header that provides the last modified timestamp +const LastModifiedHeaderName = "Last-Modified" + +// the HTTP Response header that provides the unix file mode +const FileModeHeaderName = "File-Mode" + // WebFile is an implementation of File which reads it // from a Web URL (http). A GET request will be performed // against the source when calling Read(). @@ -16,6 +24,16 @@ type WebFile struct { body io.ReadCloser url *url.URL contentLength int64 + mode os.FileMode + mtime time.Time +} + +func (wf *WebFile) Mode() os.FileMode { + return wf.mode +} + +func (wf *WebFile) ModTime() time.Time { + return wf.mtime } // NewWebFile creates a WebFile with the given URL, which @@ -38,10 +56,26 @@ func (wf *WebFile) start() error { } wf.body = resp.Body wf.contentLength = resp.ContentLength + wf.getResponseMetaData(resp) } return nil } +func (wf *WebFile) getResponseMetaData(resp *http.Response) { + ts := resp.Header.Get(LastModifiedHeaderName) + if ts != "" { + if mtime, err := time.Parse(time.RFC1123, ts); err == nil { + wf.mtime = mtime + } + } + md := resp.Header.Get(FileModeHeaderName) + if md != "" { + if mode, err := strconv.ParseInt(md, 8, 32); err == nil { + wf.mode = os.FileMode(mode) + } + } +} + // Read reads the File from it's web location. On the first // call to Read, a GET request will be performed against the // WebFile's URL, using Go's default HTTP client. Any further diff --git a/files/webfile_test.go b/files/webfile_test.go index 94cddb5d2..b2a7238ab 100644 --- a/files/webfile_test.go +++ b/files/webfile_test.go @@ -6,12 +6,19 @@ import ( "net/http" "net/http/httptest" "net/url" + "strconv" "testing" + "time" ) func TestWebFile(t *testing.T) { const content = "Hello world!" + const mode = 0644 + mtime := time.Unix(16043205005, 0) + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Add(LastModifiedHeaderName, mtime.Format(time.RFC1123)) + w.Header().Add(FileModeHeaderName, strconv.FormatUint(uint64(mode), 8)) fmt.Fprint(w, content) })) defer s.Close() @@ -28,6 +35,12 @@ func TestWebFile(t *testing.T) { if string(body) != content { t.Fatalf("expected %q but got %q", content, string(body)) } + if actual := wf.Mode(); actual != mode { + t.Fatalf("expected file mode %q but got 0%q", mode, strconv.FormatUint(uint64(actual), 8)) + } + if actual := wf.ModTime(); !actual.Equal(mtime) { + t.Fatalf("expected last modified time %q but got %q", mtime, actual) + } } func TestWebFile_notFound(t *testing.T) { diff --git a/gateway/backend_blocks.go b/gateway/backend_blocks.go index 42440dfcd..3e578e8d1 100644 --- a/gateway/backend_blocks.go +++ b/gateway/backend_blocks.go @@ -150,7 +150,7 @@ func (bb *BlocksBackend) Get(ctx context.Context, path path.ImmutablePath, range } // This code path covers full graph, single file/directory, and range requests - f, err := ufile.NewUnixfsFile(ctx, bb.dagService, nd) + f, err := ufile.NewUnixfsFile(ctx, bb.dagService, nd, nil) // Note: there is an assumption here that non-UnixFS dag-pb should not be returned which is currently valid if err != nil { return md, nil, err @@ -197,7 +197,7 @@ func (bb *BlocksBackend) GetAll(ctx context.Context, path path.ImmutablePath) (C } // This code path covers full graph, single file/directory, and range requests - n, err := ufile.NewUnixfsFile(ctx, bb.dagService, nd) + n, err := ufile.NewUnixfsFile(ctx, bb.dagService, nd, nil) if err != nil { return md, nil, err } @@ -226,7 +226,7 @@ func (bb *BlocksBackend) Head(ctx context.Context, path path.ImmutablePath) (Con // TODO: We're not handling non-UnixFS dag-pb. There's a bit of a discrepancy // between what we want from a HEAD request and a Resolve request here and we're using this for both - fileNode, err := ufile.NewUnixfsFile(ctx, bb.dagService, nd) + fileNode, err := ufile.NewUnixfsFile(ctx, bb.dagService, nd, nil) if err != nil { return ContentPathMetadata{}, nil, err } diff --git a/gateway/backend_car_files.go b/gateway/backend_car_files.go index c384bbe2c..64d10bcd6 100644 --- a/gateway/backend_car_files.go +++ b/gateway/backend_car_files.go @@ -5,6 +5,8 @@ import ( "context" "fmt" "io" + "os" + "time" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/ipld/unixfs" @@ -50,6 +52,14 @@ func (b *backpressuredFile) Close() error { return nil } +func (b *backpressuredFile) Mode() os.FileMode { + panic("not implemented") +} + +func (b *backpressuredFile) ModTime() time.Time { + panic("not implemented") +} + func (b *backpressuredFile) Size() (int64, error) { return b.size, nil } @@ -126,6 +136,14 @@ func (b *singleUseDirectory) Close() error { return nil } +func (b *singleUseDirectory) Mode() os.FileMode { + panic("not implemented") +} + +func (b *singleUseDirectory) ModTime() time.Time { + panic("not implemented") +} + func (b *singleUseDirectory) Size() (int64, error) { //TODO implement me panic("implement me") diff --git a/gateway/backend_car_test.go b/gateway/backend_car_test.go index eebd8e19b..89a077239 100644 --- a/gateway/backend_car_test.go +++ b/gateway/backend_car_test.go @@ -186,7 +186,7 @@ func TestCarBackendTar(t *testing.T) { require.NoError(t, err) nd, err := dsrv.Get(ctx, expected) require.NoError(t, err) - expectedFile, err := unixfile.NewUnixfsFile(ctx, dsrv, nd) + expectedFile, err := unixfile.NewUnixfsFile(ctx, dsrv, nd, nil) require.NoError(t, err) expectedFileData, err := io.ReadAll(expectedFile.(files.File)) @@ -361,7 +361,7 @@ func TestCarBackendTarAtEndOfPath(t *testing.T) { require.NoError(t, err) nd, err := dsrv.Get(ctx, expected) require.NoError(t, err) - expectedFile, err := unixfile.NewUnixfsFile(ctx, dsrv, nd) + expectedFile, err := unixfile.NewUnixfsFile(ctx, dsrv, nd, nil) require.NoError(t, err) expectedFileData, err := io.ReadAll(expectedFile.(files.File)) @@ -511,7 +511,7 @@ func TestCarBackendGetFile(t *testing.T) { dsrv := merkledag.NewDAGService(blockservice.New(robs, offline.Exchange(robs))) fileRootNd, err := dsrv.Get(ctx, cid.MustParse("bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa")) require.NoError(t, err) - uio, err := unixfile.NewUnixfsFile(ctx, dsrv, fileRootNd) + uio, err := unixfile.NewUnixfsFile(ctx, dsrv, fileRootNd, nil) require.NoError(t, err) f := uio.(files.File) expectedFileData, err := io.ReadAll(f) @@ -620,7 +620,7 @@ func TestCarBackendGetFileRangeRequest(t *testing.T) { dsrv := merkledag.NewDAGService(blockservice.New(robs, offline.Exchange(robs))) fileRootNd, err := dsrv.Get(ctx, cid.MustParse("bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa")) require.NoError(t, err) - uio, err := unixfile.NewUnixfsFile(ctx, dsrv, fileRootNd) + uio, err := unixfile.NewUnixfsFile(ctx, dsrv, fileRootNd, nil) require.NoError(t, err) f := uio.(files.File) _, err = f.Seek(int64(startIndex), io.SeekStart) @@ -725,7 +725,7 @@ func TestCarBackendGetFileWithBadBlockReturned(t *testing.T) { dsrv := merkledag.NewDAGService(blockservice.New(robs, offline.Exchange(robs))) fileRootNd, err := dsrv.Get(ctx, cid.MustParse("bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa")) require.NoError(t, err) - uio, err := unixfile.NewUnixfsFile(ctx, dsrv, fileRootNd) + uio, err := unixfile.NewUnixfsFile(ctx, dsrv, fileRootNd, nil) require.NoError(t, err) f := uio.(files.File) expectedFileData, err := io.ReadAll(f) diff --git a/gateway/handler_tar.go b/gateway/handler_tar.go index 0b2634804..c2f75c168 100644 --- a/gateway/handler_tar.go +++ b/gateway/handler_tar.go @@ -61,7 +61,7 @@ func (i *handler) serveTAR(ctx context.Context, w http.ResponseWriter, r *http.R w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^) // The TAR has a top-level directory (or file) named by the CID. - if err := tarw.WriteFile(file, rootCid.String()); err != nil { + if err := tarw.WriteNode(file, rootCid.String()); err != nil { // Update fail metric i.tarStreamFailMetric.WithLabelValues(rq.contentPath.Namespace()).Observe(time.Since(rq.begin).Seconds()) diff --git a/ipld/unixfs/file/unixfile.go b/ipld/unixfs/file/unixfile.go index 5ef968d1b..6e6622731 100644 --- a/ipld/unixfs/file/unixfile.go +++ b/ipld/unixfs/file/unixfile.go @@ -3,6 +3,8 @@ package unixfile import ( "context" "errors" + "os" + "time" ft "github.com/ipfs/boxo/ipld/unixfs" uio "github.com/ipfs/boxo/ipld/unixfs/io" @@ -21,6 +23,7 @@ type ufsDirectory struct { dserv ipld.DAGService dir uio.Directory size int64 + stat os.FileInfo } type ufsIterator struct { @@ -78,7 +81,7 @@ func (it *ufsIterator) Next() bool { } it.curName = l.Name - it.curFile, it.err = NewUnixfsFile(it.ctx, it.dserv, nd) + it.curFile, it.err = NewUnixfsFile(it.ctx, it.dserv, nd, nil) return it.err == nil } @@ -118,19 +121,48 @@ func (d *ufsDirectory) Entries() files.DirIterator { } } +func (d *ufsDirectory) Mode() os.FileMode { + if d.stat == nil { + return 0 + } + return d.stat.Mode() +} + +func (d *ufsDirectory) ModTime() time.Time { + if d.stat == nil { + return time.Time{} + } + return d.stat.ModTime() +} + func (d *ufsDirectory) Size() (int64, error) { return d.size, nil } type ufsFile struct { uio.DagReader + stat os.FileInfo +} + +func (f *ufsFile) Mode() os.FileMode { + if f.stat == nil { + return 0 + } + return f.stat.Mode() +} + +func (f *ufsFile) ModTime() time.Time { + if f.stat == nil { + return time.Time{} + } + return f.stat.ModTime() } func (f *ufsFile) Size() (int64, error) { return int64(f.DagReader.Size()), nil } -func newUnixfsDir(ctx context.Context, dserv ipld.DAGService, nd *dag.ProtoNode) (files.Directory, error) { +func newUnixfsDir(ctx context.Context, dserv ipld.DAGService, nd *dag.ProtoNode, stat os.FileInfo) (files.Directory, error) { dir, err := uio.NewDirectoryFromNode(dserv, nd) if err != nil { return nil, err @@ -147,10 +179,11 @@ func newUnixfsDir(ctx context.Context, dserv ipld.DAGService, nd *dag.ProtoNode) dir: dir, size: int64(size), + stat: stat, }, nil } -func NewUnixfsFile(ctx context.Context, dserv ipld.DAGService, nd ipld.Node) (files.Node, error) { +func NewUnixfsFile(ctx context.Context, dserv ipld.DAGService, nd ipld.Node, stat os.FileInfo) (files.Node, error) { switch dn := nd.(type) { case *dag.ProtoNode: fsn, err := ft.FSNodeFromBytes(dn.Data()) @@ -158,10 +191,10 @@ func NewUnixfsFile(ctx context.Context, dserv ipld.DAGService, nd ipld.Node) (fi return nil, err } if fsn.IsDir() { - return newUnixfsDir(ctx, dserv, dn) + return newUnixfsDir(ctx, dserv, dn, stat) } if fsn.Type() == ft.TSymlink { - return files.NewLinkFile(string(fsn.Data()), nil), nil + return files.NewLinkFile(string(fsn.Data()), stat), nil } case *dag.RawNode: diff --git a/tar/extractor.go b/tar/extractor.go index 8b9dfce6d..9ff62a931 100644 --- a/tar/extractor.go +++ b/tar/extractor.go @@ -7,7 +7,9 @@ import ( "io" "os" fp "path/filepath" + "runtime" "strings" + "time" ) var ( @@ -26,11 +28,14 @@ var ( // `cp`. In particular, the name of the extracted file/symlink will match the extraction path. If the extraction path // is a directory then it will extract into the directory using its original name. // +// If an associated mode and last modification time was stored in the archive it is restored. +// // Overwriting: Extraction of files and symlinks will result in overwriting the existing objects with the same name // when possible (i.e. other files, symlinks, and empty directories). type Extractor struct { - Path string - Progress func(int64) int64 + Path string + Progress func(int64) int64 + deferredUpdates []deferredUpdate } // Extract extracts a tar file to the file system. See the Extractor for more information on the limitations on the @@ -42,8 +47,6 @@ func (te *Extractor) Extract(reader io.Reader) error { tarReader := tar.NewReader(reader) - var firstObjectWasDir bool - header, err := tarReader.Next() if err != nil && err != io.EOF { return err @@ -52,6 +55,20 @@ func (te *Extractor) Extract(reader io.Reader) error { return errors.New("empty tar file") } + te.deferredUpdates = make([]deferredUpdate, 0, 80) + doUpdates := func() error { + for i := len(te.deferredUpdates) - 1; i >= 0; i-- { + m := te.deferredUpdates[i] + err := updateMeta(m.path, m.mode, m.mtime) + if err != nil { + return err + } + } + te.deferredUpdates = nil + return nil + } + defer func() { err = doUpdates() }() + // Specially handle the first entry assuming it is a single root object (e.g. root directory, single file, // or single symlink) @@ -71,6 +88,8 @@ func (te *Extractor) Extract(reader io.Reader) error { return err } + var firstObjectWasDir bool + // If the last element in the rootOutputPath (which is passed by the user) is a symlink do not follow it // this makes it easier for users to reason about where files are getting extracted to even when the tar is not // from a trusted source @@ -85,6 +104,9 @@ func (te *Extractor) Extract(reader io.Reader) error { if err := te.extractDir(rootOutputPath); err != nil { return err } + if err := te.deferUpdate(rootOutputPath, header); err != nil { + return err + } case tar.TypeReg, tar.TypeSymlink: // Check if the output path already exists, so we know whether we should // create our output with that name, or if we should put the output inside @@ -117,6 +139,9 @@ func (te *Extractor) Extract(reader io.Reader) error { if err := te.extractFile(outputPath, tarReader); err != nil { return err } + if err := updateMeta(outputPath, header.Mode, header.ModTime); err != nil { + return err + } } else if err := te.extractSymlink(outputPath, header); err != nil { return err } @@ -173,10 +198,16 @@ func (te *Extractor) Extract(reader io.Reader) error { if err := te.extractDir(outputPath); err != nil { return err } + if err := te.deferUpdate(outputPath, header); err != nil { + return err + } case tar.TypeReg: if err := te.extractFile(outputPath, tarReader); err != nil { return err } + if err := updateMeta(outputPath, header.Mode, header.ModTime); err != nil { + return err + } case tar.TypeSymlink: if err := te.extractSymlink(outputPath, header); err != nil { return err @@ -268,16 +299,28 @@ func (te *Extractor) extractDir(path string) error { } func (te *Extractor) extractSymlink(path string, h *tar.Header) error { - if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + err := os.Remove(path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + + err = os.Symlink(h.Linkname, path) + if err != nil { return err } - return os.Symlink(h.Linkname, path) + switch runtime.GOOS { + case "linux", "freebsd", "netbsd", "openbsd", "dragonfly": + return updateModTime(path, h.ModTime) + default: + return nil + } } func (te *Extractor) extractFile(path string, r *tar.Reader) error { // Attempt removing the target so we can overwrite files, symlinks and empty directories - if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + err := os.Remove(path) + if err != nil && !errors.Is(err, os.ErrNotExist) { return err } @@ -288,17 +331,17 @@ func (te *Extractor) extractFile(path string, r *tar.Reader) error { if err != nil { return err } - if err := copyWithProgress(tmpfile, r, te.Progress); err != nil { + if err = copyWithProgress(tmpfile, r, te.Progress); err != nil { _ = tmpfile.Close() _ = os.Remove(tmpfile.Name()) return err } - if err := tmpfile.Close(); err != nil { + if err = tmpfile.Close(); err != nil { _ = os.Remove(tmpfile.Name()) return err } - if err := os.Rename(tmpfile.Name(), path); err != nil { + if err = os.Rename(tmpfile.Name(), path); err != nil { _ = os.Remove(tmpfile.Name()) return err } @@ -327,3 +370,70 @@ func copyWithProgress(to io.Writer, from io.Reader, cb func(int64) int64) error } } } + +type deferredUpdate struct { + path string + mode int64 + mtime time.Time +} + +func (te *Extractor) deferUpdate(path string, header *tar.Header) error { + if header.Mode == 0 && header.ModTime.IsZero() { + return nil + } + + prefix := func() string { + for i := len(path) - 1; i >= 0; i-- { + if path[i] == '/' { + return path[:i] + } + } + return path + } + + n := len(te.deferredUpdates) + if n > 0 && len(path) < len(te.deferredUpdates[n-1].path) { + // if possible, apply the previous deferral + m := te.deferredUpdates[n-1] + if strings.HasPrefix(m.path, prefix()) { + err := updateMeta(m.path, m.mode, m.mtime) + if err != nil { + return err + } + te.deferredUpdates = te.deferredUpdates[:n-1] + } + } + + te.deferredUpdates = append(te.deferredUpdates, deferredUpdate{ + path: path, + mode: header.Mode, + mtime: header.ModTime, + }) + + return nil +} + +func updateMeta(path string, mode int64, mtime time.Time) error { + if err := updateModTime(path, mtime); err != nil { + return err + } + return updateFileMode(path, mode) +} + +// updateFileMode sets the unix mode of the filesystem object referenced by path +func updateFileMode(path string, mode int64) error { + if err := updateMode(path, mode); err != nil { + return fmt.Errorf("[%v] failed to update file mode on '%s'", err, path) + } + return nil +} + +// updateModTime sets the last access and modification time of the target filesystem +// object to the given time. +// When the given path references a symlink, if supported the symlink is updated. +func updateModTime(path string, mtime time.Time) error { + if err := updateMtime(path, mtime); err != nil { + return fmt.Errorf("[%v] failed to update last modification time on '%s'", err, path) + } + return nil +} diff --git a/tar/extractor_test.go b/tar/extractor_test.go index d2b4e00fc..336ec7626 100644 --- a/tar/extractor_test.go +++ b/tar/extractor_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/ipfs/boxo/files" "github.com/stretchr/testify/assert" ) @@ -50,7 +51,7 @@ func TestSingleFile(t *testing.T) { fileData := "file data" testTarExtraction(t, nil, []tarEntry{ - &fileTarEntry{fileName, []byte(fileData)}, + &fileTarEntry{path: fileName, buf: []byte(fileData)}, }, func(t *testing.T, extractDir string) { f, err := os.Open(fp.Join(extractDir, fileName)) @@ -64,13 +65,60 @@ func TestSingleFile(t *testing.T) { ) } +func TestSingleFileWithMeta(t *testing.T) { + fileName := "file2..ext" + fileData := "file2 data" + mode := 0654 + mtime := time.Now().Round(time.Second) + + testTarExtraction(t, nil, []tarEntry{ + &fileTarEntry{path: fileName, buf: []byte(fileData), mode: mode, mtime: mtime}, + }, + func(t *testing.T, extractDir string) { + path := fp.Join(extractDir, fileName) + testMeta(t, path, mode, mtime) + f, err := os.Open(path) + assert.NoError(t, err) + data, err := io.ReadAll(f) + assert.NoError(t, err) + assert.Equal(t, fileData, string(data)) + assert.NoError(t, f.Close()) + }, + nil, + ) +} + func TestSingleDirectory(t *testing.T) { dirName := "dir..sfx" testTarExtraction(t, nil, []tarEntry{ - &dirTarEntry{dirName}, + &dirTarEntry{path: dirName}, + }, + func(t *testing.T, extractDir string) { + f, err := os.Open(extractDir) + if err != nil { + t.Fatal(err) + } + objs, err := f.Readdir(1) + if err == io.EOF && len(objs) == 0 { + return + } + t.Fatalf("expected an empty directory") + }, + nil, + ) +} + +func TestSingleDirectoryWithMeta(t *testing.T) { + dirName := "dir2..sfx" + mode := 0765 + mtime := time.Now().Round(time.Second) + + testTarExtraction(t, nil, []tarEntry{ + &dirTarEntry{path: dirName, mode: mode, mtime: mtime}, }, func(t *testing.T, extractDir string) { + testMeta(t, extractDir, mode, mtime) f, err := os.Open(extractDir) if err != nil { t.Fatal(err) @@ -90,8 +138,8 @@ func TestDirectoryFollowSymlinkToNothing(t *testing.T) { childName := "child" entries := []tarEntry{ - &dirTarEntry{dirName}, - &dirTarEntry{dirName + "/" + childName}, + &dirTarEntry{path: dirName}, + &dirTarEntry{path: dirName + "/" + childName}, } testTarExtraction(t, func(t *testing.T, rootDir string) { @@ -109,8 +157,8 @@ func TestDirectoryFollowSymlinkToFile(t *testing.T) { childName := "child" entries := []tarEntry{ - &dirTarEntry{dirName}, - &dirTarEntry{dirName + "/" + childName}, + &dirTarEntry{path: dirName}, + &dirTarEntry{path: dirName + "/" + childName}, } testTarExtraction(t, func(t *testing.T, rootDir string) { @@ -132,8 +180,8 @@ func TestDirectoryFollowSymlinkToDirectory(t *testing.T) { childName := "child" entries := []tarEntry{ - &dirTarEntry{dirName}, - &dirTarEntry{dirName + "/" + childName}, + &dirTarEntry{path: dirName}, + &dirTarEntry{path: dirName + "/" + childName}, } testTarExtraction(t, func(t *testing.T, rootDir string) { @@ -159,7 +207,7 @@ func TestSingleSymlink(t *testing.T) { symlinkName := "symlink" testTarExtraction(t, nil, []tarEntry{ - &symlinkTarEntry{targetName, symlinkName}, + &symlinkTarEntry{target: targetName, path: symlinkName}, }, func(t *testing.T, extractDir string) { symlinkPath := fp.Join(extractDir, symlinkName) fi, err := os.Lstat(symlinkPath) @@ -177,37 +225,37 @@ func TestSingleSymlink(t *testing.T) { func TestMultipleRoots(t *testing.T) { testTarExtraction(t, nil, []tarEntry{ - &dirTarEntry{"root"}, - &dirTarEntry{"sibling"}, + &dirTarEntry{path: "root"}, + &dirTarEntry{path: "sibling"}, }, nil, errInvalidRoot) } func TestMultipleRootsNested(t *testing.T) { testTarExtraction(t, nil, []tarEntry{ - &dirTarEntry{"root/child1"}, - &dirTarEntry{"root/child2"}, + &dirTarEntry{path: "root/child1"}, + &dirTarEntry{path: "root/child2"}, }, nil, errInvalidRoot) } func TestOutOfOrderRoot(t *testing.T) { testTarExtraction(t, nil, []tarEntry{ - &dirTarEntry{"root/child"}, - &dirTarEntry{"root"}, + &dirTarEntry{path: "root/child"}, + &dirTarEntry{path: "root"}, }, nil, errInvalidRoot) } func TestOutOfOrder(t *testing.T) { testTarExtraction(t, nil, []tarEntry{ - &dirTarEntry{"root/child/grandchild"}, - &dirTarEntry{"root/child"}, + &dirTarEntry{path: "root/child/grandchild"}, + &dirTarEntry{path: "root/child"}, }, nil, errInvalidRoot) } func TestNestedDirectories(t *testing.T) { testTarExtraction(t, nil, []tarEntry{ - &dirTarEntry{"root"}, - &dirTarEntry{"root/child"}, - &dirTarEntry{"root/child/grandchild"}, + &dirTarEntry{path: "root"}, + &dirTarEntry{path: "root/child"}, + &dirTarEntry{path: "root/child/grandchild"}, }, func(t *testing.T, extractDir string) { walkIndex := 0 err := fp.Walk(extractDir, @@ -234,19 +282,161 @@ func TestNestedDirectories(t *testing.T) { func TestRootDirectoryHasSubpath(t *testing.T) { testTarExtraction(t, nil, []tarEntry{ - &dirTarEntry{"root/child"}, - &dirTarEntry{"root/child/grandchild"}, + &dirTarEntry{path: "root/child"}, + &dirTarEntry{path: "root/child/grandchild"}, }, nil, errInvalidRoot) } func TestFilesAndFolders(t *testing.T) { testTarExtraction(t, nil, []tarEntry{ - &dirTarEntry{"root"}, - &dirTarEntry{"root/childdir"}, - &fileTarEntry{"root/childdir/file1", []byte("some data")}, + &dirTarEntry{path: "root"}, + &dirTarEntry{path: "root/childdir"}, + &fileTarEntry{path: "root/childdir/file1", buf: []byte("some data")}, }, nil, nil) } +func TestFilesAndFoldersWithMetadata(t *testing.T) { + tm := time.Unix(660000000, 0) + + entries := []tarEntry{ + &dirTarEntry{path: "root", mtime: tm.Add(5 * time.Second)}, + &dirTarEntry{path: "root/childdir", mode: 03775}, + &fileTarEntry{path: "root/childdir/file1", buf: []byte("some data"), mode: 04744, + mtime: tm.Add(10 * time.Second)}, + &fileTarEntry{path: "root/childdir/file2", buf: []byte("some data"), mode: 0560, + mtime: tm.Add(10 * time.Second)}, + &fileTarEntry{path: "root/childdir/file3", buf: []byte("some data"), mode: 06540, + mtime: tm.Add(10 * time.Second)}, + } + + testTarExtraction(t, nil, entries, func(t *testing.T, extractDir string) { + walkIndex := 0 + err := fp.Walk(extractDir, + func(path string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + switch walkIndex { + case 0: // root + assert.Equal(t, tm.Add(5*time.Second), fi.ModTime()) + case 1: // childdir + if runtime.GOOS != "windows" { + assert.Equal(t, 0775, int(fi.Mode()&0xFFF)) + assert.Equal(t, os.ModeSetgid, fi.Mode()&os.ModeSetgid) + assert.Equal(t, os.ModeSticky, fi.Mode()&os.ModeSticky) + } else { + assert.Equal(t, 0777, int(fi.Mode()&0xFFF)) + } + case 2: // file1 + assert.Equal(t, tm.Add(10*time.Second), fi.ModTime()) + if runtime.GOOS != "windows" { + assert.Equal(t, 0744, int(fi.Mode()&0xFFF)) + assert.Equal(t, os.ModeSetuid, fi.Mode()&os.ModeSetuid) + } else { + assert.Equal(t, 0666, int(fi.Mode()&0xFFF)) + } + case 3: // file2 + assert.Equal(t, tm.Add(10*time.Second), fi.ModTime()) + if runtime.GOOS != "windows" { + assert.Equal(t, 0560, int(fi.Mode()&0xFFF)) + assert.Equal(t, 0, int(fi.Mode()&os.ModeSetuid)) + } else { + assert.Equal(t, 0666, int(fi.Mode()&0xFFF)) + } + case 4: // file3 + assert.Equal(t, tm.Add(10*time.Second), fi.ModTime()) + if runtime.GOOS != "windows" { + assert.Equal(t, 0540, int(fi.Mode()&0xFFF)) + assert.Equal(t, os.ModeSetgid, fi.Mode()&os.ModeSetgid) + assert.Equal(t, os.ModeSetuid, fi.Mode()&os.ModeSetuid) + } else { + assert.Equal(t, 0444, int(fi.Mode()&0xFFF)) + } + default: + assert.Fail(t, "has more than 5 entries", path) + } + walkIndex++ + return nil + }) + assert.NoError(t, err) + }, + nil) +} + +func TestSymlinkWithModTime(t *testing.T) { + if !symlinksEnabled { + t.Skip("symlinks disabled on this platform", symlinksEnabledErr) + } + if runtime.GOOS == "darwin" { + t.Skip("changing symlink modification time is not currently supported on darwin") + } + tm := time.Unix(660000000, 0) + add5 := func() time.Time { + tm = tm.Add(5 * time.Second) + return tm + } + + entries := []tarEntry{ + &dirTarEntry{path: "root", mtime: add5()}, + &symlinkTarEntry{target: "child", path: "root/a", mtime: add5()}, + &dirTarEntry{path: "root/child", mtime: add5()}, + &fileTarEntry{path: "root/child/file1", buf: []byte("data")}, + &symlinkTarEntry{target: "child/file1", path: "root/file1-sl", mtime: add5()}, + } + + testTarExtraction(t, nil, entries, func(t *testing.T, extractDir string) { + tm = time.Unix(660000000, 0) + + fi, err := os.Lstat(fp.Join(extractDir, "a")) + assert.NoError(t, err) + assert.Equal(t, add5(), fi.ModTime()) + + fi, err = os.Lstat(fp.Join(extractDir, "file1-sl")) + assert.NoError(t, err) + assert.Equal(t, add5(), fi.ModTime()) + }, + nil) +} + +func TestDeferredUpdate(t *testing.T) { + tm := time.Unix(660000000, 0) + add5 := func() time.Time { + tm = tm.Add(5 * time.Second) + return tm + } + + // must be in lexical order + entries := []tarEntry{ + &dirTarEntry{path: "root", mtime: add5()}, + &dirTarEntry{path: "root/a", mtime: add5()}, + &dirTarEntry{path: "root/a/beta", mtime: add5(), mode: 0500}, + &dirTarEntry{path: "root/a/beta/centauri", mtime: add5()}, + &dirTarEntry{path: "root/a/beta/lima", mtime: add5()}, + &dirTarEntry{path: "root/a/beta/papa", mtime: add5()}, + &dirTarEntry{path: "root/a/beta/xanadu", mtime: add5()}, + &dirTarEntry{path: "root/a/beta/z", mtime: add5()}, + &dirTarEntry{path: "root/a/delta", mtime: add5()}, + &dirTarEntry{path: "root/iota", mtime: add5()}, + &dirTarEntry{path: "root/q", mtime: add5()}, + } + + testTarExtraction(t, nil, entries, func(t *testing.T, extractDir string) { + tm = time.Unix(660000000, 0) + err := fp.Walk(extractDir, + func(path string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + + assert.Equal(t, add5(), fi.ModTime()) + return nil + }) + assert.NoError(t, err) + }, + nil) + +} + func TestInternalSymlinkTraverse(t *testing.T) { if !symlinksEnabled { t.Skip("symlinks disabled on this platform", symlinksEnabledErr) @@ -254,10 +444,10 @@ func TestInternalSymlinkTraverse(t *testing.T) { testTarExtraction(t, nil, []tarEntry{ // FIXME: We are ignoring the first element in the path check so // we add a directory at the start to bypass this. - &dirTarEntry{"root"}, - &dirTarEntry{"root/child"}, - &symlinkTarEntry{"child", "root/symlink-dir"}, - &fileTarEntry{"root/symlink-dir/file", []byte("file")}, + &dirTarEntry{path: "root"}, + &dirTarEntry{path: "root/child"}, + &symlinkTarEntry{target: "child", path: "root/symlink-dir"}, + &fileTarEntry{path: "root/symlink-dir/file", buf: []byte("file")}, }, nil, errTraverseSymlink, @@ -271,9 +461,9 @@ func TestExternalSymlinkTraverse(t *testing.T) { testTarExtraction(t, nil, []tarEntry{ // FIXME: We are ignoring the first element in the path check so // we add a directory at the start to bypass this. - &dirTarEntry{"inner"}, - &symlinkTarEntry{"..", "inner/symlink-dir"}, - &fileTarEntry{"inner/symlink-dir/file", []byte("overwrite content")}, + &dirTarEntry{path: "inner"}, + &symlinkTarEntry{target: "..", path: "inner/symlink-dir"}, + &fileTarEntry{path: "inner/symlink-dir/file", buf: []byte("overwrite content")}, }, nil, errTraverseSymlink, @@ -295,9 +485,9 @@ func TestLastElementOverwrite(t *testing.T) { assert.Equal(t, len(originalData), n) }, []tarEntry{ - &dirTarEntry{"root"}, - &symlinkTarEntry{"../outside-ref", "root/symlink"}, - &fileTarEntry{"root/symlink", []byte("overwrite content")}, + &dirTarEntry{path: "root"}, + &symlinkTarEntry{target: "../outside-ref", path: "root/symlink"}, + &fileTarEntry{path: "root/symlink", buf: []byte("overwrite content")}, }, func(t *testing.T, extractDir string) { // Check that outside-ref still exists but has not been @@ -325,18 +515,12 @@ func testTarExtraction(t *testing.T, setup func(t *testing.T, rootDir string), t err = os.MkdirAll(extractDir, 0o755) assert.NoError(t, err) - // Generated TAR file. - tarFilename := fp.Join(rootDir, "generated.tar") - tarFile, err := os.Create(tarFilename) - assert.NoError(t, err) - defer tarFile.Close() - tw := tar.NewWriter(tarFile) - defer tw.Close() - if setup != nil { setup(t, rootDir) } + // Generated TAR file. + tarFilename := fp.Join(rootDir, "generated.tar") writeTarFile(t, tarFilename, tarEntries) testExtract(t, tarFilename, extractDir, extractError) @@ -358,6 +542,23 @@ func testExtract(t *testing.T, tarFile string, extractDir string, expectedError assert.ErrorIs(t, err, expectedError) } +func testMeta(t *testing.T, path string, mode int, now time.Time) { + fi, err := os.Lstat(path) + assert.NoError(t, err) + m := files.ModePermsToUnixPerms(fi.Mode()) + if runtime.GOOS == "windows" { + if fi.IsDir() { + mode = 0777 + } else if mode&0220 != 0 { + mode = 0666 + } else if mode&0440 != 0 { + mode = 0444 + } + } + assert.Equal(t, mode, int(m)) + assert.Equal(t, now.Unix(), fi.ModTime().Unix()) +} + // Based on the `writeXXXHeader` family of functions in // github.com/ipfs/go-ipfs-files@v0.0.8/tarwriter.go. func writeTarFile(t *testing.T, path string, entries []tarEntry) { @@ -385,16 +586,19 @@ var ( ) type fileTarEntry struct { - path string - buf []byte + path string + buf []byte + mode int + mtime time.Time } func (e *fileTarEntry) write(tw *tar.Writer) error { - if err := writeFileHeader(tw, e.path, uint64(len(e.buf))); err != nil { + err := writeFileHeader(tw, e.path, uint64(len(e.buf)), e.mode, e.mtime) + if err != nil { return err } - if _, err := io.Copy(tw, bytes.NewReader(e.buf)); err != nil { + if _, err = io.Copy(tw, bytes.NewReader(e.buf)); err != nil { return err } @@ -402,34 +606,35 @@ func (e *fileTarEntry) write(tw *tar.Writer) error { return nil } -func writeFileHeader(w *tar.Writer, fpath string, size uint64) error { +func writeFileHeader(w *tar.Writer, fpath string, size uint64, mode int, mtime time.Time) error { return w.WriteHeader(&tar.Header{ Name: fpath, Size: int64(size), Typeflag: tar.TypeReg, - Mode: 0o644, - ModTime: time.Now(), - // TODO: set mode, dates, etc. when added to unixFS + Mode: int64(mode), + ModTime: mtime, }) } type dirTarEntry struct { - path string + path string + mode int + mtime time.Time } func (e *dirTarEntry) write(tw *tar.Writer) error { return tw.WriteHeader(&tar.Header{ Name: e.path, Typeflag: tar.TypeDir, - Mode: 0o777, - ModTime: time.Now(), - // TODO: set mode, dates, etc. when added to unixFS + Mode: int64(e.mode), + ModTime: e.mtime, }) } type symlinkTarEntry struct { target string path string + mtime time.Time } func (e *symlinkTarEntry) write(w *tar.Writer) error { @@ -437,6 +642,7 @@ func (e *symlinkTarEntry) write(w *tar.Writer) error { Name: e.path, Linkname: e.target, Mode: 0o777, + ModTime: e.mtime, Typeflag: tar.TypeSymlink, }) } diff --git a/tar/util_other.go b/tar/util_other.go new file mode 100644 index 000000000..99ae84367 --- /dev/null +++ b/tar/util_other.go @@ -0,0 +1,25 @@ +//go:build !linux && !freebsd && !netbsd && !openbsd && !dragonfly && !windows +// +build !linux,!freebsd,!netbsd,!openbsd,!dragonfly,!windows + +package tar + +import ( + "os" + "time" + + "github.com/ipfs/boxo/files" +) + +func updateMode(path string, mode int64) error { + if mode != 0 { + return os.Chmod(path, files.UnixPermsToModePerms(uint32(mode))) + } + return nil +} + +func updateMtime(path string, mtime time.Time) error { + if !mtime.IsZero() { + return os.Chtimes(path, mtime, mtime) + } + return nil +} diff --git a/tar/util_posix.go b/tar/util_posix.go new file mode 100644 index 000000000..418dc7dd6 --- /dev/null +++ b/tar/util_posix.go @@ -0,0 +1,41 @@ +//go:build linux || freebsd || netbsd || openbsd || dragonfly +// +build linux freebsd netbsd openbsd dragonfly + +package tar + +import ( + "golang.org/x/sys/unix" + "os" + "syscall" + "time" + "unsafe" + + "github.com/ipfs/boxo/files" +) + +func updateMode(path string, mode int64) error { + if mode != 0 { + return os.Chmod(path, files.UnixPermsToModePerms(uint32(mode))) + } + return nil +} + +func updateMtime(path string, mtime time.Time) error { + if !mtime.IsZero() { + var AtFdCwd = -100 + pathname, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + + tm := syscall.NsecToTimespec(mtime.UnixNano()) + ts := [2]syscall.Timespec{tm, tm} + _, _, e := syscall.Syscall6(syscall.SYS_UTIMENSAT, uintptr(AtFdCwd), + uintptr(unsafe.Pointer(pathname)), uintptr(unsafe.Pointer(&ts)), + uintptr(unix.AT_SYMLINK_NOFOLLOW), 0, 0) + if e != 0 { + return error(e) + } + } + return nil +} diff --git a/tar/util_windows.go b/tar/util_windows.go new file mode 100644 index 000000000..b378e45d1 --- /dev/null +++ b/tar/util_windows.go @@ -0,0 +1,31 @@ +package tar + +import ( + "os" + "time" +) + +// os.Chmod - On Windows, only the 0200 bit (owner writable) of mode is used; It +// controls whether the file's read-only attribute is set or cleared. The other +// bits are currently unused. +// +// Use mode 0400 for a read-only file and 0600 for a readable+writable file. +func updateMode(path string, mode int64) error { + if mode != 0 { + // read+write if owner, group or world writeable + if mode&0222 != 0 { + return os.Chmod(path, 0600) + } + // otherwise read-only + return os.Chmod(path, 0400) + } + + return nil +} + +func updateMtime(path string, mtime time.Time) error { + if !mtime.IsZero() { + return os.Chtimes(path, mtime, mtime) + } + return nil +} From 312a27fd7250c6781cd333231366d9c9f30fd719 Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Tue, 6 Aug 2024 14:48:09 -0700 Subject: [PATCH 02/18] Fix tests, update changelog --- CHANGELOG.md | 2 ++ examples/unixfs-file-cid/main.go | 2 +- tar/extractor_test.go | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17e71b53b..3494d094c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ The following emojis are used to highlight certain changes: ### Added +- support UnixFS 1.5 file mode and modification times + ### Changed - `go-libp2p` dependency updated to [v0.36 (release notes)](https://github.com/libp2p/go-libp2p/releases/tag/v0.36.1) diff --git a/examples/unixfs-file-cid/main.go b/examples/unixfs-file-cid/main.go index e1adad350..13fa2aa32 100644 --- a/examples/unixfs-file-cid/main.go +++ b/examples/unixfs-file-cid/main.go @@ -215,7 +215,7 @@ func runClient(ctx context.Context, h host.Host, c cid.Cid, targetPeer string) ( return nil, err } - unixFSNode, err := unixfile.NewUnixfsFile(ctx, dserv, nd) + unixFSNode, err := unixfile.NewUnixfsFile(ctx, dserv, nd, nil) if err != nil { return nil, err } diff --git a/tar/extractor_test.go b/tar/extractor_test.go index 336ec7626..86dbd889e 100644 --- a/tar/extractor_test.go +++ b/tar/extractor_test.go @@ -377,9 +377,9 @@ func TestSymlinkWithModTime(t *testing.T) { } entries := []tarEntry{ - &dirTarEntry{path: "root", mtime: add5()}, + &dirTarEntry{path: "root"}, &symlinkTarEntry{target: "child", path: "root/a", mtime: add5()}, - &dirTarEntry{path: "root/child", mtime: add5()}, + &dirTarEntry{path: "root/child"}, &fileTarEntry{path: "root/child/file1", buf: []byte("data")}, &symlinkTarEntry{target: "child/file1", path: "root/file1-sl", mtime: add5()}, } From 7371b764ad1e6c668874d3ba39072f175079d211 Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Tue, 6 Aug 2024 15:17:43 -0700 Subject: [PATCH 03/18] - Add NewUnixfsFileWithStat - Fix windows test --- examples/unixfs-file-cid/main.go | 2 +- gateway/backend_blocks.go | 6 +++--- gateway/backend_car_test.go | 10 +++++----- ipld/unixfs/file/unixfile.go | 8 ++++++-- tar/extractor_test.go | 10 ++++++++-- 5 files changed, 23 insertions(+), 13 deletions(-) diff --git a/examples/unixfs-file-cid/main.go b/examples/unixfs-file-cid/main.go index 13fa2aa32..e1adad350 100644 --- a/examples/unixfs-file-cid/main.go +++ b/examples/unixfs-file-cid/main.go @@ -215,7 +215,7 @@ func runClient(ctx context.Context, h host.Host, c cid.Cid, targetPeer string) ( return nil, err } - unixFSNode, err := unixfile.NewUnixfsFile(ctx, dserv, nd, nil) + unixFSNode, err := unixfile.NewUnixfsFile(ctx, dserv, nd) if err != nil { return nil, err } diff --git a/gateway/backend_blocks.go b/gateway/backend_blocks.go index 3e578e8d1..42440dfcd 100644 --- a/gateway/backend_blocks.go +++ b/gateway/backend_blocks.go @@ -150,7 +150,7 @@ func (bb *BlocksBackend) Get(ctx context.Context, path path.ImmutablePath, range } // This code path covers full graph, single file/directory, and range requests - f, err := ufile.NewUnixfsFile(ctx, bb.dagService, nd, nil) + f, err := ufile.NewUnixfsFile(ctx, bb.dagService, nd) // Note: there is an assumption here that non-UnixFS dag-pb should not be returned which is currently valid if err != nil { return md, nil, err @@ -197,7 +197,7 @@ func (bb *BlocksBackend) GetAll(ctx context.Context, path path.ImmutablePath) (C } // This code path covers full graph, single file/directory, and range requests - n, err := ufile.NewUnixfsFile(ctx, bb.dagService, nd, nil) + n, err := ufile.NewUnixfsFile(ctx, bb.dagService, nd) if err != nil { return md, nil, err } @@ -226,7 +226,7 @@ func (bb *BlocksBackend) Head(ctx context.Context, path path.ImmutablePath) (Con // TODO: We're not handling non-UnixFS dag-pb. There's a bit of a discrepancy // between what we want from a HEAD request and a Resolve request here and we're using this for both - fileNode, err := ufile.NewUnixfsFile(ctx, bb.dagService, nd, nil) + fileNode, err := ufile.NewUnixfsFile(ctx, bb.dagService, nd) if err != nil { return ContentPathMetadata{}, nil, err } diff --git a/gateway/backend_car_test.go b/gateway/backend_car_test.go index 89a077239..eebd8e19b 100644 --- a/gateway/backend_car_test.go +++ b/gateway/backend_car_test.go @@ -186,7 +186,7 @@ func TestCarBackendTar(t *testing.T) { require.NoError(t, err) nd, err := dsrv.Get(ctx, expected) require.NoError(t, err) - expectedFile, err := unixfile.NewUnixfsFile(ctx, dsrv, nd, nil) + expectedFile, err := unixfile.NewUnixfsFile(ctx, dsrv, nd) require.NoError(t, err) expectedFileData, err := io.ReadAll(expectedFile.(files.File)) @@ -361,7 +361,7 @@ func TestCarBackendTarAtEndOfPath(t *testing.T) { require.NoError(t, err) nd, err := dsrv.Get(ctx, expected) require.NoError(t, err) - expectedFile, err := unixfile.NewUnixfsFile(ctx, dsrv, nd, nil) + expectedFile, err := unixfile.NewUnixfsFile(ctx, dsrv, nd) require.NoError(t, err) expectedFileData, err := io.ReadAll(expectedFile.(files.File)) @@ -511,7 +511,7 @@ func TestCarBackendGetFile(t *testing.T) { dsrv := merkledag.NewDAGService(blockservice.New(robs, offline.Exchange(robs))) fileRootNd, err := dsrv.Get(ctx, cid.MustParse("bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa")) require.NoError(t, err) - uio, err := unixfile.NewUnixfsFile(ctx, dsrv, fileRootNd, nil) + uio, err := unixfile.NewUnixfsFile(ctx, dsrv, fileRootNd) require.NoError(t, err) f := uio.(files.File) expectedFileData, err := io.ReadAll(f) @@ -620,7 +620,7 @@ func TestCarBackendGetFileRangeRequest(t *testing.T) { dsrv := merkledag.NewDAGService(blockservice.New(robs, offline.Exchange(robs))) fileRootNd, err := dsrv.Get(ctx, cid.MustParse("bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa")) require.NoError(t, err) - uio, err := unixfile.NewUnixfsFile(ctx, dsrv, fileRootNd, nil) + uio, err := unixfile.NewUnixfsFile(ctx, dsrv, fileRootNd) require.NoError(t, err) f := uio.(files.File) _, err = f.Seek(int64(startIndex), io.SeekStart) @@ -725,7 +725,7 @@ func TestCarBackendGetFileWithBadBlockReturned(t *testing.T) { dsrv := merkledag.NewDAGService(blockservice.New(robs, offline.Exchange(robs))) fileRootNd, err := dsrv.Get(ctx, cid.MustParse("bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa")) require.NoError(t, err) - uio, err := unixfile.NewUnixfsFile(ctx, dsrv, fileRootNd, nil) + uio, err := unixfile.NewUnixfsFile(ctx, dsrv, fileRootNd) require.NoError(t, err) f := uio.(files.File) expectedFileData, err := io.ReadAll(f) diff --git a/ipld/unixfs/file/unixfile.go b/ipld/unixfs/file/unixfile.go index 6e6622731..0a923e937 100644 --- a/ipld/unixfs/file/unixfile.go +++ b/ipld/unixfs/file/unixfile.go @@ -81,7 +81,7 @@ func (it *ufsIterator) Next() bool { } it.curName = l.Name - it.curFile, it.err = NewUnixfsFile(it.ctx, it.dserv, nd, nil) + it.curFile, it.err = NewUnixfsFile(it.ctx, it.dserv, nd) return it.err == nil } @@ -183,7 +183,11 @@ func newUnixfsDir(ctx context.Context, dserv ipld.DAGService, nd *dag.ProtoNode, }, nil } -func NewUnixfsFile(ctx context.Context, dserv ipld.DAGService, nd ipld.Node, stat os.FileInfo) (files.Node, error) { +func NewUnixfsFile(ctx context.Context, dserv ipld.DAGService, nd ipld.Node) (files.Node, error) { + return NewUnixfsFileWithStat(ctx, dserv, nd, nil) +} + +func NewUnixfsFileWithStat(ctx context.Context, dserv ipld.DAGService, nd ipld.Node, stat os.FileInfo) (files.Node, error) { switch dn := nd.(type) { case *dag.ProtoNode: fsn, err := ft.FSNodeFromBytes(dn.Data()) diff --git a/tar/extractor_test.go b/tar/extractor_test.go index 86dbd889e..c8aa5833e 100644 --- a/tar/extractor_test.go +++ b/tar/extractor_test.go @@ -389,11 +389,17 @@ func TestSymlinkWithModTime(t *testing.T) { fi, err := os.Lstat(fp.Join(extractDir, "a")) assert.NoError(t, err) - assert.Equal(t, add5(), fi.ModTime()) + add5() + if runtime.GOOS != "windows" { + assert.Equal(t, tm, fi.ModTime()) + } fi, err = os.Lstat(fp.Join(extractDir, "file1-sl")) assert.NoError(t, err) - assert.Equal(t, add5(), fi.ModTime()) + add5() + if runtime.GOOS != "windows" { + assert.Equal(t, tm, fi.ModTime()) + } }, nil) } From 4f392d5936a7a9754e764f90723049a0cfcf70e0 Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Tue, 6 Aug 2024 15:52:48 -0700 Subject: [PATCH 04/18] Rename tarwriter.WriteNode to tarwriter.WriteFile --- files/tarwriter.go | 6 +++--- files/tarwriter_test.go | 8 ++++---- gateway/handler_tar.go | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/files/tarwriter.go b/files/tarwriter.go index f8b1d7791..a7d2b658f 100644 --- a/files/tarwriter.go +++ b/files/tarwriter.go @@ -34,7 +34,7 @@ func (w *TarWriter) writeDir(f Directory, fpath string) error { it := f.Entries() for it.Next() { - if err := w.WriteNode(it.Node(), path.Join(fpath, it.Name())); err != nil { + if err := w.WriteFile(it.Node(), path.Join(fpath, it.Name())); err != nil { return err } } @@ -78,8 +78,8 @@ func validateTarFilePath(baseDir, fpath string) bool { return true } -// WriteNode adds a node to the archive. -func (w *TarWriter) WriteNode(nd Node, fpath string) error { +// WriteFile adds a node to the archive. +func (w *TarWriter) WriteFile(nd Node, fpath string) error { if !w.baseDirSet { w.baseDirSet = true // Use a variable for this as baseDir may be an empty string. w.baseDir = fpath diff --git a/files/tarwriter_test.go b/files/tarwriter_test.go index 3b71ebc4a..559d77f3d 100644 --- a/files/tarwriter_test.go +++ b/files/tarwriter_test.go @@ -30,7 +30,7 @@ func TestTarWriter(t *testing.T) { go func() { defer tw.Close() - if err := tw.WriteNode(tf, ""); err != nil { + if err := tw.WriteFile(tf, ""); err != nil { t.Error(err) } }() @@ -116,7 +116,7 @@ func TestTarWriterRelativePathInsideRoot(t *testing.T) { } defer tw.Close() - if err = tw.WriteNode(tf, ""); err != nil { + if err = tw.WriteFile(tf, ""); err != nil { t.Error(err) } } @@ -137,7 +137,7 @@ func TestTarWriterFailsFileOutsideRoot(t *testing.T) { } defer tw.Close() - if err = tw.WriteNode(tf, ""); !errors.Is(err, ErrUnixFSPathOutsideRoot) { + if err = tw.WriteFile(tf, ""); !errors.Is(err, ErrUnixFSPathOutsideRoot) { t.Errorf("unexpected error, wanted: %v; got: %v", ErrUnixFSPathOutsideRoot, err) } } @@ -158,7 +158,7 @@ func TestTarWriterFailsFileOutsideRootWithBaseDir(t *testing.T) { } defer tw.Close() - if err = tw.WriteNode(tf, "test.tar"); !errors.Is(err, ErrUnixFSPathOutsideRoot) { + if err = tw.WriteFile(tf, "test.tar"); !errors.Is(err, ErrUnixFSPathOutsideRoot) { t.Errorf("unexpected error, wanted: %v; got: %v", ErrUnixFSPathOutsideRoot, err) } } diff --git a/gateway/handler_tar.go b/gateway/handler_tar.go index c2f75c168..0b2634804 100644 --- a/gateway/handler_tar.go +++ b/gateway/handler_tar.go @@ -61,7 +61,7 @@ func (i *handler) serveTAR(ctx context.Context, w http.ResponseWriter, r *http.R w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^) // The TAR has a top-level directory (or file) named by the CID. - if err := tarw.WriteNode(file, rootCid.String()); err != nil { + if err := tarw.WriteFile(file, rootCid.String()); err != nil { // Update fail metric i.tarStreamFailMetric.WithLabelValues(rq.contentPath.Namespace()).Observe(time.Since(rq.begin).Seconds()) From 4120ad45df56dbe8d52552a702d97bdaa9782b28 Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Tue, 6 Aug 2024 19:04:23 -0700 Subject: [PATCH 05/18] Add comment --- tar/extractor.go | 34 ++++++++++++++++++++-------------- tar/extractor_test.go | 5 +++-- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/tar/extractor.go b/tar/extractor.go index 9ff62a931..89f6fa919 100644 --- a/tar/extractor.go +++ b/tar/extractor.go @@ -6,7 +6,7 @@ import ( "fmt" "io" "os" - fp "path/filepath" + "path/filepath" "runtime" "strings" "time" @@ -83,7 +83,7 @@ func (te *Extractor) Extract(reader io.Reader) error { rootName := header.Name // Get the platform-specific output path - rootOutputPath := fp.Clean(te.Path) + rootOutputPath := filepath.Clean(te.Path) if err := validatePlatformPath(rootOutputPath); err != nil { return err } @@ -131,7 +131,7 @@ func (te *Extractor) Extract(reader io.Reader) error { } // If the output path directory exists then put the file/symlink into the directory. - outputPath = fp.Join(rootOutputPath, rootName) + outputPath = filepath.Join(rootOutputPath, rootName) } // If an object with the target name already exists overwrite it @@ -142,7 +142,7 @@ func (te *Extractor) Extract(reader io.Reader) error { if err := updateMeta(outputPath, header.Mode, header.ModTime); err != nil { return err } - } else if err := te.extractSymlink(outputPath, header); err != nil { + } else if err := te.extractSymlink(outputPath, rootOutputPath, header); err != nil { return err } default: @@ -183,11 +183,11 @@ func (te *Extractor) Extract(reader io.Reader) error { // This check should already be covered by previous validation, but may catch bugs that slip through. // Checks if the relative path matches or exceeds the root // We check for matching because the outputPath function strips the original root - rel, err := fp.Rel(rootOutputPath, outputPath) + rel, err := filepath.Rel(rootOutputPath, outputPath) if err != nil || rel == "." { return errInvalidRootMultipleRoots } - for _, e := range strings.Split(fp.ToSlash(rel), "/") { + for _, e := range strings.Split(filepath.ToSlash(rel), "/") { if e == ".." { return errors.New("relative path contains '..'") } @@ -209,7 +209,7 @@ func (te *Extractor) Extract(reader io.Reader) error { return err } case tar.TypeSymlink: - if err := te.extractSymlink(outputPath, header); err != nil { + if err := te.extractSymlink(outputPath, rootOutputPath, header); err != nil { return err } default: @@ -257,7 +257,7 @@ func (te *Extractor) outputPath(basePlatformPath, relativeTarPath string) (strin if err := validatePathComponent(e); err != nil { return "", err } - platformPath = fp.Join(platformPath, e) + platformPath = filepath.Join(platformPath, e) // Last element is not checked since it will be removed (if it exists) by any of the extraction functions. // For more details see: @@ -290,20 +290,25 @@ func (te *Extractor) extractDir(path string) error { return err } - if stat, err := os.Lstat(path); err != nil { + stat, err := os.Lstat(path) + if err != nil { return err - } else if !stat.IsDir() { + } + if !stat.IsDir() { return errExtractedDirToSymlink } return nil } -func (te *Extractor) extractSymlink(path string, h *tar.Header) error { +func (te *Extractor) extractSymlink(path, rootPath string, h *tar.Header) error { err := os.Remove(path) if err != nil && !errors.Is(err, os.ErrNotExist) { return err } + // Before extracting a file or other symlink, the old path is removed to + // prevent a simlink being created that causes a subsequent extraction to + // escape the root. err = os.Symlink(h.Linkname, path) if err != nil { return err @@ -324,9 +329,10 @@ func (te *Extractor) extractFile(path string, r *tar.Reader) error { return err } - // Create a temporary file in the target directory and then rename the temporary file to the target to better deal - // with races on the file system. - base := fp.Dir(path) + // Create a temporary file in the target directory and then rename the + // temporary file to the target to better deal with races on the file + // system. + base := filepath.Dir(path) tmpfile, err := os.CreateTemp(base, "") if err != nil { return err diff --git a/tar/extractor_test.go b/tar/extractor_test.go index c8aa5833e..7e31fbea5 100644 --- a/tar/extractor_test.go +++ b/tar/extractor_test.go @@ -496,8 +496,9 @@ func TestLastElementOverwrite(t *testing.T) { &fileTarEntry{path: "root/symlink", buf: []byte("overwrite content")}, }, func(t *testing.T, extractDir string) { - // Check that outside-ref still exists but has not been - // overwritten or truncated (still size the same). + // Check that outside-ref still exists but has not been overwritten + // or truncated (still size the same). The symlink itself have been + // overwritten by the extracted file. info, err := os.Stat(fp.Join(extractDir, "..", "outside-ref")) assert.NoError(t, err) From 23407d55b4fcd4f35e43b31bfc964ddd1bc0d67a Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Tue, 6 Aug 2024 19:50:53 -0700 Subject: [PATCH 06/18] Fix error message --- files/tarwriter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/tarwriter.go b/files/tarwriter.go index a7d2b658f..c982abf63 100644 --- a/files/tarwriter.go +++ b/files/tarwriter.go @@ -10,7 +10,7 @@ import ( "time" ) -var ErrUnixFSPathOutsideRoot = errors.New("relative UnixFS paths outside the root are now allowed, use CAR instead") +var ErrUnixFSPathOutsideRoot = errors.New("relative UnixFS paths outside the root are not allowed, use CAR instead") type TarWriter struct { TarW *tar.Writer From 66d0d8856fef08080e69bbf2de7bbb58a0f6309f Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Thu, 8 Aug 2024 12:27:02 -0700 Subject: [PATCH 07/18] format comments --- tar/extractor.go | 115 ++++++++++++++++++++++++++++------------------- 1 file changed, 68 insertions(+), 47 deletions(-) diff --git a/tar/extractor.go b/tar/extractor.go index 89f6fa919..fd526472f 100644 --- a/tar/extractor.go +++ b/tar/extractor.go @@ -20,26 +20,31 @@ var ( // Extractor is used for extracting tar files to a filesystem. // -// The Extractor can only extract tar files containing files, directories and symlinks. Additionally, the tar files must -// either have a single file, or symlink in them, or must have all of its objects inside of a single root directory -// object. +// The Extractor can only extract tar files containing files, directories and +// symlinks. Additionally, the tar files must either have a single file, or +// symlink in them, or must have all of its objects inside of a single root +// directory object. // -// If the tar file contains a single file/symlink then it will try and extract it with semantics similar to Linux's -// `cp`. In particular, the name of the extracted file/symlink will match the extraction path. If the extraction path -// is a directory then it will extract into the directory using its original name. +// If the tar file contains a single file/symlink then it will try and extract +// it with semantics similar to Linux's `cp`. In particular, the name of the +// extracted file/symlink will match the extraction path. If the extraction +// path is a directory then it will extract into the directory using its +// original name. // -// If an associated mode and last modification time was stored in the archive it is restored. +// If an associated mode and last modification time was stored in the archive +// it is restored. // -// Overwriting: Extraction of files and symlinks will result in overwriting the existing objects with the same name -// when possible (i.e. other files, symlinks, and empty directories). +// Overwriting: Extraction of files and symlinks will result in overwriting the +// existing objects with the same name when possible (i.e. other files, +// symlinks, and empty directories). type Extractor struct { Path string Progress func(int64) int64 deferredUpdates []deferredUpdate } -// Extract extracts a tar file to the file system. See the Extractor for more information on the limitations on the -// tar files that can be extracted. +// Extract extracts a tar file to the file system. See the Extractor for more +// information on the limitations on the tar files that can be extracted. func (te *Extractor) Extract(reader io.Reader) error { if isNullDevice(te.Path) { return nil @@ -69,10 +74,11 @@ func (te *Extractor) Extract(reader io.Reader) error { } defer func() { err = doUpdates() }() - // Specially handle the first entry assuming it is a single root object (e.g. root directory, single file, - // or single symlink) + // Specially handle the first entry assuming it is a single root object + // (e.g. root directory, single file, or single symlink). - // track what the root tar path is so we can ensure that all other entries are below the root + // track what the root tar path is so we can ensure that all other entries + // are below the root. if strings.Contains(header.Name, "/") { return fmt.Errorf("root name contains multiple components : %q : %w", header.Name, errInvalidRoot) } @@ -82,7 +88,7 @@ func (te *Extractor) Extract(reader io.Reader) error { } rootName := header.Name - // Get the platform-specific output path + // Get the platform-specific output path. rootOutputPath := filepath.Clean(te.Path) if err := validatePlatformPath(rootOutputPath); err != nil { return err @@ -90,16 +96,20 @@ func (te *Extractor) Extract(reader io.Reader) error { var firstObjectWasDir bool - // If the last element in the rootOutputPath (which is passed by the user) is a symlink do not follow it - // this makes it easier for users to reason about where files are getting extracted to even when the tar is not - // from a trusted source + // If the last element in the rootOutputPath (which is passed by the user) + // is a symlink do not follow it this makes it easier for users to reason + // about where files are getting extracted to even when the tar is not from + // a trusted source // - // For example, if the user extracts a mutable link to a tar file (http://sometimesbad.tld/t.tar) and situationally - // it contains a folder, file, or symlink the outputs could hop around the user's file system. This is especially - // annoying since we allow symlinks to point anywhere a user might want them to. + // For example, if the user extracts a mutable link to a tar file + // (http://sometimesbad.tld/t.tar) and situationally it contains a folder, + // file, or symlink the outputs could hop around the user's file system. + // This is especially annoying since we allow symlinks to point anywhere a + // user might want them to. switch header.Typeflag { case tar.TypeDir: - // if this is the root directory, use it as the output path for remaining files + // if this is the root directory, use it as the output path for + // remaining files. firstObjectWasDir = true if err := te.extractDir(rootOutputPath); err != nil { return err @@ -108,9 +118,9 @@ func (te *Extractor) Extract(reader io.Reader) error { return err } case tar.TypeReg, tar.TypeSymlink: - // Check if the output path already exists, so we know whether we should - // create our output with that name, or if we should put the output inside - // a preexisting directory + // Check if the output path already exists, so we know whether we + // should create our output with that name, or if we should put the + // output inside a preexisting directory. rootIsExistingDirectory := false // We do not follow links here @@ -123,18 +133,20 @@ func (te *Extractor) Extract(reader io.Reader) error { } outputPath := rootOutputPath - // If the root is a directory which already exists then put the file/symlink in the directory + // If the root is a directory which already exists then put the + // file/symlink in the directory. if rootIsExistingDirectory { - // make sure the root has a valid name + // make sure the root has a valid name. if err := validatePathComponent(rootName); err != nil { return err } - // If the output path directory exists then put the file/symlink into the directory. + // If the output path directory exists then put the file/symlink + // into the directory. outputPath = filepath.Join(rootOutputPath, rootName) } - // If an object with the target name already exists overwrite it + // If an object with the target name already exists overwrite it. if header.Typeflag == tar.TypeReg { if err := te.extractFile(outputPath, tarReader); err != nil { return err @@ -149,7 +161,7 @@ func (te *Extractor) Extract(reader io.Reader) error { return fmt.Errorf("unrecognized tar header type: %d", header.Typeflag) } - // files come recursively in order + // files come recursively in order. for { header, err := tarReader.Next() if err != nil && err != io.EOF { @@ -159,12 +171,13 @@ func (te *Extractor) Extract(reader io.Reader) error { break } - // Make sure that we only have a single root element + // Make sure that we only have a single root element. if !firstObjectWasDir { return fmt.Errorf("the root was not a directory and the tar has multiple entries: %w", errInvalidRoot) } - // validate the path to remove paths we refuse to work with and make it easier to reason about + // validate the path in order to remove paths we refuse to work with + // and make it easier to reason about. if err := validateTarPath(header.Name); err != nil { return err } @@ -180,9 +193,10 @@ func (te *Extractor) Extract(reader io.Reader) error { return err } - // This check should already be covered by previous validation, but may catch bugs that slip through. - // Checks if the relative path matches or exceeds the root - // We check for matching because the outputPath function strips the original root + // This check should already be covered by previous validation, but may + // catch bugs that slip through. Checks if the relative path matches or + // exceeds the root We check for matching because the outputPath + // function strips the original root rel, err := filepath.Rel(rootOutputPath, outputPath) if err != nil || rel == "." { return errInvalidRootMultipleRoots @@ -219,7 +233,7 @@ func (te *Extractor) Extract(reader io.Reader) error { return nil } -// validateTarPath returns an error if the path has problematic characters +// validateTarPath returns an error if the path has problematic characters. func validateTarPath(tarPath string) error { if len(tarPath) == 0 { return errors.New("path is empty") @@ -239,8 +253,9 @@ func validateTarPath(tarPath string) error { return nil } -// getRelativePath returns the relative path between rootTarPath and tarPath. Assumes both paths have been cleaned. -// Will error if the tarPath is not below the rootTarPath. +// getRelativePath returns the relative path between rootTarPath and tarPath. +// Assumes both paths have been cleaned. Will error if the tarPath is not below +// the rootTarPath. func getRelativePath(rootName, tarPath string) (string, error) { if !strings.HasPrefix(tarPath, rootName+"/") { return "", errInvalidRootMultipleRoots @@ -248,7 +263,8 @@ func getRelativePath(rootName, tarPath string) (string, error) { return tarPath[len(rootName)+1:], nil } -// outputPath returns the directory path at which to place the file relativeTarPath. Assumes relativeTarPath is cleaned. +// outputPath returns the directory path at which to place the file +// relativeTarPath. Assumes relativeTarPath is cleaned. func (te *Extractor) outputPath(basePlatformPath, relativeTarPath string) (string, error) { elems := strings.Split(relativeTarPath, "/") @@ -259,8 +275,8 @@ func (te *Extractor) outputPath(basePlatformPath, relativeTarPath string) (strin } platformPath = filepath.Join(platformPath, e) - // Last element is not checked since it will be removed (if it exists) by any of the extraction functions. - // For more details see: + // Last element is not checked since it will be removed (if it exists) + // by any of the extraction functions. For more details see: // https://github.com/libarchive/libarchive/blob/0fd2ed25d78e9f4505de5dcb6208c6c0ff8d2edb/libarchive/archive_write_disk_posix.c#L2810 if i == len(elems)-1 { break @@ -309,6 +325,10 @@ func (te *Extractor) extractSymlink(path, rootPath string, h *tar.Header) error // Before extracting a file or other symlink, the old path is removed to // prevent a simlink being created that causes a subsequent extraction to // escape the root. + // + // Each element of the path of the symlink being extracted is evaluated to + // ensure that there is not a symlink at any point in the path. This is + // done in outputPath. err = os.Symlink(h.Linkname, path) if err != nil { return err @@ -323,7 +343,8 @@ func (te *Extractor) extractSymlink(path, rootPath string, h *tar.Header) error } func (te *Extractor) extractFile(path string, r *tar.Reader) error { - // Attempt removing the target so we can overwrite files, symlinks and empty directories + // Attempt removing the target so we can overwrite files, symlinks and + // empty directories. err := os.Remove(path) if err != nil && !errors.Is(err, os.ErrNotExist) { return err @@ -399,7 +420,7 @@ func (te *Extractor) deferUpdate(path string, header *tar.Header) error { n := len(te.deferredUpdates) if n > 0 && len(path) < len(te.deferredUpdates[n-1].path) { - // if possible, apply the previous deferral + // if possible, apply the previous deferral. m := te.deferredUpdates[n-1] if strings.HasPrefix(m.path, prefix()) { err := updateMeta(m.path, m.mode, m.mtime) @@ -426,7 +447,7 @@ func updateMeta(path string, mode int64, mtime time.Time) error { return updateFileMode(path, mode) } -// updateFileMode sets the unix mode of the filesystem object referenced by path +// updateFileMode sets the unix mode of the filesystem object referenced by path. func updateFileMode(path string, mode int64) error { if err := updateMode(path, mode); err != nil { return fmt.Errorf("[%v] failed to update file mode on '%s'", err, path) @@ -434,9 +455,9 @@ func updateFileMode(path string, mode int64) error { return nil } -// updateModTime sets the last access and modification time of the target filesystem -// object to the given time. -// When the given path references a symlink, if supported the symlink is updated. +// updateModTime sets the last access and modification time of the target +// filesystem object to the given time. When the given path references a +// symlink, if supported, the symlink is updated. func updateModTime(path string, mtime time.Time) error { if err := updateMtime(path, mtime); err != nil { return fmt.Errorf("[%v] failed to update last modification time on '%s'", err, path) From 03387509bf6cfe175ba562d386e7d4ba995962b5 Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Mon, 12 Aug 2024 14:06:42 -0700 Subject: [PATCH 08/18] Do not make backend car file Mode and ModTime panic --- gateway/backend_car_files.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gateway/backend_car_files.go b/gateway/backend_car_files.go index 64d10bcd6..50c298a38 100644 --- a/gateway/backend_car_files.go +++ b/gateway/backend_car_files.go @@ -137,11 +137,11 @@ func (b *singleUseDirectory) Close() error { } func (b *singleUseDirectory) Mode() os.FileMode { - panic("not implemented") + return 0 } func (b *singleUseDirectory) ModTime() time.Time { - panic("not implemented") + return time.Time{} } func (b *singleUseDirectory) Size() (int64, error) { From 10620628e663b19ea87ed4d5daf3f24f133a2d1b Mon Sep 17 00:00:00 2001 From: Andrew Gillis <11790789+gammazero@users.noreply.github.com> Date: Tue, 13 Aug 2024 10:10:03 -0700 Subject: [PATCH 09/18] feat: Support UnixFS mode and modification times in ipld dag and mfs (#658) * feat: Support UnixFS mode and modification times in ipld dag and mfs Adds support for storing and retrieving file mode and last modification time. Support added to: - Files - LinkFiles - Webfiles - Directories Tar archives are supported by the parent branch. --- ipld/unixfs/file/unixfile.go | 47 ++-- .../unixfs/importer/balanced/balanced_test.go | 48 ++++ ipld/unixfs/importer/balanced/builder.go | 29 +- ipld/unixfs/importer/helpers/dagbuilder.go | 97 +++++-- ipld/unixfs/importer/trickle/trickle_test.go | 116 +++++++- ipld/unixfs/importer/trickle/trickledag.go | 22 +- ipld/unixfs/io/dagreader.go | 25 +- ipld/unixfs/mod/dagmodifier.go | 17 +- ipld/unixfs/pb/unixfs.pb.go | 112 ++++++-- ipld/unixfs/pb/unixfs.proto | 17 ++ ipld/unixfs/unixfs.go | 141 +++++++++- ipld/unixfs/unixfs_test.go | 250 ++++++++++++++++++ mfs/dir.go | 77 +++++- mfs/file.go | 90 +++++++ mfs/mfs_test.go | 172 +++++++++++- mfs/ops.go | 23 +- mfs/root.go | 3 + 17 files changed, 1192 insertions(+), 94 deletions(-) diff --git a/ipld/unixfs/file/unixfile.go b/ipld/unixfs/file/unixfile.go index 0a923e937..0cf7616c1 100644 --- a/ipld/unixfs/file/unixfile.go +++ b/ipld/unixfs/file/unixfile.go @@ -23,7 +23,8 @@ type ufsDirectory struct { dserv ipld.DAGService dir uio.Directory size int64 - stat os.FileInfo + mode os.FileMode + mtime time.Time } type ufsIterator struct { @@ -122,17 +123,11 @@ func (d *ufsDirectory) Entries() files.DirIterator { } func (d *ufsDirectory) Mode() os.FileMode { - if d.stat == nil { - return 0 - } - return d.stat.Mode() + return d.mode } func (d *ufsDirectory) ModTime() time.Time { - if d.stat == nil { - return time.Time{} - } - return d.stat.ModTime() + return d.mtime } func (d *ufsDirectory) Size() (int64, error) { @@ -141,28 +136,21 @@ func (d *ufsDirectory) Size() (int64, error) { type ufsFile struct { uio.DagReader - stat os.FileInfo } func (f *ufsFile) Mode() os.FileMode { - if f.stat == nil { - return 0 - } - return f.stat.Mode() + return f.DagReader.Mode() } func (f *ufsFile) ModTime() time.Time { - if f.stat == nil { - return time.Time{} - } - return f.stat.ModTime() + return f.DagReader.ModTime() } func (f *ufsFile) Size() (int64, error) { return int64(f.DagReader.Size()), nil } -func newUnixfsDir(ctx context.Context, dserv ipld.DAGService, nd *dag.ProtoNode, stat os.FileInfo) (files.Directory, error) { +func newUnixfsDir(ctx context.Context, dserv ipld.DAGService, nd *dag.ProtoNode) (files.Directory, error) { dir, err := uio.NewDirectoryFromNode(dserv, nd) if err != nil { return nil, err @@ -173,32 +161,35 @@ func newUnixfsDir(ctx context.Context, dserv ipld.DAGService, nd *dag.ProtoNode, return nil, err } + fsn, err := ft.FSNodeFromBytes(nd.Data()) + if err != nil { + return nil, err + } + return &ufsDirectory{ ctx: ctx, dserv: dserv, - dir: dir, - size: int64(size), - stat: stat, + dir: dir, + size: int64(size), + mode: fsn.Mode(), + mtime: fsn.ModTime(), }, nil } func NewUnixfsFile(ctx context.Context, dserv ipld.DAGService, nd ipld.Node) (files.Node, error) { - return NewUnixfsFileWithStat(ctx, dserv, nd, nil) -} - -func NewUnixfsFileWithStat(ctx context.Context, dserv ipld.DAGService, nd ipld.Node, stat os.FileInfo) (files.Node, error) { switch dn := nd.(type) { case *dag.ProtoNode: fsn, err := ft.FSNodeFromBytes(dn.Data()) if err != nil { return nil, err } + if fsn.IsDir() { - return newUnixfsDir(ctx, dserv, dn, stat) + return newUnixfsDir(ctx, dserv, dn) } if fsn.Type() == ft.TSymlink { - return files.NewLinkFile(string(fsn.Data()), stat), nil + return files.NewSymlinkFile(string(fsn.Data()), fsn.ModTime()), nil } case *dag.RawNode: diff --git a/ipld/unixfs/importer/balanced/balanced_test.go b/ipld/unixfs/importer/balanced/balanced_test.go index 5a5dcf9ad..4ea4cb8a9 100644 --- a/ipld/unixfs/importer/balanced/balanced_test.go +++ b/ipld/unixfs/importer/balanced/balanced_test.go @@ -7,6 +7,7 @@ import ( "io" mrand "math/rand" "testing" + "time" h "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" uio "github.com/ipfs/boxo/ipld/unixfs/io" @@ -26,6 +27,10 @@ func buildTestDag(ds ipld.DAGService, spl chunker.Splitter) (*dag.ProtoNode, err Maxlinks: h.DefaultLinksPerBlock, } + return buildTestDagWithParams(spl, dbp) +} + +func buildTestDagWithParams(spl chunker.Splitter, dbp h.DagBuilderParams) (*dag.ProtoNode, error) { db, err := dbp.New(spl) if err != nil { return nil, err @@ -335,3 +340,46 @@ func TestSeekingConsistency(t *testing.T) { t.Fatal(err) } } + +func TestMetadataNoData(t *testing.T) { + testMetadata(t, new(bytes.Buffer)) +} + +func TestMetadata(t *testing.T) { + nbytes := 3 * chunker.DefaultBlockSize + buf := new(bytes.Buffer) + _, err := io.CopyN(buf, random.NewRand(), nbytes) + if err != nil { + t.Fatal(err) + } + + testMetadata(t, buf) +} + +func testMetadata(t *testing.T, buf *bytes.Buffer) { + dagserv := mdtest.Mock() + dbp := h.DagBuilderParams{ + Dagserv: dagserv, + Maxlinks: h.DefaultLinksPerBlock, + FileMode: 0522, + FileModTime: time.Unix(1638111600, 76552), + } + + nd, err := buildTestDagWithParams(chunker.DefaultSplitter(buf), dbp) + if err != nil { + t.Fatal(err) + } + + dr, err := uio.NewDagReader(context.Background(), nd, dagserv) + if err != nil { + t.Fatal(err) + } + + if !dr.ModTime().Equal(dbp.FileModTime) { + t.Errorf("got modtime %v, wanted %v", dr.ModTime(), dbp.FileModTime) + } + + if dr.Mode() != dbp.FileMode { + t.Errorf("got filemode %o, wanted %o", dr.Mode(), dbp.FileMode) + } +} diff --git a/ipld/unixfs/importer/balanced/builder.go b/ipld/unixfs/importer/balanced/builder.go index 0fdb0fd28..915d0a439 100644 --- a/ipld/unixfs/importer/balanced/builder.go +++ b/ipld/unixfs/importer/balanced/builder.go @@ -130,18 +130,33 @@ import ( // | Chunk 1 | | Chunk 2 | | Chunk 3 | // +=========+ +=========+ + - - - - + func Layout(db *h.DagBuilderHelper) (ipld.Node, error) { + var root ipld.Node + var err error + if db.Done() { - // No data, return just an empty node. - root, err := db.NewLeafNode(nil, ft.TFile) - if err != nil { - return nil, err - } + // No data, just create an empty node. + root, err = db.NewLeafNode(nil, ft.TFile) // This works without Filestore support (`ProcessFileStore`). // TODO: Why? Is there a test case missing? + } else { + root, err = layoutData(db) + } + + if err != nil { + return nil, err + } - return root, db.Add(root) + if db.HasFileAttributes() { + err = db.SetFileAttributes(root) + if err != nil { + return nil, err + } } + return root, db.Add(root) +} + +func layoutData(db *h.DagBuilderHelper) (ipld.Node, error) { // The first `root` will be a single leaf node with data // (corner case), after that subsequent `root` nodes will // always be internal nodes (with a depth > 0) that can @@ -172,7 +187,7 @@ func Layout(db *h.DagBuilderHelper) (ipld.Node, error) { } } - return root, db.Add(root) + return root, nil } // fillNodeRec will "fill" the given internal (non-leaf) `node` with data by diff --git a/ipld/unixfs/importer/helpers/dagbuilder.go b/ipld/unixfs/importer/helpers/dagbuilder.go index 25514d795..aefffad15 100644 --- a/ipld/unixfs/importer/helpers/dagbuilder.go +++ b/ipld/unixfs/importer/helpers/dagbuilder.go @@ -5,6 +5,7 @@ import ( "errors" "io" "os" + "time" dag "github.com/ipfs/boxo/ipld/merkledag" @@ -23,13 +24,15 @@ var ErrMissingFsRef = errors.New("missing file path or URL, can't create filesto // DagBuilderHelper wraps together a bunch of objects needed to // efficiently create unixfs dag trees type DagBuilderHelper struct { - dserv ipld.DAGService - spl chunker.Splitter - recvdErr error - rawLeaves bool - nextData []byte // the next item to return. - maxlinks int - cidBuilder cid.Builder + dserv ipld.DAGService + spl chunker.Splitter + recvdErr error + rawLeaves bool + nextData []byte // the next item to return. + maxlinks int + cidBuilder cid.Builder + fileMode os.FileMode + fileModTime time.Time // Filestore support variables. // ---------------------------- @@ -62,6 +65,12 @@ type DagBuilderParams struct { // DAGService to write blocks to (required) Dagserv ipld.DAGService + // The unixfs file mode + FileMode os.FileMode + + // The unixfs last modified time + FileModTime time.Time + // NoCopy signals to the chunker that it should track fileinfo for // filestore adds NoCopy bool @@ -71,11 +80,13 @@ type DagBuilderParams struct { // chunker.Splitter as data source. func (dbp *DagBuilderParams) New(spl chunker.Splitter) (*DagBuilderHelper, error) { db := &DagBuilderHelper{ - dserv: dbp.Dagserv, - spl: spl, - rawLeaves: dbp.RawLeaves, - cidBuilder: dbp.CidBuilder, - maxlinks: dbp.Maxlinks, + dserv: dbp.Dagserv, + spl: spl, + rawLeaves: dbp.RawLeaves, + cidBuilder: dbp.CidBuilder, + maxlinks: dbp.Maxlinks, + fileMode: dbp.FileMode, + fileModTime: dbp.FileModTime, } if fi, ok := spl.Reader().(files.FileInfo); dbp.NoCopy && ok { db.fullPath = fi.AbsPath() @@ -138,9 +149,9 @@ func (db *DagBuilderHelper) GetCidBuilder() cid.Builder { return db.cidBuilder } -// NewLeafNode creates a leaf node filled with data. If rawLeaves is -// defined then a raw leaf will be returned. Otherwise, it will create -// and return `FSNodeOverDag` with `fsNodeType`. +// NewLeafNode creates a leaf node filled with data. If rawLeaves is defined +// then a raw leaf will be returned. Otherwise, it will create and return +// `FSNodeOverDag` with `fsNodeType`. func (db *DagBuilderHelper) NewLeafNode(data []byte, fsNodeType pb.Data_DataType) (ipld.Node, error) { if len(data) > BlockSizeLimit { return nil, ErrSizeLimitExceeded @@ -161,6 +172,7 @@ func (db *DagBuilderHelper) NewLeafNode(data []byte, fsNodeType pb.Data_DataType // Encapsulate the data in UnixFS node (instead of a raw node). fsNodeOverDag := db.NewFSNodeOverDag(fsNodeType) fsNodeOverDag.SetFileData(data) + node, err := fsNodeOverDag.Commit() if err != nil { return nil, err @@ -172,9 +184,10 @@ func (db *DagBuilderHelper) NewLeafNode(data []byte, fsNodeType pb.Data_DataType return node, nil } -// FillNodeLayer will add datanodes as children to the give node until +// FillNodeLayer will add data-nodes as children to the given node until // it is full in this layer or no more data. -// NOTE: This function creates raw data nodes so it only works +// +// NOTE: This function creates raw data nodes, so it only works // for the `trickle.Layout`. func (db *DagBuilderHelper) FillNodeLayer(node *FSNodeOverDag) error { // while we have room AND we're not done @@ -265,6 +278,34 @@ func (db *DagBuilderHelper) Maxlinks() int { return db.maxlinks } +// HasFileAttributes will return false if Filestore is being used, +// otherwise returns true if a file mode or last modification time is set. +func (db *DagBuilderHelper) HasFileAttributes() bool { + return db.fullPath == "" && (db.fileMode != 0 || !db.fileModTime.IsZero()) +} + +// SetFileAttributes stores file attributes present in the `DagBuilderHelper` +// into the associated `ft.FSNode`. +func (db *DagBuilderHelper) SetFileAttributes(n ipld.Node) error { + if pn, ok := n.(*dag.ProtoNode); ok { + fsn, err := ft.FSNodeFromBytes(pn.Data()) + if err != nil { + return err + } + fsn.SetModTime(db.fileModTime) + fsn.SetMode(db.fileMode) + + d, err := fsn.GetBytes() + if err != nil { + return err + } + + pn.SetData(d) + } + + return nil +} + // FSNodeOverDag encapsulates an `unixfs.FSNode` that will be stored in a // `dag.ProtoNode`. Instead of just having a single `ipld.Node` that // would need to be constantly (un)packed to access and modify its @@ -288,7 +329,7 @@ type FSNodeOverDag struct { } // NewFSNodeOverDag creates a new `dag.ProtoNode` and `ft.FSNode` -// decoupled from one onther (and will continue in that way until +// decoupled from one anonther (and will continue in that way until // `Commit` is called), with `fsNodeType` specifying the type of // the UnixFS layer node (either `File` or `Raw`). func (db *DagBuilderHelper) NewFSNodeOverDag(fsNodeType pb.Data_DataType) *FSNodeOverDag { @@ -374,6 +415,26 @@ func (n *FSNodeOverDag) SetFileData(fileData []byte) { n.file.SetData(fileData) } +// SetMode sets the file mode of the associated `ft.FSNode`. +func (n *FSNodeOverDag) SetMode(mode os.FileMode) { + n.file.SetMode(mode) +} + +// SetModTime sets the file modification time of the associated `ft.FSNode`. +func (n *FSNodeOverDag) SetModTime(ts time.Time) { + n.file.SetModTime(ts) +} + +// Mode returns the file mode of the associated `ft.FSNode` +func (n *FSNodeOverDag) Mode() os.FileMode { + return n.file.Mode() +} + +// ModTime returns the last modification time of the associated `ft.FSNode` +func (n *FSNodeOverDag) ModTime() time.Time { + return n.file.ModTime() +} + // GetDagNode fills out the proper formatting for the FSNodeOverDag node // inside of a DAG node and returns the dag node. // TODO: Check if we have committed (passed the UnixFS information diff --git a/ipld/unixfs/importer/trickle/trickle_test.go b/ipld/unixfs/importer/trickle/trickle_test.go index 9078fdc02..d495fd208 100644 --- a/ipld/unixfs/importer/trickle/trickle_test.go +++ b/ipld/unixfs/importer/trickle/trickle_test.go @@ -6,7 +6,9 @@ import ( "fmt" "io" mrand "math/rand" + "runtime" "testing" + "time" ft "github.com/ipfs/boxo/ipld/unixfs" h "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" @@ -40,6 +42,10 @@ func buildTestDag(ds ipld.DAGService, spl chunker.Splitter, rawLeaves UseRawLeav RawLeaves: bool(rawLeaves), } + return buildTestDagWithParams(ds, spl, dbp) +} + +func buildTestDagWithParams(ds ipld.DAGService, spl chunker.Splitter, dbp h.DagBuilderParams) (*merkledag.ProtoNode, error) { db, err := dbp.New(spl) if err != nil { return nil, err @@ -59,7 +65,7 @@ func buildTestDag(ds ipld.DAGService, spl chunker.Splitter, rawLeaves UseRawLeav Getter: ds, Direct: dbp.Maxlinks, LayerRepeat: depthRepeat, - RawLeaves: bool(rawLeaves), + RawLeaves: dbp.RawLeaves, }) } @@ -668,3 +674,111 @@ func TestAppendSingleBytesToEmpty(t *testing.T) { t.Fatal(err) } } + +func TestAppendWithModTime(t *testing.T) { + const nbytes = 128 * 1024 + + timestamp := time.Now() + buf := random.Bytes(nbytes) + + nd := new(merkledag.ProtoNode) + nd.SetData(ft.FilePBDataWithStat(buf[:nbytes/2], nbytes/2, 0, timestamp)) + + if runtime.GOOS == "windows" { + time.Sleep(3 * time.Second) // for os with low-res mod time. + } + + dbp := &h.DagBuilderParams{ + Dagserv: mdtest.Mock(), + Maxlinks: h.DefaultLinksPerBlock, + } + + r := bytes.NewReader(buf[nbytes/2:]) + db, err := dbp.New(chunker.NewSizeSplitter(r, 500)) + if err != nil { + t.Fatal(err) + } + + nd2, err := Append(context.Background(), nd, db) + if err != nil { + t.Fatal(err) + } + + fsn, _ := ft.ExtractFSNode(nd2) + + if !fsn.ModTime().After(timestamp) { + t.Errorf("expected modification time to be updated") + } + +} + +func TestAppendToEmptyWithModTime(t *testing.T) { + timestamp := time.Now() + nd := new(merkledag.ProtoNode) + nd.SetData(ft.FilePBDataWithStat(nil, 0, 0, timestamp)) + + if runtime.GOOS == "windows" { + time.Sleep(3 * time.Second) // for os with low-res mod time. + } + + dbp := &h.DagBuilderParams{ + Dagserv: mdtest.Mock(), + Maxlinks: h.DefaultLinksPerBlock, + } + + db, err := dbp.New(chunker.DefaultSplitter(bytes.NewReader([]byte("test")))) + if err != nil { + t.Fatal(err) + } + + nd2, err := Append(context.Background(), nd, db) + if err != nil { + t.Fatal(err) + } + + fsn, _ := ft.ExtractFSNode(nd2) + + if !fsn.ModTime().After(timestamp) { + t.Errorf("expected modification time to be updated") + } +} + +func TestMetadata(t *testing.T) { + runBothSubtests(t, testMetadata) +} + +func testMetadata(t *testing.T, rawLeaves UseRawLeaves) { + const nbytes = 3 * chunker.DefaultBlockSize + buf := new(bytes.Buffer) + _, err := io.CopyN(buf, random.NewRand(), nbytes) + if err != nil { + t.Fatal(err) + } + + dagserv := mdtest.Mock() + dbp := h.DagBuilderParams{ + Dagserv: dagserv, + Maxlinks: h.DefaultLinksPerBlock, + RawLeaves: bool(rawLeaves), + FileMode: 0522, + FileModTime: time.Unix(1638111600, 76552), + } + + nd, err := buildTestDagWithParams(dagserv, chunker.DefaultSplitter(buf), dbp) + if err != nil { + t.Fatal(err) + } + + dr, err := uio.NewDagReader(context.Background(), nd, dagserv) + if err != nil { + t.Fatal(err) + } + + if !dr.ModTime().Equal(dbp.FileModTime) { + t.Errorf("got modtime %v, wanted %v", dr.ModTime(), dbp.FileModTime) + } + + if dr.Mode() != dbp.FileMode { + t.Errorf("got filemode %o, wanted %o", dr.Mode(), dbp.FileMode) + } +} diff --git a/ipld/unixfs/importer/trickle/trickledag.go b/ipld/unixfs/importer/trickle/trickledag.go index 09a8b8672..2b9d31dfa 100644 --- a/ipld/unixfs/importer/trickle/trickledag.go +++ b/ipld/unixfs/importer/trickle/trickledag.go @@ -19,6 +19,7 @@ import ( "context" "errors" "fmt" + "time" ft "github.com/ipfs/boxo/ipld/unixfs" h "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" @@ -43,6 +44,13 @@ func Layout(db *h.DagBuilderHelper) (ipld.Node, error) { return nil, err } + if db.HasFileAttributes() { + err = db.SetFileAttributes(root) + } + + if err != nil { + return nil, err + } return root, db.Add(root) } @@ -94,7 +102,6 @@ func Append(ctx context.Context, basen ipld.Node, db *h.DagBuilderHelper) (out i } // Convert to unixfs node for working with easily - fsn, err := h.NewFSNFromDag(base) if err != nil { return nil, err @@ -109,9 +116,10 @@ func Append(ctx context.Context, basen ipld.Node, db *h.DagBuilderHelper) (out i } if db.Done() { - // TODO: If `FillNodeLayer` stop `Commit`ing this should be - // the place (besides the function end) to call it. - return fsn.GetDagNode() + if !fsn.ModTime().IsZero() { + fsn.SetModTime(time.Now()) + } + return fsn.Commit() } // If continuing, our depth has increased by one @@ -142,11 +150,7 @@ func Append(ctx context.Context, basen ipld.Node, db *h.DagBuilderHelper) (out i } } } - _, err = fsn.Commit() - if err != nil { - return nil, err - } - return fsn.GetDagNode() + return fsn.Commit() } func appendFillLastChild(ctx context.Context, fsn *h.FSNodeOverDag, depth int, repeatNumber int, db *h.DagBuilderHelper) error { diff --git a/ipld/unixfs/io/dagreader.go b/ipld/unixfs/io/dagreader.go index 77dc8d921..bb1c83800 100644 --- a/ipld/unixfs/io/dagreader.go +++ b/ipld/unixfs/io/dagreader.go @@ -5,6 +5,8 @@ import ( "context" "errors" "io" + "os" + "time" mdag "github.com/ipfs/boxo/ipld/merkledag" unixfs "github.com/ipfs/boxo/ipld/unixfs" @@ -29,6 +31,8 @@ var ( type DagReader interface { ReadSeekCloser Size() uint64 + Mode() os.FileMode + ModTime() time.Time CtxReadFull(context.Context, []byte) (int, error) } @@ -44,6 +48,8 @@ type ReadSeekCloser interface { // the given node, using the passed in DAGService for data retrieval. func NewDagReader(ctx context.Context, n ipld.Node, serv ipld.NodeGetter) (DagReader, error) { var size uint64 + var mode os.FileMode + var modTime time.Time switch n := n.(type) { case *mdag.RawNode: @@ -55,6 +61,9 @@ func NewDagReader(ctx context.Context, n ipld.Node, serv ipld.NodeGetter) (DagRe return nil, err } + mode = fsNode.Mode() + modTime = fsNode.ModTime() + switch fsNode.Type() { case unixfs.TFile, unixfs.TRaw: size = fsNode.FileSize() @@ -93,6 +102,8 @@ func NewDagReader(ctx context.Context, n ipld.Node, serv ipld.NodeGetter) (DagRe cancel: cancel, serv: serv, size: size, + mode: mode, + modTime: modTime, rootNode: n, dagWalker: ipld.NewWalker(ctxWithCancel, ipld.NewNavigableIPLDNode(n, serv)), }, nil @@ -129,7 +140,19 @@ type dagReader struct { // Passed to the `dagWalker` that will use it to request nodes. // TODO: Revisit name. - serv ipld.NodeGetter + serv ipld.NodeGetter + mode os.FileMode + modTime time.Time +} + +// Mode returns the UnixFS file mode or 0 if not set. +func (dr *dagReader) Mode() os.FileMode { + return dr.mode +} + +// ModTime returns the UnixFS file last modification time if set. +func (dr *dagReader) ModTime() time.Time { + return dr.modTime } // Size returns the total size of the data from the DAG structured file. diff --git a/ipld/unixfs/mod/dagmodifier.go b/ipld/unixfs/mod/dagmodifier.go index f662a0a71..c075523f8 100644 --- a/ipld/unixfs/mod/dagmodifier.go +++ b/ipld/unixfs/mod/dagmodifier.go @@ -7,6 +7,7 @@ import ( "context" "errors" "io" + "time" ft "github.com/ipfs/boxo/ipld/unixfs" help "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" @@ -258,6 +259,9 @@ func (dm *DagModifier) modifyDag(n ipld.Node, offset uint64) (cid.Cid, error) { } // Update newly written node.. + if !fsn.ModTime().IsZero() { + fsn.SetModTime(time.Now()) + } b, err := fsn.GetBytes() if err != nil { return cid.Cid{}, err @@ -527,8 +531,17 @@ func (dm *DagModifier) dagTruncate(ctx context.Context, n ipld.Node, size uint64 if err != nil { return nil, err } - nd.SetData(ft.WrapData(fsn.Data()[:size])) - return nd, nil + + fsn.SetData(fsn.Data()[:size]) + if !fsn.ModTime().IsZero() { + fsn.SetModTime(time.Now()) + } + data, err := fsn.GetBytes() + if err != nil { + return nil, err + } + + return mdag.NodeWithData(data), nil case *mdag.RawNode: return mdag.NewRawNodeWPrefix(nd.RawData()[:size], nd.Cid().Prefix()) } diff --git a/ipld/unixfs/pb/unixfs.pb.go b/ipld/unixfs/pb/unixfs.pb.go index 805c11289..d02e110f2 100644 --- a/ipld/unixfs/pb/unixfs.pb.go +++ b/ipld/unixfs/pb/unixfs.pb.go @@ -82,6 +82,8 @@ type Data struct { Blocksizes []uint64 `protobuf:"varint,4,rep,name=blocksizes" json:"blocksizes,omitempty"` HashType *uint64 `protobuf:"varint,5,opt,name=hashType" json:"hashType,omitempty"` Fanout *uint64 `protobuf:"varint,6,opt,name=fanout" json:"fanout,omitempty"` + Mode *uint32 `protobuf:"varint,7,opt,name=mode" json:"mode,omitempty"` + Mtime *IPFSTimestamp `protobuf:"bytes,8,opt,name=mtime" json:"mtime,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -158,6 +160,20 @@ func (m *Data) GetFanout() uint64 { return 0 } +func (m *Data) GetMode() uint32 { + if m != nil && m.Mode != nil { + return *m.Mode + } + return 0 +} + +func (m *Data) GetMtime() *IPFSTimestamp { + if m != nil { + return m.Mtime + } + return nil +} + type Metadata struct { MimeType *string `protobuf:"bytes,1,opt,name=MimeType" json:"MimeType,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -201,31 +217,91 @@ func (m *Metadata) GetMimeType() string { return "" } +// mostly copied from proto 3 - with int32 nanos changed to fixed32 for js-ipfs compatibility +// https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/timestamp.proto +type IPFSTimestamp struct { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + Seconds *int64 `protobuf:"varint,1,req,name=seconds" json:"seconds,omitempty"` + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + Nanos *uint32 `protobuf:"fixed32,2,opt,name=nanos" json:"nanos,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *IPFSTimestamp) Reset() { *m = IPFSTimestamp{} } +func (m *IPFSTimestamp) String() string { return proto.CompactTextString(m) } +func (*IPFSTimestamp) ProtoMessage() {} +func (*IPFSTimestamp) Descriptor() ([]byte, []int) { + return fileDescriptor_e2fd76cc44dfc7c3, []int{2} +} +func (m *IPFSTimestamp) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_IPFSTimestamp.Unmarshal(m, b) +} +func (m *IPFSTimestamp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_IPFSTimestamp.Marshal(b, m, deterministic) +} +func (m *IPFSTimestamp) XXX_Merge(src proto.Message) { + xxx_messageInfo_IPFSTimestamp.Merge(m, src) +} +func (m *IPFSTimestamp) XXX_Size() int { + return xxx_messageInfo_IPFSTimestamp.Size(m) +} +func (m *IPFSTimestamp) XXX_DiscardUnknown() { + xxx_messageInfo_IPFSTimestamp.DiscardUnknown(m) +} + +var xxx_messageInfo_IPFSTimestamp proto.InternalMessageInfo + +func (m *IPFSTimestamp) GetSeconds() int64 { + if m != nil && m.Seconds != nil { + return *m.Seconds + } + return 0 +} + +func (m *IPFSTimestamp) GetNanos() uint32 { + if m != nil && m.Nanos != nil { + return *m.Nanos + } + return 0 +} + func init() { proto.RegisterEnum("unixfs.v1.pb.Data_DataType", Data_DataType_name, Data_DataType_value) proto.RegisterType((*Data)(nil), "unixfs.v1.pb.Data") proto.RegisterType((*Metadata)(nil), "unixfs.v1.pb.Metadata") + proto.RegisterType((*IPFSTimestamp)(nil), "unixfs.pb.IPFSTimestamp") } func init() { proto.RegisterFile("unixfs.proto", fileDescriptor_e2fd76cc44dfc7c3) } var fileDescriptor_e2fd76cc44dfc7c3 = []byte{ - // 267 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x4c, 0x90, 0x41, 0x4f, 0x83, 0x30, - 0x18, 0x86, 0x05, 0xba, 0x0d, 0xbe, 0xa1, 0x69, 0xbe, 0x83, 0x21, 0x9a, 0x18, 0xc2, 0xc1, 0x70, - 0xc2, 0xe8, 0x3f, 0xd0, 0x2c, 0xc6, 0x0b, 0x97, 0x6e, 0xf1, 0xe0, 0xc5, 0x94, 0xad, 0x84, 0x66, - 0x8c, 0x12, 0xe8, 0x54, 0xfc, 0x1b, 0xfe, 0x61, 0x53, 0x18, 0xdb, 0x2e, 0x4d, 0x9e, 0xf6, 0x79, - 0x9b, 0x37, 0x2f, 0xf8, 0xfb, 0x4a, 0xfe, 0xe4, 0x6d, 0x52, 0x37, 0x4a, 0x2b, 0x1c, 0xe9, 0xeb, - 0x31, 0xa9, 0xb3, 0xe8, 0xcf, 0x06, 0xb2, 0xe0, 0x9a, 0xe3, 0x03, 0x90, 0x55, 0x57, 0x8b, 0xc0, - 0x0a, 0xed, 0xf8, 0xea, 0xe9, 0x36, 0x39, 0xb7, 0x12, 0x63, 0xf4, 0x87, 0x51, 0x58, 0x2f, 0x22, - 0x0e, 0xc1, 0xc0, 0x0e, 0xad, 0xd8, 0x67, 0xc3, 0x27, 0x37, 0xe0, 0xe6, 0xb2, 0x14, 0xad, 0xfc, - 0x15, 0x81, 0x13, 0x5a, 0x31, 0x61, 0x47, 0xc6, 0x3b, 0x80, 0xac, 0x54, 0xeb, 0xad, 0x81, 0x36, - 0x20, 0xa1, 0x13, 0x13, 0x76, 0x76, 0x63, 0xb2, 0x05, 0x6f, 0x8b, 0xbe, 0xc4, 0x64, 0xc8, 0x8e, - 0x8c, 0xd7, 0x30, 0xcd, 0x79, 0xa5, 0xf6, 0x3a, 0x98, 0xf6, 0x2f, 0x07, 0x8a, 0xde, 0xc1, 0x1d, - 0x5b, 0xe1, 0x0c, 0x1c, 0xc6, 0xbf, 0xe9, 0x05, 0x5e, 0x82, 0xb7, 0x90, 0x8d, 0x58, 0x6b, 0xd5, - 0x74, 0xd4, 0x42, 0x17, 0xc8, 0xab, 0x2c, 0x05, 0xb5, 0xd1, 0x07, 0x37, 0x15, 0x9a, 0x6f, 0xb8, - 0xe6, 0xd4, 0xc1, 0x39, 0xcc, 0x96, 0xdd, 0xae, 0x94, 0xd5, 0x96, 0x12, 0x93, 0x79, 0x7b, 0x4e, - 0x57, 0xcb, 0x82, 0x37, 0x1b, 0x3a, 0x89, 0xee, 0x4f, 0xa6, 0xe9, 0x95, 0xca, 0x9d, 0x38, 0x8c, - 0x63, 0xc5, 0x1e, 0x3b, 0xf2, 0xcb, 0xfc, 0xc3, 0x1b, 0x76, 0xfa, 0xac, 0xb3, 0xff, 0x00, 0x00, - 0x00, 0xff, 0xff, 0xbd, 0x16, 0xf8, 0x45, 0x67, 0x01, 0x00, 0x00, + // 322 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x91, 0x5f, 0x4b, 0xc3, 0x30, + 0x14, 0xc5, 0xed, 0xbf, 0xb5, 0xbb, 0xdb, 0xa4, 0x5c, 0x44, 0x82, 0x0f, 0x52, 0xfa, 0x20, 0x7d, + 0x90, 0x3e, 0xf8, 0x05, 0x44, 0x18, 0x43, 0x1f, 0x06, 0x92, 0x0d, 0xdf, 0xb3, 0x35, 0x63, 0x61, + 0x4d, 0x33, 0x9a, 0x0c, 0x9d, 0x9f, 0xd3, 0x0f, 0x24, 0x49, 0xd7, 0xe9, 0x5e, 0x4a, 0x7f, 0xb9, + 0xe7, 0x84, 0x73, 0x6e, 0x60, 0x7c, 0x68, 0xc4, 0xd7, 0x46, 0x97, 0xfb, 0x56, 0x19, 0x85, 0xc3, + 0x9e, 0x56, 0xf9, 0x8f, 0x0f, 0xe1, 0x94, 0x19, 0x86, 0x8f, 0x10, 0x2e, 0x8f, 0x7b, 0x4e, 0xbc, + 0xcc, 0x2f, 0xae, 0x9f, 0x48, 0x79, 0x96, 0x94, 0x76, 0xec, 0x3e, 0x76, 0x4e, 0x9d, 0x0a, 0xb1, + 0x73, 0x11, 0x3f, 0xf3, 0x8a, 0x31, 0xed, 0x6e, 0xb8, 0x83, 0x64, 0x23, 0x6a, 0xae, 0xc5, 0x37, + 0x27, 0x41, 0xe6, 0x15, 0x21, 0x3d, 0x33, 0xde, 0x03, 0xac, 0x6a, 0xb5, 0xde, 0x59, 0xd0, 0x24, + 0xcc, 0x82, 0x22, 0xa4, 0xff, 0x4e, 0xac, 0x77, 0xcb, 0xf4, 0xd6, 0x25, 0x88, 0x3a, 0x6f, 0xcf, + 0x78, 0x0b, 0x83, 0x0d, 0x6b, 0xd4, 0xc1, 0x90, 0x81, 0x9b, 0x9c, 0xc8, 0x66, 0x90, 0xaa, 0xe2, + 0x24, 0xce, 0xbc, 0x62, 0x42, 0xdd, 0x3f, 0x96, 0x10, 0x49, 0x23, 0x24, 0x27, 0x49, 0xe6, 0x15, + 0xa3, 0x8b, 0x1a, 0x6f, 0xef, 0xb3, 0xc5, 0x52, 0x48, 0xae, 0x0d, 0x93, 0x7b, 0xda, 0xc9, 0xf2, + 0x0f, 0x48, 0xfa, 0x66, 0x18, 0x43, 0x40, 0xd9, 0x67, 0x7a, 0x85, 0x13, 0x18, 0x4e, 0x45, 0xcb, + 0xd7, 0x46, 0xb5, 0xc7, 0xd4, 0xc3, 0x04, 0xc2, 0x99, 0xa8, 0x79, 0xea, 0xe3, 0x18, 0x92, 0x39, + 0x37, 0xac, 0x62, 0x86, 0xa5, 0x01, 0x8e, 0x20, 0x5e, 0x1c, 0x65, 0x2d, 0x9a, 0x5d, 0x1a, 0x5a, + 0xcf, 0xeb, 0xcb, 0x7c, 0xb9, 0xd8, 0xb2, 0xb6, 0x4a, 0xa3, 0xfc, 0xe1, 0x4f, 0x69, 0xbb, 0xcd, + 0x85, 0xe4, 0xa7, 0xed, 0x7a, 0xc5, 0x90, 0x9e, 0x39, 0x7f, 0x86, 0xc9, 0x45, 0x2e, 0x24, 0x10, + 0x6b, 0xbe, 0x56, 0x4d, 0xa5, 0xdd, 0x4b, 0x04, 0xb4, 0x47, 0xbc, 0x81, 0xa8, 0x61, 0x8d, 0xd2, + 0x6e, 0xe7, 0x31, 0xed, 0xe0, 0x37, 0x00, 0x00, 0xff, 0xff, 0x36, 0xaf, 0xfa, 0x7c, 0xd9, 0x01, + 0x00, 0x00, } diff --git a/ipld/unixfs/pb/unixfs.proto b/ipld/unixfs/pb/unixfs.proto index f65673f54..bd02aa410 100644 --- a/ipld/unixfs/pb/unixfs.proto +++ b/ipld/unixfs/pb/unixfs.proto @@ -21,8 +21,25 @@ message Data { optional uint64 hashType = 5; optional uint64 fanout = 6; + optional uint32 mode = 7; + optional IPFSTimestamp mtime = 8; } message Metadata { optional string MimeType = 1; } + +// mostly copied from proto 3 - with int32 nanos changed to fixed32 for js-ipfs compatibility +// https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/timestamp.proto +message IPFSTimestamp { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + required int64 seconds = 1; + + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + optional fixed32 nanos = 2; +} diff --git a/ipld/unixfs/unixfs.go b/ipld/unixfs/unixfs.go index 4131df837..bdc54518f 100644 --- a/ipld/unixfs/unixfs.go +++ b/ipld/unixfs/unixfs.go @@ -6,10 +6,12 @@ package unixfs import ( "errors" "fmt" + "os" + "time" proto "github.com/gogo/protobuf/proto" + files "github.com/ipfs/boxo/files" dag "github.com/ipfs/boxo/ipld/merkledag" - pb "github.com/ipfs/boxo/ipld/unixfs/pb" ipld "github.com/ipfs/go-ipld-format" ) @@ -69,6 +71,22 @@ func FilePBData(data []byte, totalsize uint64) []byte { return data } +func FilePBDataWithStat(data []byte, totalsize uint64, mode os.FileMode, mtime time.Time) []byte { + pbfile := new(pb.Data) + typ := pb.Data_File + pbfile.Type = &typ + pbfile.Data = data + pbfile.Filesize = proto.Uint64(totalsize) + + pbDataAddStat(pbfile, mode, mtime) + + data, err := proto.Marshal(pbfile) + if err != nil { + panic(err) + } + return data +} + // FolderPBData returns Bytes that represent a Directory. func FolderPBData() []byte { pbfile := new(pb.Data) @@ -83,6 +101,36 @@ func FolderPBData() []byte { return data } +func FolderPBDataWithStat(mode os.FileMode, mtime time.Time) []byte { + pbfile := new(pb.Data) + typ := pb.Data_Directory + pbfile.Type = &typ + + pbDataAddStat(pbfile, mode, mtime) + + data, err := proto.Marshal(pbfile) + if err != nil { + //this really shouldnt happen, i promise + panic(err) + } + return data +} + +func pbDataAddStat(data *pb.Data, mode os.FileMode, mtime time.Time) { + if mode != 0 { + data.Mode = proto.Uint32(files.ModePermsToUnixPerms(mode)) + } + if !mtime.IsZero() { + data.Mtime = &pb.IPFSTimestamp{ + Seconds: proto.Int64(mtime.Unix()), + } + + if nanos := uint32(mtime.Nanosecond()); nanos > 0 { + data.Mtime.Nanos = &nanos + } + } +} + // WrapData marshals raw bytes into a `Data_Raw` type protobuf message. func WrapData(b []byte) []byte { pbdata := new(pb.Data) @@ -303,6 +351,93 @@ func (n *FSNode) IsDir() bool { } } +// Mode returns the optionally stored file permissions +func (n *FSNode) Mode() (m os.FileMode) { + perms := n.format.GetMode() & 0xFFF + if perms != 0 { + m = files.UnixPermsToModePerms(perms) + switch n.Type() { + case pb.Data_Directory, pb.Data_HAMTShard: + m |= os.ModeDir + case pb.Data_Symlink: + m |= os.ModeSymlink + } + } + return m +} + +// SetMode stores the given mode permissions, or nullifies stored permissions +// if none were provided and there are no extended bits set. +func (n *FSNode) SetMode(m os.FileMode) { + n.SetModeFromUnixPermissions(files.ModePermsToUnixPerms(m)) +} + +// SetModeFromUnixPermissions stores the given unix permissions, or nullifies stored permissions +// if none were provided and there are no extended bits set. +func (n *FSNode) SetModeFromUnixPermissions(unixPerms uint32) { + // preserve existing most significant 20 bits + newMode := (n.format.GetMode() & 0xFFFFF000) | (unixPerms & 0xFFF) + + if unixPerms == 0 { + if newMode&0xFFFFF000 == 0 { + n.format.Mode = nil + return + } + } + n.format.Mode = &newMode +} + +// ExtendedMode returns the 20 bits of extended file mode +func (n *FSNode) ExtendedMode() uint32 { + return (n.format.GetMode() & 0xFFFFF000) >> 12 +} + +// SetExtendedMode stores the 20 bits of extended file mode, only the first +// 20 bits of the `mode` argument are used, the remaining 12 bits are ignored. +func (n *FSNode) SetExtendedMode(mode uint32) { + newMode := (mode << 12) | (0xFFF & n.format.GetMode()) + if newMode == 0 { + n.format.Mode = nil + } else { + n.format.Mode = &newMode + } +} + +// ModTime returns the stored last modified timestamp if available. +func (n *FSNode) ModTime() time.Time { + ts := n.format.GetMtime() + if ts == nil || ts.Seconds == nil { + return time.Time{} + } + if ts.Nanos == nil { + return time.Unix(*ts.Seconds, 0) + } + if *ts.Nanos < 1 || *ts.Nanos > 999999999 { + return time.Time{} + } + + return time.Unix(*ts.Seconds, int64(*ts.Nanos)) +} + +// SetModTime stores the given last modified timestamp, otherwise nullifies stored timestamp. +func (n *FSNode) SetModTime(ts time.Time) { + if ts.IsZero() { + n.format.Mtime = nil + return + } + + if n.format.Mtime == nil { + n.format.Mtime = &pb.IPFSTimestamp{} + } + + n.format.Mtime.Seconds = proto.Int64(ts.Unix()) + if ts.Nanosecond() > 0 { + n.format.Mtime.Nanos = proto.Uint32(uint32(ts.Nanosecond())) + } else { + n.format.Mtime.Nanos = nil + } +} + // Metadata is used to store additional FSNode information. type Metadata struct { MimeType string @@ -360,6 +495,10 @@ func EmptyDirNode() *dag.ProtoNode { return dag.NodeWithData(FolderPBData()) } +func EmptyDirNodeWithStat(mode os.FileMode, mtime time.Time) *dag.ProtoNode { + return dag.NodeWithData(FolderPBDataWithStat(mode, mtime)) +} + // EmptyFileNode creates an empty file Protonode. func EmptyFileNode() *dag.ProtoNode { return dag.NodeWithData(FilePBData(nil, 0)) diff --git a/ipld/unixfs/unixfs_test.go b/ipld/unixfs/unixfs_test.go index b785be8ad..4cbc22ca8 100644 --- a/ipld/unixfs/unixfs_test.go +++ b/ipld/unixfs/unixfs_test.go @@ -2,7 +2,9 @@ package unixfs import ( "bytes" + "os" "testing" + "time" proto "github.com/gogo/protobuf/proto" @@ -183,3 +185,251 @@ func TestIsDir(t *testing.T) { } } } + +func (n *FSNode) getPbData(t *testing.T) *pb.Data { + b, err := n.GetBytes() + if err != nil { + t.Fatal(err) + } + + pbn := new(pb.Data) + err = proto.Unmarshal(b, pbn) + if err != nil { + t.Fatal(err) + } + return pbn +} + +func TestMode(t *testing.T) { + fsn := NewFSNode(TDirectory) + fsn.SetMode(1) + if !fsn.Mode().IsDir() { + t.Fatal("expected mode for directory") + } + + fsn = NewFSNode(TSymlink) + fsn.SetMode(1) + if fsn.Mode()&os.ModeSymlink != os.ModeSymlink { + t.Fatal("expected mode for symlink") + } + + fsn = NewFSNode(TFile) + + // not stored + if fsn.Mode() != 0 { + t.Fatal("expected mode not to be set") + } + + fileMode := os.FileMode(0640) + fsn.SetMode(fileMode) + if !fsn.Mode().IsRegular() { + t.Fatal("expected a regular file mode") + } + mode := fsn.Mode() + + if mode&os.ModePerm != fileMode { + t.Fatalf("expected permissions to be %O but got %O", fileMode, mode&os.ModePerm) + } + if mode&0xFFFFF000 != 0 { + t.Fatalf("expected high-order 20 bits of mode to be clear but got %b", (mode&0xFFFFF000)>>12) + } + + fsn.SetMode(fileMode | os.ModeSticky) + mode = fsn.Mode() + if mode&os.ModePerm != fileMode&os.ModePerm { + t.Fatalf("expected permissions to be %O but got %O", fileMode, mode&os.ModePerm) + } + if mode&os.ModeSticky == 0 { + t.Fatal("expected permissions to have sticky bit set") + } + if mode&os.ModeSetuid != 0 { + t.Fatal("expected permissions to have setuid bit unset") + } + if mode&os.ModeSetgid != 0 { + t.Fatal("expected permissions to have setgid bit unset") + } + + fsn.SetMode(fileMode | os.ModeSticky | os.ModeSetuid) + mode = fsn.Mode() + if mode&os.ModePerm != fileMode&os.ModePerm { + t.Fatalf("expected permissions to be %O but got %O", fileMode, mode&os.ModePerm) + } + if mode&os.ModeSticky == 0 { + t.Fatal("expected permissions to have sticky bit set") + } + if mode&os.ModeSetuid == 0 { + t.Fatal("expected permissions to have setuid bit set") + } + if mode&os.ModeSetgid != 0 { + t.Fatal("expected permissions to have setgid bit unset") + } + + fsn.SetMode(fileMode | os.ModeSetuid | os.ModeSetgid) + mode = fsn.Mode() + if mode&os.ModePerm != fileMode&os.ModePerm { + t.Fatalf("expected permissions to be %O but got %O", fileMode, mode&os.ModePerm) + } + if mode&os.ModeSticky != 0 { + t.Fatal("expected permissions to have sticky bit unset") + } + if mode&os.ModeSetuid == 0 { + t.Fatal("expected permissions to have setuid bit set") + } + if mode&os.ModeSetgid == 0 { + t.Fatal("expected permissions to have setgid bit set") + } + + // check the internal format (unix permissions) + fsn.SetMode(fileMode | os.ModeSetuid | os.ModeSticky) + pbn := fsn.getPbData(t) + // unix perms setuid and sticky bits should also be set + expected := uint32(05000 | (fileMode & os.ModePerm)) + if *pbn.Mode != expected { + t.Fatalf("expected stored permissions to be %O but got %O", expected, *pbn.Mode) + } + + fsn.SetMode(0) + pbn = fsn.getPbData(t) + if pbn.Mode != nil { + t.Fatal("expected file mode to be unset") + } + + fsn.SetExtendedMode(1) + fsn.SetMode(0) + pbn = fsn.getPbData(t) + if pbn.Mode == nil { + t.Fatal("expected extended mode to be preserved") + } +} + +func TestExtendedMode(t *testing.T) { + fsn := NewFSNode(TFile) + fsn.SetMode(os.ModePerm | os.ModeSetuid | os.ModeSticky) + const expectedUnixMode = uint32(05777) + + expectedExtMode := uint32(0xAAAAA) + fsn.SetExtendedMode(expectedExtMode) + extMode := fsn.ExtendedMode() + if extMode != expectedExtMode { + t.Fatalf("expected extended mode to be %X but got %X", expectedExtMode, extMode) + } + pbn := fsn.getPbData(t) + expectedPbMode := (expectedExtMode << 12) | (expectedUnixMode & 0xFFF) + if *pbn.Mode != expectedPbMode { + t.Fatalf("expected stored mode to be %b but got %b", expectedPbMode, *pbn.Mode) + } + + expectedExtMode = uint32(0x55555) + fsn.SetExtendedMode(expectedExtMode) + extMode = fsn.ExtendedMode() + if extMode != expectedExtMode { + t.Fatalf("expected extended mode to be %X but got %X", expectedExtMode, extMode) + } + pbn = fsn.getPbData(t) + expectedPbMode = (expectedExtMode << 12) | (expectedUnixMode & 0xFFF) + if *pbn.Mode != expectedPbMode { + t.Fatalf("expected stored mode to be %b but got %b", expectedPbMode, *pbn.Mode) + } + + // ignore bits 21..32 + expectedExtMode = uint32(0xFFFFF) + fsn.SetExtendedMode(0xAAAFFFFF) + extMode = fsn.ExtendedMode() + if extMode != expectedExtMode { + t.Fatalf("expected extended mode to be %X but got %X", expectedExtMode, extMode) + } + pbn = fsn.getPbData(t) + expectedPbMode = (expectedExtMode << 12) | (expectedUnixMode & 0xFFF) + if *pbn.Mode != expectedPbMode { + t.Fatalf("expected raw mode to be %b but got %b", expectedPbMode, *pbn.Mode) + } + + fsn.SetMode(0) + fsn.SetExtendedMode(0) + pbn = fsn.getPbData(t) + if pbn.Mode != nil { + t.Fatal("expected file mode to be unset") + } +} + +func (n *FSNode) setPbModTime(seconds *int64, nanos *uint32) { + if n.format.Mtime == nil { + n.format.Mtime = &pb.IPFSTimestamp{} + } + + n.format.Mtime.Seconds = seconds + n.format.Mtime.Nanos = nanos +} + +func TestModTime(t *testing.T) { + tm := time.Now() + expectedUnix := tm.Unix() + n := NewFSNode(TFile) + + // not stored + mt := n.ModTime() + if !mt.IsZero() { + t.Fatal("expected modification time not to be set") + } + + // valid timestamps + n.SetModTime(tm) + mt = n.ModTime() + if !mt.Equal(tm) { + t.Fatalf("expected modification time to be %v but got %v", tm, mt) + } + + tm = time.Unix(expectedUnix, 0) + n.SetModTime(tm) + pbn := n.getPbData(t) + if pbn.Mtime.Nanos != nil { + t.Fatal("expected nanoseconds to be nil") + } + mt = n.ModTime() + if !mt.Equal(tm) { + t.Fatalf("expected modification time to be %v but got %v", tm, mt) + } + + tm = time.Unix(expectedUnix, 3489753) + n.SetModTime(tm) + mt = n.ModTime() + if !mt.Equal(tm) { + t.Fatalf("expected modification time to be %v but got %v", tm, mt) + } + + tm = time.Time{} + n.SetModTime(tm) + pbn = n.getPbData(t) + if pbn.Mtime != nil { + t.Fatal("expected modification time to be unset") + } + mt = n.ModTime() + if !mt.Equal(tm) { + t.Fatalf("expected modification time to be %v but got %v", tm, mt) + } + + n.setPbModTime(&expectedUnix, nil) + mt = n.ModTime() + if !mt.Equal(time.Unix(expectedUnix, 0)) { + t.Fatalf("expected modification time to be %v but got %v", time.Unix(expectedUnix, 0), mt) + } + + // invalid timestamps + n.setPbModTime(nil, proto.Uint32(1000)) + mt = n.ModTime() + if !mt.IsZero() { + t.Fatal("expected modification time not to be set") + } + + n.setPbModTime(&expectedUnix, proto.Uint32(0)) + mt = n.ModTime() + if !mt.IsZero() { + t.Fatal("expected modification time not to be set") + } + + n.setPbModTime(&expectedUnix, proto.Uint32(1000000000)) + mt = n.ModTime() + if !mt.IsZero() { + t.Fatal("expected modification time not to be set") + } +} diff --git a/mfs/dir.go b/mfs/dir.go index 86c85d1c5..38302ac39 100644 --- a/mfs/dir.go +++ b/mfs/dir.go @@ -41,8 +41,6 @@ type Directory struct { // UnixFS directory implementation used for creating, // reading and editing directories. unixfsDir uio.Directory - - modTime time.Time } // NewDirectory constructs a new MFS directory. @@ -64,7 +62,6 @@ func NewDirectory(ctx context.Context, name string, node ipld.Node, parent paren ctx: ctx, unixfsDir: db, entriesCache: make(map[string]FSNode), - modTime: time.Now(), }, nil } @@ -135,8 +132,6 @@ func (d *Directory) updateChild(c child) error { return err } - d.modTime = time.Now() - return nil } @@ -292,6 +287,10 @@ func (d *Directory) ForEachEntry(ctx context.Context, f func(NodeListing) error) } func (d *Directory) Mkdir(name string) (*Directory, error) { + return d.MkdirWithOpts(name, MkdirOpts{}) +} + +func (d *Directory) MkdirWithOpts(name string, opts MkdirOpts) (*Directory, error) { d.lock.Lock() defer d.lock.Unlock() @@ -307,7 +306,7 @@ func (d *Directory) Mkdir(name string) (*Directory, error) { } } - ndir := ft.EmptyDirNode() + ndir := ft.EmptyDirNodeWithStat(opts.Mode, opts.ModTime) ndir.SetCidBuilder(d.GetCidBuilder()) err = d.dagService.Add(d.ctx, ndir) @@ -367,7 +366,6 @@ func (d *Directory) AddChild(name string, nd ipld.Node) error { return err } - d.modTime = time.Now() return nil } @@ -427,3 +425,68 @@ func (d *Directory) GetNode() (ipld.Node, error) { return nd.Copy(), err } + +func (d *Directory) SetMode(mode os.FileMode) error { + nd, err := d.GetNode() + if err != nil { + return err + } + + fsn, err := ft.ExtractFSNode(nd) + if err != nil { + return err + } + + fsn.SetMode(mode) + data, err := fsn.GetBytes() + if err != nil { + return err + } + + return d.setNodeData(data, nd.Links()) +} + +func (d *Directory) SetModTime(ts time.Time) error { + nd, err := d.GetNode() + if err != nil { + return err + } + + fsn, err := ft.ExtractFSNode(nd) + if err != nil { + return err + } + + fsn.SetModTime(ts) + data, err := fsn.GetBytes() + if err != nil { + return err + } + + return d.setNodeData(data, nd.Links()) +} + +func (d *Directory) setNodeData(data []byte, links []*ipld.Link) error { + nd := dag.NodeWithData(data) + nd.SetLinks(links) + + err := d.dagService.Add(d.ctx, nd) + if err != nil { + return err + } + + err = d.parent.updateChildEntry(child{d.name, nd}) + if err != nil { + return err + } + + d.lock.Lock() + defer d.lock.Unlock() + db, err := uio.NewDirectoryFromNode(d.dagService, nd) + if err != nil { + return err + } + d.unixfsDir = db + + return nil +} diff --git a/mfs/file.go b/mfs/file.go index 56c2b0046..ef8ce5dfa 100644 --- a/mfs/file.go +++ b/mfs/file.go @@ -3,7 +3,9 @@ package mfs import ( "context" "errors" + "os" "sync" + "time" dag "github.com/ipfs/boxo/ipld/merkledag" ft "github.com/ipfs/boxo/ipld/unixfs" @@ -177,3 +179,91 @@ func (fi *File) Sync() error { func (fi *File) Type() NodeType { return TFile } + +func (fi *File) Mode() (os.FileMode, error) { + fi.nodeLock.RLock() + defer fi.nodeLock.RUnlock() + + nd, err := fi.GetNode() + if err == nil { + fsn, err := ft.ExtractFSNode(nd) + if err == nil { + return fsn.Mode() & 0xFFF, nil + } + } + + return 0, err +} + +func (fi *File) SetMode(mode os.FileMode) error { + nd, err := fi.GetNode() + if err != nil { + return err + } + + fsn, err := ft.ExtractFSNode(nd) + if err != nil { + return err + } + + fsn.SetMode(mode) + data, err := fsn.GetBytes() + if err != nil { + return err + } + + return fi.setNodeData(data) +} + +// ModTime returns the files' last modification time +func (fi *File) ModTime() (time.Time, error) { + fi.nodeLock.RLock() + defer fi.nodeLock.RUnlock() + + nd, err := fi.GetNode() + if err == nil { + fsn, err := ft.ExtractFSNode(nd) + if err == nil { + return fsn.ModTime(), nil + } + } + + return time.Time{}, err +} + +// SetModTime sets the files' last modification time +func (fi *File) SetModTime(ts time.Time) error { + nd, err := fi.GetNode() + if err != nil { + return err + } + + fsn, err := ft.ExtractFSNode(nd) + if err != nil { + return err + } + + fsn.SetModTime(ts) + data, err := fsn.GetBytes() + if err != nil { + return err + } + + return fi.setNodeData(data) +} + +func (fi *File) setNodeData(data []byte) error { + nd := dag.NodeWithData(data) + err := fi.inode.dagService.Add(context.TODO(), nd) + if err != nil { + return err + } + + fi.nodeLock.Lock() + defer fi.nodeLock.Unlock() + fi.node = nd + parent := fi.inode.parent + name := fi.inode.name + + return parent.updateChildEntry(child{name, fi.node}) +} diff --git a/mfs/mfs_test.go b/mfs/mfs_test.go index eb5585a64..80d0026a1 100644 --- a/mfs/mfs_test.go +++ b/mfs/mfs_test.go @@ -11,6 +11,7 @@ import ( "math/rand" "os" gopath "path" + "runtime" "sort" "strings" "sync" @@ -511,7 +512,19 @@ func TestMfsFile(t *testing.T) { fi := fsn.(*File) if fi.Type() != TFile { - t.Fatal("some is seriously wrong here") + t.Fatal("something is seriously wrong here") + } + + if m, err := fi.Mode(); err != nil { + t.Fatal("failed to get file mode: ", err) + } else if m != 0 { + t.Fatal("mode should not be set on a new file") + } + + if ts, err := fi.ModTime(); err != nil { + t.Fatal("failed to get file mtime: ", err) + } else if !ts.IsZero() { + t.Fatal("modification time should not be set on a new file") } wfd, err := fi.Open(Flags{Read: true, Write: true, Sync: true}) @@ -615,6 +628,12 @@ func TestMfsFile(t *testing.T) { t.Fatal(err) } + if ts, err := fi.ModTime(); err != nil { + t.Fatal("failed to get file mtime: ", err) + } else if !ts.IsZero() { + t.Fatal("file with unset modification time should not update modification time") + } + // make sure we can get node. TODO: verify it later _, err = fi.GetNode() if err != nil { @@ -622,6 +641,157 @@ func TestMfsFile(t *testing.T) { } } +func TestMfsModeAndModTime(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ds, rt := setupRoot(ctx, t) + rootdir := rt.GetDirectory() + nd := getRandFile(t, ds, 1000) + + err := rootdir.AddChild("file", nd) + if err != nil { + t.Fatal(err) + } + + fsn, err := rootdir.Child("file") + if err != nil { + t.Fatal(err) + } + + fi := fsn.(*File) + + if fi.Type() != TFile { + t.Fatal("something is seriously wrong here") + } + + var mode os.FileMode + ts, _ := time.Now(), time.Time{} + + // can set mode + if err = fi.SetMode(0644); err == nil { + if mode, err = fi.Mode(); mode != 0644 { + t.Fatal("failed to get correct mode of file") + } + } + if err != nil { + t.Fatal("failed to check file mode: ", err) + } + + // can set last modification time + if err = fi.SetModTime(ts); err == nil { + ts2, err := fi.ModTime() + if err != nil { + t.Fatal(err) + } + if !ts2.Equal(ts) { + t.Fatal("failed to get correct modification time of file") + } + } + if err != nil { + t.Fatal("failed to check file modification time: ", err) + } + + // test modification time update after write (on closing file) + if runtime.GOOS == "windows" { + time.Sleep(3 * time.Second) // for os with low-res mod time. + } + wfd, err := fi.Open(Flags{Read: false, Write: true, Sync: true}) + if err != nil { + t.Fatal(err) + } + _, err = wfd.Write([]byte("test")) + if err != nil { + t.Fatal(err) + } + err = wfd.Close() + if err != nil { + t.Fatal(err) + } + ts2, err := fi.ModTime() + if err != nil { + t.Fatal(err) + } + if !ts2.After(ts) { + t.Fatal("modification time should be updated after file write") + } + + // writeAt + ts = ts2 + if runtime.GOOS == "windows" { + time.Sleep(3 * time.Second) // for os with low-res mod time. + } + wfd, err = fi.Open(Flags{Read: false, Write: true, Sync: true}) + if err != nil { + t.Fatal(err) + } + _, err = wfd.WriteAt([]byte("test"), 42) + if err != nil { + t.Fatal(err) + } + err = wfd.Close() + if err != nil { + t.Fatal(err) + } + ts2, err = fi.ModTime() + if err != nil { + t.Fatal(err) + } + if !ts2.After(ts) { + t.Fatal("modification time should be updated after file writeAt") + } + + // truncate (shrink) + ts = ts2 + if runtime.GOOS == "windows" { + time.Sleep(3 * time.Second) // for os with low-res mod time. + } + wfd, err = fi.Open(Flags{Read: false, Write: true, Sync: true}) + if err != nil { + t.Fatal(err) + } + err = wfd.Truncate(100) + if err != nil { + t.Fatal(err) + } + err = wfd.Close() + if err != nil { + t.Fatal(err) + } + ts2, err = fi.ModTime() + if err != nil { + t.Fatal(err) + } + if !ts2.After(ts) { + t.Fatal("modification time should be updated after file truncate (shrink)") + } + + // truncate (expand) + ts = ts2 + if runtime.GOOS == "windows" { + time.Sleep(3 * time.Second) // for os with low-res mod time. + } + wfd, err = fi.Open(Flags{Read: false, Write: true, Sync: true}) + if err != nil { + t.Fatal(err) + } + err = wfd.Truncate(1500) + if err != nil { + t.Fatal(err) + } + err = wfd.Close() + if err != nil { + t.Fatal(err) + } + ts2, err = fi.ModTime() + if err != nil { + t.Fatal(err) + } + if !ts2.After(ts) { + t.Fatal("modification time should be updated after file truncate (expand)") + } +} + func TestMfsDirListNames(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/mfs/ops.go b/mfs/ops.go index 693264704..09dbab00f 100644 --- a/mfs/ops.go +++ b/mfs/ops.go @@ -7,6 +7,7 @@ import ( "os" gopath "path" "strings" + "time" cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" @@ -122,6 +123,8 @@ type MkdirOpts struct { Mkparents bool Flush bool CidBuilder cid.Builder + Mode os.FileMode + ModTime time.Time } // Mkdir creates a directory at 'path' under the directory 'd', creating @@ -171,7 +174,7 @@ func Mkdir(r *Root, pth string, opts MkdirOpts) error { cur = next } - final, err := cur.Mkdir(parts[len(parts)-1]) + final, err := cur.MkdirWithOpts(parts[len(parts)-1], opts) if err != nil { if !opts.Mkparents || err != os.ErrExist || final == nil { return err @@ -243,3 +246,21 @@ func FlushPath(ctx context.Context, rt *Root, pth string) (ipld.Node, error) { rt.repub.WaitPub(ctx) return nd.GetNode() } + +func Chmod(rt *Root, pth string, mode os.FileMode) error { + nd, err := Lookup(rt, pth) + if err != nil { + return err + } + + return nd.SetMode(mode) +} + +func Touch(rt *Root, pth string, ts time.Time) error { + nd, err := Lookup(rt, pth) + if err != nil { + return err + } + + return nd.SetModTime(ts) +} diff --git a/mfs/root.go b/mfs/root.go index c08d2d053..5a7cb7ed1 100644 --- a/mfs/root.go +++ b/mfs/root.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "os" "time" dag "github.com/ipfs/boxo/ipld/merkledag" @@ -73,6 +74,8 @@ type FSNode interface { Flush() error Type() NodeType + SetModTime(ts time.Time) error + SetMode(mode os.FileMode) error } // IsDir checks whether the FSNode is dir type From 91e095473391e3475ca69561e2280ae03aa030cb Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Wed, 14 Aug 2024 00:15:45 -0700 Subject: [PATCH 10/18] Moved functionality to set file metadata into files package --- files/meta.go | 46 ++++++++++++++++++++ files/meta_other.go | 23 ++++++++++ files/meta_posix.go | 41 +++++++++++++++++ tar/util_windows.go => files/meta_windows.go | 27 ++++++------ out | 0 tar/extractor.go | 37 +++------------- tar/util_other.go | 25 ----------- tar/util_posix.go | 41 ----------------- 8 files changed, 130 insertions(+), 110 deletions(-) create mode 100644 files/meta.go create mode 100644 files/meta_other.go create mode 100644 files/meta_posix.go rename tar/util_windows.go => files/meta_windows.go (52%) delete mode 100644 out delete mode 100644 tar/util_other.go delete mode 100644 tar/util_posix.go diff --git a/files/meta.go b/files/meta.go new file mode 100644 index 000000000..4ae38a991 --- /dev/null +++ b/files/meta.go @@ -0,0 +1,46 @@ +package files + +import ( + "fmt" + "os" + "time" +) + +// UpdateMeta sets the unix mode and modification time of the filesystem object +// referenced by path. +func UpdateMeta(path string, mode os.FileMode, mtime time.Time) error { + if err := UpdateModTime(path, mtime); err != nil { + return err + } + return UpdateFileMode(path, mode) +} + +// UpdateUnix sets the unix mode and modification time of the filesystem object +// referenced by path. The mode is in the form of a unix mode. +func UpdateMetaUnix(path string, mode uint32, mtime time.Time) error { + return UpdateMeta(path, UnixPermsToModePerms(mode), mtime) +} + +// UpdateFileMode sets the unix mode of the filesystem object referenced by path. +func UpdateFileMode(path string, mode os.FileMode) error { + if err := updateMode(path, mode); err != nil { + return fmt.Errorf("[%v] failed to update file mode on '%s'", err, path) + } + return nil +} + +// UpdateFileModeUnix sets the unix mode of the filesystem object referenced by +// path. It takes the mode in the form of a unix mode. +func UpdateFileModeUnix(path string, mode uint32) error { + return UpdateFileMode(path, UnixPermsToModePerms(mode)) +} + +// UpdateModTime sets the last access and modification time of the target +// filesystem object to the given time. When the given path references a +// symlink, if supported, the symlink is updated. +func UpdateModTime(path string, mtime time.Time) error { + if err := updateMtime(path, mtime); err != nil { + return fmt.Errorf("[%v] failed to update last modification time on '%s'", err, path) + } + return nil +} diff --git a/files/meta_other.go b/files/meta_other.go new file mode 100644 index 000000000..2c4645049 --- /dev/null +++ b/files/meta_other.go @@ -0,0 +1,23 @@ +//go:build !linux && !freebsd && !netbsd && !openbsd && !dragonfly && !windows +// +build !linux,!freebsd,!netbsd,!openbsd,!dragonfly,!windows + +package files + +import ( + "os" + "time" +) + +func updateMode(path string, mode os.FileMode) error { + if mode == 0 { + return nil + } + return os.Chmod(path, mode) +} + +func updateMtime(path string, mtime time.Time) error { + if mtime.IsZero() { + return nil + } + return os.Chtimes(path, mtime, mtime) +} diff --git a/files/meta_posix.go b/files/meta_posix.go new file mode 100644 index 000000000..808cbb997 --- /dev/null +++ b/files/meta_posix.go @@ -0,0 +1,41 @@ +//go:build linux || freebsd || netbsd || openbsd || dragonfly +// +build linux freebsd netbsd openbsd dragonfly + +package files + +import ( + "os" + "syscall" + "time" + "unsafe" + + "golang.org/x/sys/unix" +) + +func updateMode(path string, mode os.FileMode) error { + if mode == 0 { + return nil + } + return os.Chmod(path, mode) +} + +func updateMtime(path string, mtime time.Time) error { + if mtime.IsZero() { + return nil + } + var AtFdCwd = -100 + pathname, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + + tm := syscall.NsecToTimespec(mtime.UnixNano()) + ts := [2]syscall.Timespec{tm, tm} + _, _, e := syscall.Syscall6(syscall.SYS_UTIMENSAT, uintptr(AtFdCwd), + uintptr(unsafe.Pointer(pathname)), uintptr(unsafe.Pointer(&ts)), + uintptr(unix.AT_SYMLINK_NOFOLLOW), 0, 0) + if e != 0 { + return error(e) + } + return nil +} diff --git a/tar/util_windows.go b/files/meta_windows.go similarity index 52% rename from tar/util_windows.go rename to files/meta_windows.go index b378e45d1..e060ec02f 100644 --- a/tar/util_windows.go +++ b/files/meta_windows.go @@ -1,4 +1,4 @@ -package tar +package files import ( "os" @@ -10,22 +10,21 @@ import ( // bits are currently unused. // // Use mode 0400 for a read-only file and 0600 for a readable+writable file. -func updateMode(path string, mode int64) error { - if mode != 0 { - // read+write if owner, group or world writeable - if mode&0222 != 0 { - return os.Chmod(path, 0600) - } - // otherwise read-only - return os.Chmod(path, 0400) +func updateMode(path string, mode os.FileMode) error { + if mode == 0 { + return nil } - - return nil + // read+write if owner, group or world writeable + if mode&0222 != 0 { + return os.Chmod(path, 0600) + } + // otherwise read-only + return os.Chmod(path, 0400) } func updateMtime(path string, mtime time.Time) error { - if !mtime.IsZero() { - return os.Chtimes(path, mtime, mtime) + if mtime.IsZero() { + return nil } - return nil + return os.Chtimes(path, mtime, mtime) } diff --git a/out b/out deleted file mode 100644 index e69de29bb..000000000 diff --git a/tar/extractor.go b/tar/extractor.go index fd526472f..550fb895c 100644 --- a/tar/extractor.go +++ b/tar/extractor.go @@ -10,6 +10,8 @@ import ( "runtime" "strings" "time" + + "github.com/ipfs/boxo/files" ) var ( @@ -64,7 +66,7 @@ func (te *Extractor) Extract(reader io.Reader) error { doUpdates := func() error { for i := len(te.deferredUpdates) - 1; i >= 0; i-- { m := te.deferredUpdates[i] - err := updateMeta(m.path, m.mode, m.mtime) + err := files.UpdateMetaUnix(m.path, uint32(m.mode), m.mtime) if err != nil { return err } @@ -151,7 +153,7 @@ func (te *Extractor) Extract(reader io.Reader) error { if err := te.extractFile(outputPath, tarReader); err != nil { return err } - if err := updateMeta(outputPath, header.Mode, header.ModTime); err != nil { + if err := files.UpdateMetaUnix(outputPath, uint32(header.Mode), header.ModTime); err != nil { return err } } else if err := te.extractSymlink(outputPath, rootOutputPath, header); err != nil { @@ -219,7 +221,7 @@ func (te *Extractor) Extract(reader io.Reader) error { if err := te.extractFile(outputPath, tarReader); err != nil { return err } - if err := updateMeta(outputPath, header.Mode, header.ModTime); err != nil { + if err := files.UpdateMetaUnix(outputPath, uint32(header.Mode), header.ModTime); err != nil { return err } case tar.TypeSymlink: @@ -336,7 +338,7 @@ func (te *Extractor) extractSymlink(path, rootPath string, h *tar.Header) error switch runtime.GOOS { case "linux", "freebsd", "netbsd", "openbsd", "dragonfly": - return updateModTime(path, h.ModTime) + return files.UpdateModTime(path, h.ModTime) default: return nil } @@ -423,7 +425,7 @@ func (te *Extractor) deferUpdate(path string, header *tar.Header) error { // if possible, apply the previous deferral. m := te.deferredUpdates[n-1] if strings.HasPrefix(m.path, prefix()) { - err := updateMeta(m.path, m.mode, m.mtime) + err := files.UpdateMetaUnix(m.path, uint32(m.mode), m.mtime) if err != nil { return err } @@ -439,28 +441,3 @@ func (te *Extractor) deferUpdate(path string, header *tar.Header) error { return nil } - -func updateMeta(path string, mode int64, mtime time.Time) error { - if err := updateModTime(path, mtime); err != nil { - return err - } - return updateFileMode(path, mode) -} - -// updateFileMode sets the unix mode of the filesystem object referenced by path. -func updateFileMode(path string, mode int64) error { - if err := updateMode(path, mode); err != nil { - return fmt.Errorf("[%v] failed to update file mode on '%s'", err, path) - } - return nil -} - -// updateModTime sets the last access and modification time of the target -// filesystem object to the given time. When the given path references a -// symlink, if supported, the symlink is updated. -func updateModTime(path string, mtime time.Time) error { - if err := updateMtime(path, mtime); err != nil { - return fmt.Errorf("[%v] failed to update last modification time on '%s'", err, path) - } - return nil -} diff --git a/tar/util_other.go b/tar/util_other.go deleted file mode 100644 index 99ae84367..000000000 --- a/tar/util_other.go +++ /dev/null @@ -1,25 +0,0 @@ -//go:build !linux && !freebsd && !netbsd && !openbsd && !dragonfly && !windows -// +build !linux,!freebsd,!netbsd,!openbsd,!dragonfly,!windows - -package tar - -import ( - "os" - "time" - - "github.com/ipfs/boxo/files" -) - -func updateMode(path string, mode int64) error { - if mode != 0 { - return os.Chmod(path, files.UnixPermsToModePerms(uint32(mode))) - } - return nil -} - -func updateMtime(path string, mtime time.Time) error { - if !mtime.IsZero() { - return os.Chtimes(path, mtime, mtime) - } - return nil -} diff --git a/tar/util_posix.go b/tar/util_posix.go deleted file mode 100644 index 418dc7dd6..000000000 --- a/tar/util_posix.go +++ /dev/null @@ -1,41 +0,0 @@ -//go:build linux || freebsd || netbsd || openbsd || dragonfly -// +build linux freebsd netbsd openbsd dragonfly - -package tar - -import ( - "golang.org/x/sys/unix" - "os" - "syscall" - "time" - "unsafe" - - "github.com/ipfs/boxo/files" -) - -func updateMode(path string, mode int64) error { - if mode != 0 { - return os.Chmod(path, files.UnixPermsToModePerms(uint32(mode))) - } - return nil -} - -func updateMtime(path string, mtime time.Time) error { - if !mtime.IsZero() { - var AtFdCwd = -100 - pathname, err := syscall.BytePtrFromString(path) - if err != nil { - return err - } - - tm := syscall.NsecToTimespec(mtime.UnixNano()) - ts := [2]syscall.Timespec{tm, tm} - _, _, e := syscall.Syscall6(syscall.SYS_UTIMENSAT, uintptr(AtFdCwd), - uintptr(unsafe.Pointer(pathname)), uintptr(unsafe.Pointer(&ts)), - uintptr(unix.AT_SYMLINK_NOFOLLOW), 0, 0) - if e != 0 { - return error(e) - } - } - return nil -} From b27ba953ad58e59bed42e3ed7849e97c8fa9632d Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Wed, 14 Aug 2024 00:52:07 -0700 Subject: [PATCH 11/18] Update required go version --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 3ccf62b41..060e9c5db 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/ipfs/boxo -go 1.21 +go 1.22 require ( github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 From 1e7b8530d354e0b641fd776e3c6b45948b6f5e90 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 19 Aug 2024 20:33:33 +0200 Subject: [PATCH 12/18] chore: go mod tidy --- examples/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/go.mod b/examples/go.mod index 615b39356..fc94d2519 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -1,6 +1,6 @@ module github.com/ipfs/boxo/examples -go 1.21 +go 1.22 require ( github.com/ipfs/boxo v0.19.0 From f35ddcbd4de7d9148fe6bea344260d6e3ea8be1d Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 19 Aug 2024 21:50:01 +0200 Subject: [PATCH 13/18] chore: comments and longer changelog --- CHANGELOG.md | 2 +- files/file.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7bbdcac9..25f652999 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ The following emojis are used to highlight certain changes: ### Added -- support UnixFS 1.5 file mode and modification times +- `files`, `ipld/unixfs`, `mfs` and `tar` now support optional UnixFS 1.5 mode and modification time metadata ### Changed diff --git a/files/file.go b/files/file.go index 2fd907931..e2ece2862 100644 --- a/files/file.go +++ b/files/file.go @@ -18,10 +18,11 @@ var ( type Node interface { io.Closer - // Mode returns the file's mode. + // Mode returns the mode. + // Optional, if unknown/unspecified returns zero. Mode() os.FileMode - // ModTime returns the file's last modification time. If the last + // ModTime returns the last modification time. If the last // modification time is unknown/unspecified ModTime returns zero. ModTime() (mtime time.Time) From f4259d68d2d12bbde204d1564b0c8cad7832cf55 Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Tue, 20 Aug 2024 14:28:29 -0700 Subject: [PATCH 14/18] Fix mfs touch and chmod Fixes #660 --- ipld/unixfs/unixfs.go | 3 ++- mfs/file.go | 38 ++++++++++++++++++++++++-------------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/ipld/unixfs/unixfs.go b/ipld/unixfs/unixfs.go index bdc54518f..fb2c9bbf2 100644 --- a/ipld/unixfs/unixfs.go +++ b/ipld/unixfs/unixfs.go @@ -36,6 +36,7 @@ const ( // Common errors var ( ErrMalformedFileFormat = errors.New("malformed data in file format") + ErrNotProtoNode = errors.New("expected a ProtoNode as internal node") ErrUnrecognizedType = errors.New("unrecognized node type") ) @@ -544,7 +545,7 @@ func ReadUnixFSNodeData(node ipld.Node) (data []byte, err error) { func ExtractFSNode(node ipld.Node) (*FSNode, error) { protoNode, ok := node.(*dag.ProtoNode) if !ok { - return nil, errors.New("expected a ProtoNode as internal node") + return nil, ErrNotProtoNode } fsNode, err := FSNodeFromBytes(protoNode.Data()) diff --git a/mfs/file.go b/mfs/file.go index ef8ce5dfa..aff025db6 100644 --- a/mfs/file.go +++ b/mfs/file.go @@ -185,14 +185,14 @@ func (fi *File) Mode() (os.FileMode, error) { defer fi.nodeLock.RUnlock() nd, err := fi.GetNode() - if err == nil { - fsn, err := ft.ExtractFSNode(nd) - if err == nil { - return fsn.Mode() & 0xFFF, nil - } + if err != nil { + return 0, err } - - return 0, err + fsn, err := ft.ExtractFSNode(nd) + if err != nil { + return 0, err + } + return fsn.Mode() & 0xFFF, nil } func (fi *File) SetMode(mode os.FileMode) error { @@ -203,6 +203,11 @@ func (fi *File) SetMode(mode os.FileMode) error { fsn, err := ft.ExtractFSNode(nd) if err != nil { + if errors.Is(err, ft.ErrNotProtoNode) { + // Wrap raw node in protonode. + data := nd.RawData() + return fi.setNodeData(ft.FilePBDataWithStat(data, uint64(len(data)), mode, time.Time{})) + } return err } @@ -221,14 +226,14 @@ func (fi *File) ModTime() (time.Time, error) { defer fi.nodeLock.RUnlock() nd, err := fi.GetNode() - if err == nil { - fsn, err := ft.ExtractFSNode(nd) - if err == nil { - return fsn.ModTime(), nil - } + if err != nil { + return time.Time{}, err } - - return time.Time{}, err + fsn, err := ft.ExtractFSNode(nd) + if err != nil { + return time.Time{}, err + } + return fsn.ModTime(), nil } // SetModTime sets the files' last modification time @@ -240,6 +245,11 @@ func (fi *File) SetModTime(ts time.Time) error { fsn, err := ft.ExtractFSNode(nd) if err != nil { + if errors.Is(err, ft.ErrNotProtoNode) { + // Wrap raw node in protonode. + data := nd.RawData() + return fi.setNodeData(ft.FilePBDataWithStat(data, uint64(len(data)), 0, ts)) + } return err } From 76a83de88d6c08b92db81e4f1bbcd6b85764875c Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Tue, 20 Aug 2024 15:20:56 -0700 Subject: [PATCH 15/18] Add test setting mode and mtime on raw file --- mfs/mfs_test.go | 76 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/mfs/mfs_test.go b/mfs/mfs_test.go index 80d0026a1..ee2726160 100644 --- a/mfs/mfs_test.go +++ b/mfs/mfs_test.go @@ -792,6 +792,82 @@ func TestMfsModeAndModTime(t *testing.T) { } } +func TestMfsRawNodeSetModeAndMtime(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + _, rt := setupRoot(ctx, t) + rootdir := rt.GetDirectory() + + // Create raw-node file. + nd := dag.NewRawNode(random.Bytes(256)) + _, err := ft.ExtractFSNode(nd) + if !errors.Is(err, ft.ErrNotProtoNode) { + t.Fatal("Expected non-proto node") + } + + err = rootdir.AddChild("file", nd) + if err != nil { + t.Fatal(err) + } + + fsn, err := rootdir.Child("file") + if err != nil { + t.Fatal(err) + } + + fi := fsn.(*File) + if fi.Type() != TFile { + t.Fatal("something is seriously wrong here") + } + + // Check for expected error when getting mode and mtime. + _, err = fi.Mode() + if !errors.Is(err, ft.ErrNotProtoNode) { + t.Fatal("Expected non-proto node") + } + _, err = fi.ModTime() + if !errors.Is(err, ft.ErrNotProtoNode) { + t.Fatal("Expected non-proto node") + } + + // Set and check mode. + err = fi.SetMode(0644) + if err != nil { + t.Fatalf("failed to set file mode: %s", err) + } + mode, err := fi.Mode() + if err != nil { + t.Fatalf("failed to check file mode: %s", err) + } + if mode != 0644 { + t.Fatal("failed to get correct mode of file, got", mode.String()) + } + + // Mtime should still be unset. + mtime, err := fi.ModTime() + if err != nil { + t.Fatalf("failed to get file modification time: %s", err) + } + if !mtime.IsZero() { + t.Fatalf("expected mtime to be unset") + } + + // Set and check mtime. + now := time.Now() + err = fi.SetModTime(now) + if err != nil { + t.Fatalf("failed to set file modification time: %s", err) + } + mtime, err = fi.ModTime() + if err != nil { + t.Fatalf("failed to get file modification time: %s", err) + } + if !mtime.Equal(now) { + t.Fatal("failed to get correct modification time of file") + } +} + func TestMfsDirListNames(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() From a23fa5b4896e09df4d12d4abed5b32acf6e96146 Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Tue, 20 Aug 2024 15:51:57 -0700 Subject: [PATCH 16/18] Require go1.21 and use latest go-libp2p --- go.mod | 12 +++++------- go.sum | 17 ++++++++--------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 060e9c5db..a625a8b81 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/ipfs/boxo -go 1.22 +go 1.21 require ( github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 @@ -38,7 +38,7 @@ require ( github.com/jbenet/goprocess v0.1.4 github.com/libp2p/go-buffer-pool v0.1.0 github.com/libp2p/go-doh-resolver v0.4.0 - github.com/libp2p/go-libp2p v0.36.1 + github.com/libp2p/go-libp2p v0.36.2 github.com/libp2p/go-libp2p-kad-dht v0.25.2 github.com/libp2p/go-libp2p-record v0.2.0 github.com/libp2p/go-libp2p-routing-helpers v0.7.3 @@ -147,7 +147,7 @@ require ( github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect github.com/pion/datachannel v1.5.8 // indirect github.com/pion/dtls/v2 v2.2.12 // indirect - github.com/pion/ice/v2 v2.3.32 // indirect + github.com/pion/ice/v2 v2.3.34 // indirect github.com/pion/interceptor v0.1.29 // indirect github.com/pion/logging v0.2.2 // indirect github.com/pion/mdns v0.0.12 // indirect @@ -158,9 +158,9 @@ require ( github.com/pion/sdp/v3 v3.0.9 // indirect github.com/pion/srtp/v2 v2.0.20 // indirect github.com/pion/stun v0.6.1 // indirect - github.com/pion/transport/v2 v2.2.9 // indirect + github.com/pion/transport/v2 v2.2.10 // indirect github.com/pion/turn/v2 v2.1.6 // indirect - github.com/pion/webrtc/v3 v3.2.50 // indirect + github.com/pion/webrtc/v3 v3.3.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect @@ -195,5 +195,3 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.3.0 // indirect ) - -replace github.com/libp2p/go-libp2p => github.com/libp2p/go-libp2p v0.35.1-0.20240804142423-e2e0d2917f55 diff --git a/go.sum b/go.sum index 354e5378c..e539af643 100644 --- a/go.sum +++ b/go.sum @@ -274,8 +274,8 @@ github.com/libp2p/go-doh-resolver v0.4.0 h1:gUBa1f1XsPwtpE1du0O+nnZCUqtG7oYi7Bb+ github.com/libp2p/go-doh-resolver v0.4.0/go.mod h1:v1/jwsFusgsWIGX/c6vCRrnJ60x7bhTiq/fs2qt0cAg= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.35.1-0.20240804142423-e2e0d2917f55 h1:/iBsYYCzlVCiMMUfXWiHzgWpTFzZwes3cTlamdzXv6g= -github.com/libp2p/go-libp2p v0.35.1-0.20240804142423-e2e0d2917f55/go.mod h1:mdtNGqy0AQuiYJuO1bXPdFOyFeyMTMSVZ03OBi/XLS4= +github.com/libp2p/go-libp2p v0.36.2 h1:BbqRkDaGC3/5xfaJakLV/BrpjlAuYqSB0lRvtzL3B/U= +github.com/libp2p/go-libp2p v0.36.2/go.mod h1:XO3joasRE4Eup8yCTTP/+kX+g92mOgRaadk46LmPhHY= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= github.com/libp2p/go-libp2p-kad-dht v0.25.2 h1:FOIk9gHoe4YRWXTu8SY9Z1d0RILol0TrtApsMDPjAVQ= @@ -375,8 +375,8 @@ github.com/pion/datachannel v1.5.8/go.mod h1:PgmdpoaNBLX9HNzNClmdki4DYW5JtI7Yibu github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/dtls/v2 v2.2.12 h1:KP7H5/c1EiVAAKUmXyCzPiQe5+bCJrpOeKg/L05dunk= github.com/pion/dtls/v2 v2.2.12/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= -github.com/pion/ice/v2 v2.3.32 h1:VwE/uEeqiMm0zUWpdt1DJtnqEkj3UjEbhX92/CurtWI= -github.com/pion/ice/v2 v2.3.32/go.mod h1:8fac0+qftclGy1tYd/nfwfHC729BLaxtVqMdMVCAVPU= +github.com/pion/ice/v2 v2.3.34 h1:Ic1ppYCj4tUOcPAp76U6F3fVrlSw8A9JtRXLqw6BbUM= +github.com/pion/ice/v2 v2.3.34/go.mod h1:mBF7lnigdqgtB+YHkaY/Y6s6tsyRyo4u4rPGRuOjUBQ= github.com/pion/interceptor v0.1.29 h1:39fsnlP1U8gw2JzOFWdfCU82vHvhW9o0rZnZF56wF+M= github.com/pion/interceptor v0.1.29/go.mod h1:ri+LGNjRUc5xUNtDEPzfdkmSqISixVTBF/z/Zms/6T4= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= @@ -402,17 +402,16 @@ github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/ github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= github.com/pion/transport/v2 v2.2.3/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= -github.com/pion/transport/v2 v2.2.8/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E= -github.com/pion/transport/v2 v2.2.9 h1:WEDygVovkJlV2CCunM9KS2kds+kcl7zdIefQA5y/nkE= -github.com/pion/transport/v2 v2.2.9/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E= +github.com/pion/transport/v2 v2.2.10 h1:ucLBLE8nuxiHfvkFKnkDQRYWYfp8ejf4YBOPfaQpw6Q= +github.com/pion/transport/v2 v2.2.10/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E= github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= github.com/pion/transport/v3 v3.0.6 h1:k1mQU06bmmX143qSWgXFqSH1KUJceQvIUuVH/K5ELWw= github.com/pion/transport/v3 v3.0.6/go.mod h1:HvJr2N/JwNJAfipsRleqwFoR3t/pWyHeZUs89v3+t5s= github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc= github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= -github.com/pion/webrtc/v3 v3.2.50 h1:C/rwL2mBfCxHv6tlLzDAO3krJpQXfVx8A8WHnGJ2j34= -github.com/pion/webrtc/v3 v3.2.50/go.mod h1:dytYYoSBy7ZUWhJMbndx9UckgYvzNAfL7xgVnrIKxqo= +github.com/pion/webrtc/v3 v3.3.0 h1:Rf4u6n6U5t5sUxhYPQk/samzU/oDv7jk6BA5hyO2F9I= +github.com/pion/webrtc/v3 v3.3.0/go.mod h1:hVmrDJvwhEertRWObeb1xzulzHGeVUoPlWvxdGzcfU0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= From 95a5c3093271d8b46cb9713cc8ca73c30c1564bb Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Tue, 20 Aug 2024 15:58:32 -0700 Subject: [PATCH 17/18] mod tidy examples --- examples/go.mod | 8 ++++---- examples/go.sum | 17 ++++++++--------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/examples/go.mod b/examples/go.mod index fc94d2519..48f786a59 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -9,7 +9,7 @@ require ( github.com/ipfs/go-datastore v0.6.0 github.com/ipld/go-car/v2 v2.13.1 github.com/ipld/go-ipld-prime v0.21.0 - github.com/libp2p/go-libp2p v0.36.1 + github.com/libp2p/go-libp2p v0.36.2 github.com/libp2p/go-libp2p-routing-helpers v0.7.3 github.com/multiformats/go-multiaddr v0.13.0 github.com/multiformats/go-multicodec v0.9.0 @@ -122,7 +122,7 @@ require ( github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect github.com/pion/datachannel v1.5.8 // indirect github.com/pion/dtls/v2 v2.2.12 // indirect - github.com/pion/ice/v2 v2.3.32 // indirect + github.com/pion/ice/v2 v2.3.34 // indirect github.com/pion/interceptor v0.1.29 // indirect github.com/pion/logging v0.2.2 // indirect github.com/pion/mdns v0.0.12 // indirect @@ -133,9 +133,9 @@ require ( github.com/pion/sdp/v3 v3.0.9 // indirect github.com/pion/srtp/v2 v2.0.20 // indirect github.com/pion/stun v0.6.1 // indirect - github.com/pion/transport/v2 v2.2.9 // indirect + github.com/pion/transport/v2 v2.2.10 // indirect github.com/pion/turn/v2 v2.1.6 // indirect - github.com/pion/webrtc/v3 v3.2.50 // indirect + github.com/pion/webrtc/v3 v3.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.89.0 // indirect diff --git a/examples/go.sum b/examples/go.sum index 67b7b18ea..e247582dd 100644 --- a/examples/go.sum +++ b/examples/go.sum @@ -271,8 +271,8 @@ github.com/libp2p/go-doh-resolver v0.4.0 h1:gUBa1f1XsPwtpE1du0O+nnZCUqtG7oYi7Bb+ github.com/libp2p/go-doh-resolver v0.4.0/go.mod h1:v1/jwsFusgsWIGX/c6vCRrnJ60x7bhTiq/fs2qt0cAg= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.36.1 h1:piAHesy0/8ifBEBUS8HF2m7ywR5vnktUFv00dTsVKcs= -github.com/libp2p/go-libp2p v0.36.1/go.mod h1:vHzel3CpRB+vS11fIjZSJAU4ALvieKV9VZHC9VerHj8= +github.com/libp2p/go-libp2p v0.36.2 h1:BbqRkDaGC3/5xfaJakLV/BrpjlAuYqSB0lRvtzL3B/U= +github.com/libp2p/go-libp2p v0.36.2/go.mod h1:XO3joasRE4Eup8yCTTP/+kX+g92mOgRaadk46LmPhHY= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= github.com/libp2p/go-libp2p-kad-dht v0.25.2 h1:FOIk9gHoe4YRWXTu8SY9Z1d0RILol0TrtApsMDPjAVQ= @@ -372,8 +372,8 @@ github.com/pion/datachannel v1.5.8/go.mod h1:PgmdpoaNBLX9HNzNClmdki4DYW5JtI7Yibu github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/dtls/v2 v2.2.12 h1:KP7H5/c1EiVAAKUmXyCzPiQe5+bCJrpOeKg/L05dunk= github.com/pion/dtls/v2 v2.2.12/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= -github.com/pion/ice/v2 v2.3.32 h1:VwE/uEeqiMm0zUWpdt1DJtnqEkj3UjEbhX92/CurtWI= -github.com/pion/ice/v2 v2.3.32/go.mod h1:8fac0+qftclGy1tYd/nfwfHC729BLaxtVqMdMVCAVPU= +github.com/pion/ice/v2 v2.3.34 h1:Ic1ppYCj4tUOcPAp76U6F3fVrlSw8A9JtRXLqw6BbUM= +github.com/pion/ice/v2 v2.3.34/go.mod h1:mBF7lnigdqgtB+YHkaY/Y6s6tsyRyo4u4rPGRuOjUBQ= github.com/pion/interceptor v0.1.29 h1:39fsnlP1U8gw2JzOFWdfCU82vHvhW9o0rZnZF56wF+M= github.com/pion/interceptor v0.1.29/go.mod h1:ri+LGNjRUc5xUNtDEPzfdkmSqISixVTBF/z/Zms/6T4= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= @@ -399,17 +399,16 @@ github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/ github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= github.com/pion/transport/v2 v2.2.3/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= -github.com/pion/transport/v2 v2.2.8/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E= -github.com/pion/transport/v2 v2.2.9 h1:WEDygVovkJlV2CCunM9KS2kds+kcl7zdIefQA5y/nkE= -github.com/pion/transport/v2 v2.2.9/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E= +github.com/pion/transport/v2 v2.2.10 h1:ucLBLE8nuxiHfvkFKnkDQRYWYfp8ejf4YBOPfaQpw6Q= +github.com/pion/transport/v2 v2.2.10/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E= github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= github.com/pion/transport/v3 v3.0.6 h1:k1mQU06bmmX143qSWgXFqSH1KUJceQvIUuVH/K5ELWw= github.com/pion/transport/v3 v3.0.6/go.mod h1:HvJr2N/JwNJAfipsRleqwFoR3t/pWyHeZUs89v3+t5s= github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc= github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= -github.com/pion/webrtc/v3 v3.2.50 h1:C/rwL2mBfCxHv6tlLzDAO3krJpQXfVx8A8WHnGJ2j34= -github.com/pion/webrtc/v3 v3.2.50/go.mod h1:dytYYoSBy7ZUWhJMbndx9UckgYvzNAfL7xgVnrIKxqo= +github.com/pion/webrtc/v3 v3.3.0 h1:Rf4u6n6U5t5sUxhYPQk/samzU/oDv7jk6BA5hyO2F9I= +github.com/pion/webrtc/v3 v3.3.0/go.mod h1:hVmrDJvwhEertRWObeb1xzulzHGeVUoPlWvxdGzcfU0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= From b2660c5c452246fde0e3ce5833a087cdcf8364b3 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 21 Aug 2024 01:08:09 +0200 Subject: [PATCH 18/18] chore: align examples with main go.mod --- examples/go.mod | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/go.mod b/examples/go.mod index 48f786a59..40d3c3696 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -1,9 +1,9 @@ module github.com/ipfs/boxo/examples -go 1.22 +go 1.21 require ( - github.com/ipfs/boxo v0.19.0 + github.com/ipfs/boxo v0.22.0 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0