From 2d3f8dc6068fa65fef9597cf7d73adc067b5f7e8 Mon Sep 17 00:00:00 2001 From: Jan Winkelmann Date: Sun, 16 Apr 2017 18:10:49 +0200 Subject: [PATCH 01/76] add files/ - was previously in go-ipfs-cmds This commit was moved from ipfs/go-ipfs-files@8cd78b708a823d6402fa272a42463e82ee2517b7 --- files/file.go | 62 +++++++++++ files/file_test.go | 203 +++++++++++++++++++++++++++++++++++++ files/is_hidden.go | 19 ++++ files/is_hidden_windows.go | 29 ++++++ files/linkfile.go | 50 +++++++++ files/multipartfile.go | 116 +++++++++++++++++++++ files/readerfile.go | 70 +++++++++++++ files/serialfile.go | 146 ++++++++++++++++++++++++++ files/slicefile.go | 76 ++++++++++++++ 9 files changed, 771 insertions(+) create mode 100644 files/file.go create mode 100644 files/file_test.go create mode 100644 files/is_hidden.go create mode 100644 files/is_hidden_windows.go create mode 100644 files/linkfile.go create mode 100644 files/multipartfile.go create mode 100644 files/readerfile.go create mode 100644 files/serialfile.go create mode 100644 files/slicefile.go diff --git a/files/file.go b/files/file.go new file mode 100644 index 000000000..c5e820336 --- /dev/null +++ b/files/file.go @@ -0,0 +1,62 @@ +package files + +import ( + "errors" + "io" + "os" +) + +var ( + ErrNotDirectory = errors.New("Couldn't call NextFile(), this isn't a directory") + ErrNotReader = errors.New("This file is a directory, can't use Reader functions") +) + +// File is an interface that provides functionality for handling +// files/directories as values that can be supplied to commands. For +// directories, child files are accessed serially by calling `NextFile()`. +type File interface { + // Files implement ReadCloser, but can only be read from or closed if + // they are not directories + io.ReadCloser + + // FileName returns a filename associated with this file + FileName() string + + // FullPath returns the full path used when adding with this file + FullPath() string + + // IsDirectory returns true if the File is a directory (and therefore + // supports calling `NextFile`) and false if the File is a normal file + // (and therefor supports calling `Read` and `Close`) + IsDirectory() bool + + // NextFile returns the next child file available (if the File is a + // directory). It will return (nil, io.EOF) if no more files are + // available. If the file is a regular file (not a directory), NextFile + // will return a non-nil error. + NextFile() (File, error) +} + +type StatFile interface { + File + + Stat() os.FileInfo +} + +type PeekFile interface { + SizeFile + + Peek(n int) File + Length() int +} + +type SizeFile interface { + File + + Size() (int64, error) +} + +type FileInfo interface { + AbsPath() string + Stat() os.FileInfo +} diff --git a/files/file_test.go b/files/file_test.go new file mode 100644 index 000000000..a5d60102f --- /dev/null +++ b/files/file_test.go @@ -0,0 +1,203 @@ +package files + +import ( + "io" + "io/ioutil" + "mime/multipart" + "strings" + "testing" +) + +func TestSliceFiles(t *testing.T) { + name := "testname" + files := []File{ + NewReaderFile("file.txt", "file.txt", ioutil.NopCloser(strings.NewReader("Some text!\n")), nil), + NewReaderFile("beep.txt", "beep.txt", ioutil.NopCloser(strings.NewReader("beep")), nil), + NewReaderFile("boop.txt", "boop.txt", ioutil.NopCloser(strings.NewReader("boop")), nil), + } + buf := make([]byte, 20) + + sf := NewSliceFile(name, name, files) + + if !sf.IsDirectory() { + t.Fatal("SliceFile should always be a directory") + } + + if n, err := sf.Read(buf); n > 0 || err != io.EOF { + t.Fatal("Shouldn't be able to read data from a SliceFile") + } + + if err := sf.Close(); err != ErrNotReader { + t.Fatal("Shouldn't be able to call `Close` on a SliceFile") + } + + file, err := sf.NextFile() + if file == nil || err != nil { + t.Fatal("Expected a file and nil error") + } + read, err := file.Read(buf) + if read != 11 || err != nil { + t.Fatal("NextFile got a file in the wrong order") + } + + file, err = sf.NextFile() + if file == nil || err != nil { + t.Fatal("Expected a file and nil error") + } + file, err = sf.NextFile() + if file == nil || err != nil { + t.Fatal("Expected a file and nil error") + } + + file, err = sf.NextFile() + if file != nil || err != io.EOF { + t.Fatal("Expected a nil file and io.EOF") + } +} + +func TestReaderFiles(t *testing.T) { + message := "beep boop" + rf := NewReaderFile("file.txt", "file.txt", ioutil.NopCloser(strings.NewReader(message)), nil) + buf := make([]byte, len(message)) + + if rf.IsDirectory() { + t.Fatal("ReaderFile should never be a directory") + } + file, err := rf.NextFile() + if file != nil || err != ErrNotDirectory { + t.Fatal("Expected a nil file and ErrNotDirectory") + } + + if n, err := rf.Read(buf); n == 0 || err != nil { + t.Fatal("Expected to be able to read") + } + if err := rf.Close(); err != nil { + t.Fatal("Should be able to close") + } + if n, err := rf.Read(buf); n != 0 || err != io.EOF { + t.Fatal("Expected EOF when reading after close") + } +} + +func TestMultipartFiles(t *testing.T) { + data := ` +--Boundary! +Content-Type: text/plain +Content-Disposition: file; filename="name" +Some-Header: beep + +beep +--Boundary! +Content-Type: application/x-directory +Content-Disposition: file; filename="dir" + +--Boundary! +Content-Type: text/plain +Content-Disposition: file; filename="dir/nested" + +some content +--Boundary! +Content-Type: application/symlink +Content-Disposition: file; filename="dir/simlynk" + +anotherfile +--Boundary!-- + +` + + reader := strings.NewReader(data) + mpReader := multipart.NewReader(reader, "Boundary!") + buf := make([]byte, 20) + + // test properties of a file created from the first part + part, err := mpReader.NextPart() + if part == nil || err != nil { + t.Fatal("Expected non-nil part, nil error") + } + mpf, err := NewFileFromPart(part) + if mpf == nil || err != nil { + t.Fatal("Expected non-nil MultipartFile, nil error") + } + if mpf.IsDirectory() { + t.Fatal("Expected file to not be a directory") + } + if mpf.FileName() != "name" { + t.Fatal("Expected filename to be \"name\"") + } + if file, err := mpf.NextFile(); file != nil || err != ErrNotDirectory { + t.Fatal("Expected a nil file and ErrNotDirectory") + } + if n, err := mpf.Read(buf); n != 4 || err != nil { + t.Fatal("Expected to be able to read 4 bytes") + } + if err := mpf.Close(); err != nil { + t.Fatal("Expected to be able to close file") + } + + // test properties of file created from second part (directory) + part, err = mpReader.NextPart() + if part == nil || err != nil { + t.Fatal("Expected non-nil part, nil error") + } + mpf, err = NewFileFromPart(part) + if mpf == nil || err != nil { + t.Fatal("Expected non-nil MultipartFile, nil error") + } + if !mpf.IsDirectory() { + t.Fatal("Expected file to be a directory") + } + if mpf.FileName() != "dir" { + t.Fatal("Expected filename to be \"dir\"") + } + if n, err := mpf.Read(buf); n > 0 || err != ErrNotReader { + t.Fatal("Shouldn't be able to call `Read` on a directory") + } + if err := mpf.Close(); err != ErrNotReader { + t.Fatal("Shouldn't be able to call `Close` on a directory") + } + + // test properties of file created from third part (nested file) + part, err = mpReader.NextPart() + if part == nil || err != nil { + t.Fatal("Expected non-nil part, nil error") + } + mpf, err = NewFileFromPart(part) + if mpf == nil || err != nil { + t.Fatal("Expected non-nil MultipartFile, nil error") + } + if mpf.IsDirectory() { + t.Fatal("Expected file, got directory") + } + if mpf.FileName() != "dir/nested" { + t.Fatalf("Expected filename to be \"nested\", got %s", mpf.FileName()) + } + if n, err := mpf.Read(buf); n != 12 || err != nil { + t.Fatalf("expected to be able to read 12 bytes from file: %s (got %d)", err, n) + } + if err := mpf.Close(); err != nil { + t.Fatalf("should be able to close file: %s", err) + } + + // test properties of symlink created from fourth part (symlink) + part, err = mpReader.NextPart() + if part == nil || err != nil { + t.Fatal("Expected non-nil part, nil error") + } + mpf, err = NewFileFromPart(part) + if mpf == nil || err != nil { + t.Fatal("Expected non-nil MultipartFile, nil error") + } + if mpf.IsDirectory() { + t.Fatal("Expected file to be a symlink") + } + if mpf.FileName() != "dir/simlynk" { + t.Fatal("Expected filename to be \"dir/simlynk\"") + } + slink, ok := mpf.(*Symlink) + if !ok { + t.Fatalf("expected file to be a symlink") + } + if slink.Target != "anotherfile" { + t.Fatal("expected link to point to anotherfile") + } +} diff --git a/files/is_hidden.go b/files/is_hidden.go new file mode 100644 index 000000000..b0360685b --- /dev/null +++ b/files/is_hidden.go @@ -0,0 +1,19 @@ +// +build !windows + +package files + +import ( + "path/filepath" + "strings" +) + +func IsHidden(f File) bool { + + fName := filepath.Base(f.FileName()) + + if strings.HasPrefix(fName, ".") && len(fName) > 1 { + return true + } + + return false +} diff --git a/files/is_hidden_windows.go b/files/is_hidden_windows.go new file mode 100644 index 000000000..5d2639310 --- /dev/null +++ b/files/is_hidden_windows.go @@ -0,0 +1,29 @@ +// +build windows + +package files + +import ( + "path/filepath" + "strings" + "syscall" +) + +func IsHidden(f File) bool { + + fName := filepath.Base(f.FileName()) + + if strings.HasPrefix(fName, ".") && len(fName) > 1 { + return true + } + + p, e := syscall.UTF16PtrFromString(f.FileName()) + if e != nil { + return false + } + + attrs, e := syscall.GetFileAttributes(p) + if e != nil { + return false + } + return attrs&syscall.FILE_ATTRIBUTE_HIDDEN != 0 +} diff --git a/files/linkfile.go b/files/linkfile.go new file mode 100644 index 000000000..18466f4bd --- /dev/null +++ b/files/linkfile.go @@ -0,0 +1,50 @@ +package files + +import ( + "io" + "os" + "strings" +) + +type Symlink struct { + name string + path string + Target string + stat os.FileInfo + + reader io.Reader +} + +func NewLinkFile(name, path, target string, stat os.FileInfo) File { + return &Symlink{ + name: name, + path: path, + Target: target, + stat: stat, + reader: strings.NewReader(target), + } +} + +func (lf *Symlink) IsDirectory() bool { + return false +} + +func (lf *Symlink) NextFile() (File, error) { + return nil, io.EOF +} + +func (f *Symlink) FileName() string { + return f.name +} + +func (f *Symlink) Close() error { + return nil +} + +func (f *Symlink) FullPath() string { + return f.path +} + +func (f *Symlink) Read(b []byte) (int, error) { + return f.reader.Read(b) +} diff --git a/files/multipartfile.go b/files/multipartfile.go new file mode 100644 index 000000000..21e0d44c1 --- /dev/null +++ b/files/multipartfile.go @@ -0,0 +1,116 @@ +package files + +import ( + "io" + "io/ioutil" + "mime" + "mime/multipart" + "net/url" +) + +const ( + multipartFormdataType = "multipart/form-data" + multipartMixedType = "multipart/mixed" + + applicationDirectory = "application/x-directory" + applicationSymlink = "application/symlink" + applicationFile = "application/octet-stream" + + contentTypeHeader = "Content-Type" +) + +// MultipartFile implements File, and is created from a `multipart.Part`. +// It can be either a directory or file (checked by calling `IsDirectory()`). +type MultipartFile struct { + File + + Part *multipart.Part + Reader *multipart.Reader + Mediatype string +} + +func NewFileFromPart(part *multipart.Part) (File, error) { + f := &MultipartFile{ + Part: part, + } + + contentType := part.Header.Get(contentTypeHeader) + switch contentType { + case applicationSymlink: + out, err := ioutil.ReadAll(part) + if err != nil { + return nil, err + } + + return &Symlink{ + Target: string(out), + name: f.FileName(), + }, nil + case applicationFile: + return &ReaderFile{ + reader: part, + filename: f.FileName(), + abspath: part.Header.Get("abspath"), + fullpath: f.FullPath(), + }, nil + } + + var err error + f.Mediatype, _, err = mime.ParseMediaType(contentType) + if err != nil { + return nil, err + } + + return f, nil +} + +func (f *MultipartFile) IsDirectory() bool { + return f.Mediatype == multipartFormdataType || f.Mediatype == applicationDirectory +} + +func (f *MultipartFile) NextFile() (File, error) { + if !f.IsDirectory() { + return nil, ErrNotDirectory + } + if f.Reader != nil { + part, err := f.Reader.NextPart() + if err != nil { + return nil, err + } + + return NewFileFromPart(part) + } + + return nil, io.EOF +} + +func (f *MultipartFile) FileName() string { + if f == nil || f.Part == nil { + return "" + } + + filename, err := url.QueryUnescape(f.Part.FileName()) + if err != nil { + // if there is a unescape error, just treat the name as unescaped + return f.Part.FileName() + } + return filename +} + +func (f *MultipartFile) FullPath() string { + return f.FileName() +} + +func (f *MultipartFile) Read(p []byte) (int, error) { + if f.IsDirectory() { + return 0, ErrNotReader + } + return f.Part.Read(p) +} + +func (f *MultipartFile) Close() error { + if f.IsDirectory() { + return ErrNotReader + } + return f.Part.Close() +} diff --git a/files/readerfile.go b/files/readerfile.go new file mode 100644 index 000000000..863641479 --- /dev/null +++ b/files/readerfile.go @@ -0,0 +1,70 @@ +package files + +import ( + "errors" + "io" + "os" + "path/filepath" +) + +// ReaderFile is a implementation of File created from an `io.Reader`. +// ReaderFiles are never directories, and can be read from and closed. +type ReaderFile struct { + filename string + fullpath string + abspath string + reader io.ReadCloser + stat os.FileInfo +} + +func NewReaderFile(filename, path string, reader io.ReadCloser, stat os.FileInfo) *ReaderFile { + return &ReaderFile{filename, path, path, reader, stat} +} + +func NewReaderPathFile(filename, path string, reader io.ReadCloser, stat os.FileInfo) (*ReaderFile, error) { + abspath, err := filepath.Abs(path) + if err != nil { + return nil, err + } + + return &ReaderFile{filename, path, abspath, reader, stat}, nil +} + +func (f *ReaderFile) IsDirectory() bool { + return false +} + +func (f *ReaderFile) NextFile() (File, error) { + return nil, ErrNotDirectory +} + +func (f *ReaderFile) FileName() string { + return f.filename +} + +func (f *ReaderFile) FullPath() string { + return f.fullpath +} + +func (f *ReaderFile) AbsPath() string { + return f.abspath +} + +func (f *ReaderFile) Read(p []byte) (int, error) { + return f.reader.Read(p) +} + +func (f *ReaderFile) Close() error { + return f.reader.Close() +} + +func (f *ReaderFile) Stat() os.FileInfo { + return f.stat +} + +func (f *ReaderFile) Size() (int64, error) { + if f.stat == nil { + return 0, errors.New("File size unknown") + } + return f.stat.Size(), nil +} diff --git a/files/serialfile.go b/files/serialfile.go new file mode 100644 index 000000000..2fe35bee6 --- /dev/null +++ b/files/serialfile.go @@ -0,0 +1,146 @@ +package files + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + "syscall" +) + +// serialFile implements File, and reads from a path on the OS filesystem. +// No more than one file will be opened at a time (directories will advance +// to the next file when NextFile() is called). +type serialFile struct { + name string + path string + files []os.FileInfo + stat os.FileInfo + current *File + handleHiddenFiles bool +} + +func NewSerialFile(name, path string, hidden bool, stat os.FileInfo) (File, error) { + switch mode := stat.Mode(); { + case mode.IsRegular(): + file, err := os.Open(path) + if err != nil { + return nil, err + } + return NewReaderPathFile(name, path, file, stat) + case mode.IsDir(): + // for directories, stat all of the contents first, so we know what files to + // open when NextFile() is called + contents, err := ioutil.ReadDir(path) + if err != nil { + return nil, err + } + return &serialFile{name, path, contents, stat, nil, hidden}, nil + case mode&os.ModeSymlink != 0: + target, err := os.Readlink(path) + if err != nil { + return nil, err + } + return NewLinkFile(name, path, target, stat), nil + default: + return nil, fmt.Errorf("Unrecognized file type for %s: %s", name, mode.String()) + } +} + +func (f *serialFile) IsDirectory() bool { + // non-directories get created as a ReaderFile, so serialFiles should only + // represent directories + return true +} + +func (f *serialFile) NextFile() (File, error) { + // if a file was opened previously, close it + err := f.Close() + if err != nil { + return nil, err + } + + // if there aren't any files left in the root directory, we're done + if len(f.files) == 0 { + return nil, io.EOF + } + + stat := f.files[0] + f.files = f.files[1:] + + for !f.handleHiddenFiles && strings.HasPrefix(stat.Name(), ".") { + if len(f.files) == 0 { + return nil, io.EOF + } + + stat = f.files[0] + f.files = f.files[1:] + } + + // open the next file + fileName := filepath.ToSlash(filepath.Join(f.name, stat.Name())) + filePath := filepath.ToSlash(filepath.Join(f.path, stat.Name())) + + // recursively call the constructor on the next file + // if it's a regular file, we will open it as a ReaderFile + // if it's a directory, files in it will be opened serially + sf, err := NewSerialFile(fileName, filePath, f.handleHiddenFiles, stat) + if err != nil { + return nil, err + } + + f.current = &sf + + return sf, nil +} + +func (f *serialFile) FileName() string { + return f.name +} + +func (f *serialFile) FullPath() string { + return f.path +} + +func (f *serialFile) Read(p []byte) (int, error) { + return 0, io.EOF +} + +func (f *serialFile) Close() error { + // close the current file if there is one + if f.current != nil { + err := (*f.current).Close() + // ignore EINVAL error, the file might have already been closed + if err != nil && err != syscall.EINVAL { + return err + } + } + + return nil +} + +func (f *serialFile) Stat() os.FileInfo { + return f.stat +} + +func (f *serialFile) Size() (int64, error) { + if !f.stat.IsDir() { + return f.stat.Size(), nil + } + + var du int64 + err := filepath.Walk(f.FullPath(), func(p string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + + if fi != nil && fi.Mode()&(os.ModeSymlink|os.ModeNamedPipe) == 0 { + du += fi.Size() + } + return nil + }) + + return du, err +} diff --git a/files/slicefile.go b/files/slicefile.go new file mode 100644 index 000000000..8d18dcaa3 --- /dev/null +++ b/files/slicefile.go @@ -0,0 +1,76 @@ +package files + +import ( + "errors" + "io" +) + +// SliceFile implements File, and provides simple directory handling. +// It contains children files, and is created from a `[]File`. +// SliceFiles are always directories, and can't be read from or closed. +type SliceFile struct { + filename string + path string + files []File + n int +} + +func NewSliceFile(filename, path string, files []File) *SliceFile { + return &SliceFile{filename, path, files, 0} +} + +func (f *SliceFile) IsDirectory() bool { + return true +} + +func (f *SliceFile) NextFile() (File, error) { + if f.n >= len(f.files) { + return nil, io.EOF + } + file := f.files[f.n] + f.n++ + return file, nil +} + +func (f *SliceFile) FileName() string { + return f.filename +} + +func (f *SliceFile) FullPath() string { + return f.path +} + +func (f *SliceFile) Read(p []byte) (int, error) { + return 0, io.EOF +} + +func (f *SliceFile) Close() error { + return ErrNotReader +} + +func (f *SliceFile) Peek(n int) File { + return f.files[n] +} + +func (f *SliceFile) Length() int { + return len(f.files) +} + +func (f *SliceFile) Size() (int64, error) { + var size int64 + + for _, file := range f.files { + sizeFile, ok := file.(SizeFile) + if !ok { + return 0, errors.New("Could not get size of child file") + } + + s, err := sizeFile.Size() + if err != nil { + return 0, err + } + size += s + } + + return size, nil +} From 088a78565824471234c176879e088133d0ff4c72 Mon Sep 17 00:00:00 2001 From: Jan Winkelmann Date: Wed, 26 Apr 2017 15:46:59 +0200 Subject: [PATCH 02/76] make tests also except EOF error This commit was moved from ipfs/go-ipfs-files@80cbd2e7007aa988d8d942ca8230a7f18df6405e --- files/file_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/files/file_test.go b/files/file_test.go index a5d60102f..733a8c87d 100644 --- a/files/file_test.go +++ b/files/file_test.go @@ -127,8 +127,8 @@ anotherfile if file, err := mpf.NextFile(); file != nil || err != ErrNotDirectory { t.Fatal("Expected a nil file and ErrNotDirectory") } - if n, err := mpf.Read(buf); n != 4 || err != nil { - t.Fatal("Expected to be able to read 4 bytes") + if n, err := mpf.Read(buf); n != 4 || !(err == io.EOF || err == nil) { + t.Fatal("Expected to be able to read 4 bytes", n, err) } if err := mpf.Close(); err != nil { t.Fatal("Expected to be able to close file") @@ -171,7 +171,7 @@ anotherfile if mpf.FileName() != "dir/nested" { t.Fatalf("Expected filename to be \"nested\", got %s", mpf.FileName()) } - if n, err := mpf.Read(buf); n != 12 || err != nil { + if n, err := mpf.Read(buf); n != 12 || !(err == nil || err == io.EOF) { t.Fatalf("expected to be able to read 12 bytes from file: %s (got %d)", err, n) } if err := mpf.Close(); err != nil { From e49369757e192a9ca249ddea47a29087e39d8e87 Mon Sep 17 00:00:00 2001 From: keks Date: Sun, 15 Oct 2017 14:07:03 +0200 Subject: [PATCH 03/76] fix 'file already closed' in Go 1.9 This commit was moved from ipfs/go-ipfs-files@2538708dfb6c4b3e02bac958b2dc2854f30680c0 --- files/serialfile.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/files/serialfile.go b/files/serialfile.go index 2fe35bee6..611d3f1db 100644 --- a/files/serialfile.go +++ b/files/serialfile.go @@ -59,7 +59,14 @@ func (f *serialFile) NextFile() (File, error) { // if a file was opened previously, close it err := f.Close() if err != nil { - return nil, err + switch err2 := err.(type) { + case *os.PathError: + if err2.Err != os.ErrClosed { + return nil, err + } + default: + return nil, err + } } // if there aren't any files left in the root directory, we're done From b407a665f0840cce8a193477a415b95807478e6a Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Sat, 18 Nov 2017 12:06:40 -0800 Subject: [PATCH 04/76] use goang.org/x/sys/windows syscall is mostly deprecated This commit was moved from ipfs/go-ipfs-files@845964d0ad1f5ebb92d8c3e27f019e34e813ee40 --- files/is_hidden_windows.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/files/is_hidden_windows.go b/files/is_hidden_windows.go index 5d2639310..7679433df 100644 --- a/files/is_hidden_windows.go +++ b/files/is_hidden_windows.go @@ -5,7 +5,8 @@ package files import ( "path/filepath" "strings" - "syscall" + + windows "golang.org/x/sys/windows" ) func IsHidden(f File) bool { @@ -16,14 +17,14 @@ func IsHidden(f File) bool { return true } - p, e := syscall.UTF16PtrFromString(f.FileName()) + p, e := windows.UTF16PtrFromString(f.FullPath()) if e != nil { return false } - attrs, e := syscall.GetFileAttributes(p) + attrs, e := windows.GetFileAttributes(p) if e != nil { return false } - return attrs&syscall.FILE_ATTRIBUTE_HIDDEN != 0 + return attrs&windows.FILE_ATTRIBUTE_HIDDEN != 0 } From 789420576751e1d4dae2adbbdb66e8e89e783613 Mon Sep 17 00:00:00 2001 From: Lars Gierth Date: Sun, 19 Nov 2017 23:55:08 +0100 Subject: [PATCH 05/76] Import MultiFileReader from go-ipfs-cmds This commit was moved from ipfs/go-ipfs-files@648e53bf5e061aad42149c013f69777fac5b5501 --- files/multifilereader.go | 124 ++++++++++++++++++++++++++++++++++ files/multifilereader_test.go | 112 ++++++++++++++++++++++++++++++ 2 files changed, 236 insertions(+) create mode 100644 files/multifilereader.go create mode 100644 files/multifilereader_test.go diff --git a/files/multifilereader.go b/files/multifilereader.go new file mode 100644 index 000000000..4833e8d18 --- /dev/null +++ b/files/multifilereader.go @@ -0,0 +1,124 @@ +package files + +import ( + "bytes" + "fmt" + "io" + "mime/multipart" + "net/textproto" + "net/url" + "sync" +) + +// MultiFileReader reads from a `commands.File` (which can be a directory of files +// or a regular file) as HTTP multipart encoded data. +type MultiFileReader struct { + io.Reader + + files []File + currentFile io.Reader + buf bytes.Buffer + mpWriter *multipart.Writer + closed bool + mutex *sync.Mutex + + // if true, the data will be type 'multipart/form-data' + // if false, the data will be type 'multipart/mixed' + form bool +} + +// NewMultiFileReader constructs a MultiFileReader. `file` can be any `commands.File`. +// If `form` is set to true, the multipart data will have a Content-Type of 'multipart/form-data', +// if `form` is false, the Content-Type will be 'multipart/mixed'. +func NewMultiFileReader(file File, form bool) *MultiFileReader { + mfr := &MultiFileReader{ + files: []File{file}, + form: form, + mutex: &sync.Mutex{}, + } + mfr.mpWriter = multipart.NewWriter(&mfr.buf) + + return mfr +} + +func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { + mfr.mutex.Lock() + defer mfr.mutex.Unlock() + + // if we are closed and the buffer is flushed, end reading + if mfr.closed && mfr.buf.Len() == 0 { + return 0, io.EOF + } + + // if the current file isn't set, advance to the next file + if mfr.currentFile == nil { + var file File + for file == nil { + if len(mfr.files) == 0 { + mfr.mpWriter.Close() + mfr.closed = true + return mfr.buf.Read(buf) + } + + nextfile, err := mfr.files[len(mfr.files)-1].NextFile() + if err == io.EOF { + mfr.files = mfr.files[:len(mfr.files)-1] + continue + } else if err != nil { + return 0, err + } + + file = nextfile + } + + // handle starting a new file part + if !mfr.closed { + + var contentType string + if _, ok := file.(*Symlink); ok { + contentType = "application/symlink" + } else if file.IsDirectory() { + mfr.files = append(mfr.files, file) + contentType = "application/x-directory" + } else { + // otherwise, use the file as a reader to read its contents + contentType = "application/octet-stream" + } + + mfr.currentFile = file + + // write the boundary and headers + header := make(textproto.MIMEHeader) + filename := url.QueryEscape(file.FileName()) + header.Set("Content-Disposition", fmt.Sprintf("file; filename=\"%s\"", filename)) + + header.Set("Content-Type", contentType) + if rf, ok := file.(*ReaderFile); ok { + header.Set("abspath", rf.AbsPath()) + } + + _, err := mfr.mpWriter.CreatePart(header) + if err != nil { + return 0, err + } + } + } + + // if the buffer has something in it, read from it + if mfr.buf.Len() > 0 { + return mfr.buf.Read(buf) + } + + // otherwise, read from file data + written, err = mfr.currentFile.Read(buf) + if err == io.EOF { + mfr.currentFile = nil + return written, nil + } + return written, err +} + +// Boundary returns the boundary string to be used to separate files in the multipart data +func (mfr *MultiFileReader) Boundary() string { + return mfr.mpWriter.Boundary() +} diff --git a/files/multifilereader_test.go b/files/multifilereader_test.go new file mode 100644 index 000000000..3d2c97892 --- /dev/null +++ b/files/multifilereader_test.go @@ -0,0 +1,112 @@ +package files + +import ( + "io" + "io/ioutil" + "mime/multipart" + "strings" + "testing" +) + +func TestOutput(t *testing.T) { + text := "Some text! :)" + fileset := []File{ + NewReaderFile("file.txt", "file.txt", ioutil.NopCloser(strings.NewReader(text)), nil), + NewSliceFile("boop", "boop", []File{ + NewReaderFile("boop/a.txt", "boop/a.txt", ioutil.NopCloser(strings.NewReader("bleep")), nil), + NewReaderFile("boop/b.txt", "boop/b.txt", ioutil.NopCloser(strings.NewReader("bloop")), nil), + }), + NewReaderFile("beep.txt", "beep.txt", ioutil.NopCloser(strings.NewReader("beep")), nil), + } + sf := NewSliceFile("", "", fileset) + buf := make([]byte, 20) + + // testing output by reading it with the go stdlib "mime/multipart" Reader + mfr := NewMultiFileReader(sf, true) + mpReader := multipart.NewReader(mfr, mfr.Boundary()) + + part, err := mpReader.NextPart() + if part == nil || err != nil { + t.Fatal("Expected non-nil part, nil error") + } + mpf, err := NewFileFromPart(part) + if mpf == nil || err != nil { + t.Fatal("Expected non-nil MultipartFile, nil error") + } + if mpf.IsDirectory() { + t.Fatal("Expected file to not be a directory") + } + if mpf.FileName() != "file.txt" { + t.Fatal("Expected filename to be \"file.txt\"") + } + if n, err := mpf.Read(buf); n != len(text) || err != nil { + t.Fatal("Expected to read from file", n, err) + } + if string(buf[:len(text)]) != text { + t.Fatal("Data read was different than expected") + } + + part, err = mpReader.NextPart() + if part == nil || err != nil { + t.Fatal("Expected non-nil part, nil error") + } + mpf, err = NewFileFromPart(part) + if mpf == nil || err != nil { + t.Fatal("Expected non-nil MultipartFile, nil error") + } + if !mpf.IsDirectory() { + t.Fatal("Expected file to be a directory") + } + if mpf.FileName() != "boop" { + t.Fatal("Expected filename to be \"boop\"") + } + + part, err = mpReader.NextPart() + if part == nil || err != nil { + t.Fatal("Expected non-nil part, nil error") + } + child, err := NewFileFromPart(part) + if child == nil || err != nil { + t.Fatal("Expected to be able to read a child file") + } + if child.IsDirectory() { + t.Fatal("Expected file to not be a directory") + } + if child.FileName() != "boop/a.txt" { + t.Fatal("Expected filename to be \"some/file/path\"") + } + + part, err = mpReader.NextPart() + if part == nil || err != nil { + t.Fatal("Expected non-nil part, nil error") + } + child, err = NewFileFromPart(part) + if child == nil || err != nil { + t.Fatal("Expected to be able to read a child file") + } + if child.IsDirectory() { + t.Fatal("Expected file to not be a directory") + } + if child.FileName() != "boop/b.txt" { + t.Fatal("Expected filename to be \"some/file/path\"") + } + + child, err = mpf.NextFile() + if child != nil || err != io.EOF { + t.Fatal("Expected to get (nil, io.EOF)") + } + + part, err = mpReader.NextPart() + if part == nil || err != nil { + t.Fatal("Expected non-nil part, nil error") + } + mpf, err = NewFileFromPart(part) + if mpf == nil || err != nil { + t.Fatal("Expected non-nil MultipartFile, nil error") + } + + part, err = mpReader.NextPart() + if part != nil || err != io.EOF { + t.Fatal("Expected to get (nil, io.EOF)") + } +} From 27e92b6b766761badff5da907ea83ba1351e3ad3 Mon Sep 17 00:00:00 2001 From: keks Date: Mon, 11 Dec 2017 17:08:38 +0100 Subject: [PATCH 06/76] fall back to application/octet-stream if Content-Type is empty This commit was moved from ipfs/go-ipfs-files@d398783bcc2e607529bbe832655cb361d3806bed --- files/multipartfile.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/files/multipartfile.go b/files/multipartfile.go index 21e0d44c1..8a4b0b2a5 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -46,6 +46,8 @@ func NewFileFromPart(part *multipart.Part) (File, error) { Target: string(out), name: f.FileName(), }, nil + case "": // default to application/octet-stream + fallthrough case applicationFile: return &ReaderFile{ reader: part, From 6cfc7908a32ea5d02822727e848706155a70aa3c Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 26 Jul 2018 15:50:16 -0700 Subject: [PATCH 07/76] only count size for regular files Otherwise, we incorrectly draw progress bars. fixes #24 This commit was moved from ipfs/go-ipfs-files@964db8d30bdd00e59d8e8b1b54f18f2715bf957c --- files/serialfile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/serialfile.go b/files/serialfile.go index 611d3f1db..15e6c9051 100644 --- a/files/serialfile.go +++ b/files/serialfile.go @@ -143,7 +143,7 @@ func (f *serialFile) Size() (int64, error) { return err } - if fi != nil && fi.Mode()&(os.ModeSymlink|os.ModeNamedPipe) == 0 { + if fi != nil && fi.Mode().IsRegular() { du += fi.Size() } return nil From f2156426500797bb9f78a73becd82c6118a600f6 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Sun, 19 Aug 2018 19:10:40 +0200 Subject: [PATCH 08/76] Feat: add WebFile File implementation. A WebFile is a File which is read from a Web URL using a GET request. This commit was moved from ipfs/go-ipfs-files@93a3bed955d9dfb40fd8517d4d818500499a69a6 --- files/webfile.go | 68 +++++++++++++++++++++++++++++++++++++++++++ files/webfile_test.go | 38 ++++++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 files/webfile.go create mode 100644 files/webfile_test.go diff --git a/files/webfile.go b/files/webfile.go new file mode 100644 index 000000000..fcf4412ea --- /dev/null +++ b/files/webfile.go @@ -0,0 +1,68 @@ +package files + +import ( + "io" + "net/http" + "net/url" + "path/filepath" +) + +// WebFile is an implementation of File which reads it +// from a Web URL (http). A GET request will be performed +// against the source when calling Read(). +type WebFile struct { + body io.ReadCloser + url *url.URL +} + +// NewWebFile creates a WebFile with the given URL, which +// will be used to perform the GET request on Read(). +func NewWebFile(url *url.URL) *WebFile { + return &WebFile{ + url: url, + } +} + +// Read reads the File from it's web location. On the first +// call to Read, a GET request will be performed against the +// WebFile's URL, using Go's default HTTP client. Any further +// reads will keep reading from the HTTP Request body. +func (wf *WebFile) Read(b []byte) (int, error) { + if wf.body == nil { + resp, err := http.Get(wf.url.String()) + if err != nil { + return 0, err + } + wf.body = resp.Body + } + return wf.body.Read(b) +} + +// Close closes the WebFile (or the request body). +func (wf *WebFile) Close() error { + if wf.body == nil { + return nil + } + return wf.body.Close() +} + +// FullPath returns the "Host+Path" for this WebFile. +func (wf *WebFile) FullPath() string { + return wf.url.Host + wf.url.Path +} + +// FileName returns the last element of the URL +// path for this file. +func (wf *WebFile) FileName() string { + return filepath.Base(wf.url.Path) +} + +// IsDirectory returns false. +func (wf *WebFile) IsDirectory() bool { + return false +} + +// NextFile always returns an ErrNotDirectory error. +func (wf *WebFile) NextFile() (File, error) { + return nil, ErrNotDirectory +} diff --git a/files/webfile_test.go b/files/webfile_test.go new file mode 100644 index 000000000..d06bd68ea --- /dev/null +++ b/files/webfile_test.go @@ -0,0 +1,38 @@ +package files + +import ( + "fmt" + "io/ioutil" + "net" + "net/http" + "net/url" + "testing" +) + +func TestWebFile(t *testing.T) { + http.HandleFunc("/my/url/content.txt", func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "Hello world!") + }) + listener, err := net.Listen("tcp", ":18281") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + + go func() { + http.Serve(listener, nil) + }() + + u, err := url.Parse("http://127.0.0.1:18281/my/url/content.txt") + if err != nil { + t.Fatal(err) + } + wf := NewWebFile(u) + body, err := ioutil.ReadAll(wf) + if err != nil { + t.Fatal(err) + } + if string(body) != "Hello world!" { + t.Fatal("should have read the web file") + } +} From 69fcace97313cdfb4645b329cdccdbddfe3453ad Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Mon, 20 Aug 2018 19:35:31 +0200 Subject: [PATCH 09/76] Webfile tests: use httptest.Server This commit was moved from ipfs/go-ipfs-files@cbb84796970b5a4c631c0fb451994387f741132b --- files/webfile_test.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/files/webfile_test.go b/files/webfile_test.go index d06bd68ea..889cdc48d 100644 --- a/files/webfile_test.go +++ b/files/webfile_test.go @@ -3,8 +3,8 @@ package files import ( "fmt" "io/ioutil" - "net" "net/http" + "net/http/httptest" "net/url" "testing" ) @@ -13,17 +13,13 @@ func TestWebFile(t *testing.T) { http.HandleFunc("/my/url/content.txt", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "Hello world!") }) - listener, err := net.Listen("tcp", ":18281") - if err != nil { - t.Fatal(err) - } - defer listener.Close() - go func() { - http.Serve(listener, nil) - }() + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "Hello world!") + })) + defer s.Close() - u, err := url.Parse("http://127.0.0.1:18281/my/url/content.txt") + u, err := url.Parse(s.URL) if err != nil { t.Fatal(err) } From 0a522e3ab3cd3ed6d7aee8b13af7e9b5101afb13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 16 Oct 2018 13:31:18 +0200 Subject: [PATCH 10/76] Add README, LICENSE This commit was moved from ipfs/go-ipfs-files@efcd0bdfbaf5ab3d80f0006a44599f906dba9aaf --- files/README.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 files/README.md diff --git a/files/README.md b/files/README.md new file mode 100644 index 000000000..4f7046954 --- /dev/null +++ b/files/README.md @@ -0,0 +1,27 @@ +# go-ipfs-files + +[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) +[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) +[![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) +[![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) + +> File interfaces and utils used in IPFS + +## Documentation + +https://godoc.org/github.com/ipfs/go-ipfs-files + +## Contribute + +Feel free to join in. All welcome. Open an [issue](https://github.com/ipfs/go-ipfs-files/issues)! + +This repository falls under the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +### Want to hack on IPFS? + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/contributing.md) + +## License + +MIT + From ef11939815c95de3eb61a6977faa654ec9e5dd68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 24 Oct 2018 00:24:19 +0200 Subject: [PATCH 11/76] [WIP] Refactor filename - file relation This commit was moved from ipfs/go-ipfs-files@6c677812cf71e9d78a6d6ca5971c11a94adbcdd4 --- files/file.go | 56 +++++++--------- files/file_test.go | 43 ++++++------- files/is_hidden.go | 5 +- files/is_hidden_windows.go | 9 ++- files/linkfile.go | 32 ++++++---- files/multifilereader.go | 32 ++++++---- files/multifilereader_test.go | 111 ++++++++++++++++++++++++-------- files/multipartfile.go | 117 ++++++++++++++++++++++++++-------- files/readerfile.go | 38 ++++++----- files/serialfile.go | 55 ++++++---------- files/slicefile.go | 41 +++++------- files/webfile.go | 17 ++--- 12 files changed, 325 insertions(+), 231 deletions(-) diff --git a/files/file.go b/files/file.go index c5e820336..ceb60dded 100644 --- a/files/file.go +++ b/files/file.go @@ -7,56 +7,46 @@ import ( ) var ( - ErrNotDirectory = errors.New("Couldn't call NextFile(), this isn't a directory") - ErrNotReader = errors.New("This file is a directory, can't use Reader functions") + ErrNotDirectory = errors.New("couldn't call NextFile(), this isn't a directory") + ErrNotReader = errors.New("this file is a directory, can't use Reader functions") + + ErrNotSupported = errors.New("operation not supported") ) // File is an interface that provides functionality for handling // files/directories as values that can be supplied to commands. For -// directories, child files are accessed serially by calling `NextFile()`. +// directories, child files are accessed serially by calling `Files()` +// or `Walk()`. +// +// Read/Seek/Close methods are only valid for files +// Files/Walk methods are only valid for directories type File interface { - // Files implement ReadCloser, but can only be read from or closed if - // they are not directories - io.ReadCloser - - // FileName returns a filename associated with this file - FileName() string + io.Reader + io.Closer + io.Seeker - // FullPath returns the full path used when adding with this file - FullPath() string + // Size returns size of the + Size() (int64, error) // IsDirectory returns true if the File is a directory (and therefore - // supports calling `NextFile`) and false if the File is a normal file - // (and therefor supports calling `Read` and `Close`) + // supports calling `Files`/`Walk`) and false if the File is a normal file + // (and therefore supports calling `Read`/`Close`/`Seek`) IsDirectory() bool // NextFile returns the next child file available (if the File is a - // directory). It will return (nil, io.EOF) if no more files are + // directory). It will return io.EOF if no more files are // available. If the file is a regular file (not a directory), NextFile // will return a non-nil error. - NextFile() (File, error) + NextFile() (string, File, error) } -type StatFile interface { - File - - Stat() os.FileInfo -} - -type PeekFile interface { - SizeFile - - Peek(n int) File - Length() int -} - -type SizeFile interface { +// FileInfo exposes information on files in local filesystem +type FileInfo interface { File - Size() (int64, error) -} - -type FileInfo interface { + // AbsPath returns full/real file path. AbsPath() string + + // Stat returns os.Stat of this file Stat() os.FileInfo } diff --git a/files/file_test.go b/files/file_test.go index 733a8c87d..60d1bc482 100644 --- a/files/file_test.go +++ b/files/file_test.go @@ -9,15 +9,14 @@ import ( ) func TestSliceFiles(t *testing.T) { - name := "testname" - files := []File{ - NewReaderFile("file.txt", "file.txt", ioutil.NopCloser(strings.NewReader("Some text!\n")), nil), - NewReaderFile("beep.txt", "beep.txt", ioutil.NopCloser(strings.NewReader("beep")), nil), - NewReaderFile("boop.txt", "boop.txt", ioutil.NopCloser(strings.NewReader("boop")), nil), + files := []FileEntry{ + {NewReaderFile(ioutil.NopCloser(strings.NewReader("Some text!\n")), nil), ""}, + {NewReaderFile(ioutil.NopCloser(strings.NewReader("beep")), nil), ""}, + {NewReaderFile(ioutil.NopCloser(strings.NewReader("boop")), nil), ""}, } buf := make([]byte, 20) - sf := NewSliceFile(name, name, files) + sf := NewSliceFile(files) if !sf.IsDirectory() { t.Fatal("SliceFile should always be a directory") @@ -31,7 +30,7 @@ func TestSliceFiles(t *testing.T) { t.Fatal("Shouldn't be able to call `Close` on a SliceFile") } - file, err := sf.NextFile() + _, file, err := sf.NextFile() if file == nil || err != nil { t.Fatal("Expected a file and nil error") } @@ -40,16 +39,16 @@ func TestSliceFiles(t *testing.T) { t.Fatal("NextFile got a file in the wrong order") } - file, err = sf.NextFile() + _, file, err = sf.NextFile() if file == nil || err != nil { t.Fatal("Expected a file and nil error") } - file, err = sf.NextFile() + _, file, err = sf.NextFile() if file == nil || err != nil { t.Fatal("Expected a file and nil error") } - file, err = sf.NextFile() + _, file, err = sf.NextFile() if file != nil || err != io.EOF { t.Fatal("Expected a nil file and io.EOF") } @@ -57,13 +56,13 @@ func TestSliceFiles(t *testing.T) { func TestReaderFiles(t *testing.T) { message := "beep boop" - rf := NewReaderFile("file.txt", "file.txt", ioutil.NopCloser(strings.NewReader(message)), nil) + rf := NewReaderFile(ioutil.NopCloser(strings.NewReader(message)), nil) buf := make([]byte, len(message)) if rf.IsDirectory() { t.Fatal("ReaderFile should never be a directory") } - file, err := rf.NextFile() + _, file, err := rf.NextFile() if file != nil || err != ErrNotDirectory { t.Fatal("Expected a nil file and ErrNotDirectory") } @@ -114,17 +113,17 @@ anotherfile if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - mpf, err := NewFileFromPart(part) + mpname, mpf, err := newFileFromPart("", part, mpReader) if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } if mpf.IsDirectory() { t.Fatal("Expected file to not be a directory") } - if mpf.FileName() != "name" { + if mpname != "name" { t.Fatal("Expected filename to be \"name\"") } - if file, err := mpf.NextFile(); file != nil || err != ErrNotDirectory { + if _, file, err := mpf.NextFile(); file != nil || err != ErrNotDirectory { t.Fatal("Expected a nil file and ErrNotDirectory") } if n, err := mpf.Read(buf); n != 4 || !(err == io.EOF || err == nil) { @@ -139,14 +138,14 @@ anotherfile if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - mpf, err = NewFileFromPart(part) + mpname, mpf, err = newFileFromPart("", part, mpReader) if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } if !mpf.IsDirectory() { t.Fatal("Expected file to be a directory") } - if mpf.FileName() != "dir" { + if mpname != "dir" { t.Fatal("Expected filename to be \"dir\"") } if n, err := mpf.Read(buf); n > 0 || err != ErrNotReader { @@ -161,15 +160,15 @@ anotherfile if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - mpf, err = NewFileFromPart(part) + mpname, mpf, err = newFileFromPart("dir/", part, mpReader) if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } if mpf.IsDirectory() { t.Fatal("Expected file, got directory") } - if mpf.FileName() != "dir/nested" { - t.Fatalf("Expected filename to be \"nested\", got %s", mpf.FileName()) + if mpname != "nested" { + t.Fatalf("Expected filename to be \"nested\", got %s", mpname) } if n, err := mpf.Read(buf); n != 12 || !(err == nil || err == io.EOF) { t.Fatalf("expected to be able to read 12 bytes from file: %s (got %d)", err, n) @@ -183,14 +182,14 @@ anotherfile if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - mpf, err = NewFileFromPart(part) + mpname, mpf, err = newFileFromPart("dir/", part, mpReader) if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } if mpf.IsDirectory() { t.Fatal("Expected file to be a symlink") } - if mpf.FileName() != "dir/simlynk" { + if mpname != "simlynk" { t.Fatal("Expected filename to be \"dir/simlynk\"") } slink, ok := mpf.(*Symlink) diff --git a/files/is_hidden.go b/files/is_hidden.go index b0360685b..d5bc88683 100644 --- a/files/is_hidden.go +++ b/files/is_hidden.go @@ -7,9 +7,8 @@ import ( "strings" ) -func IsHidden(f File) bool { - - fName := filepath.Base(f.FileName()) +func IsHidden(name string, f File) bool { + fName := filepath.Base(name) if strings.HasPrefix(fName, ".") && len(fName) > 1 { return true diff --git a/files/is_hidden_windows.go b/files/is_hidden_windows.go index 7679433df..40f40ae62 100644 --- a/files/is_hidden_windows.go +++ b/files/is_hidden_windows.go @@ -9,7 +9,7 @@ import ( windows "golang.org/x/sys/windows" ) -func IsHidden(f File) bool { +func IsHidden(name string, f File) bool { fName := filepath.Base(f.FileName()) @@ -17,7 +17,12 @@ func IsHidden(f File) bool { return true } - p, e := windows.UTF16PtrFromString(f.FullPath()) + fi, ok := f.(FileInfo) + if !ok { + return false + } + + p, e := windows.UTF16PtrFromString(fi.AbsPath()) if e != nil { return false } diff --git a/files/linkfile.go b/files/linkfile.go index 18466f4bd..0182d3969 100644 --- a/files/linkfile.go +++ b/files/linkfile.go @@ -7,7 +7,6 @@ import ( ) type Symlink struct { - name string path string Target string stat os.FileInfo @@ -15,9 +14,8 @@ type Symlink struct { reader io.Reader } -func NewLinkFile(name, path, target string, stat os.FileInfo) File { +func NewLinkFile(path, target string, stat os.FileInfo) File { return &Symlink{ - name: name, path: path, Target: target, stat: stat, @@ -29,22 +27,30 @@ func (lf *Symlink) IsDirectory() bool { return false } -func (lf *Symlink) NextFile() (File, error) { - return nil, io.EOF +func (lf *Symlink) NextFile() (string, File, error) { + return "", nil, ErrNotDirectory } -func (f *Symlink) FileName() string { - return f.name -} +func (lf *Symlink) Close() error { + if c, ok := lf.reader.(io.Closer); ok { + return c.Close() + } -func (f *Symlink) Close() error { return nil } -func (f *Symlink) FullPath() string { - return f.path +func (lf *Symlink) Read(b []byte) (int, error) { + return lf.reader.Read(b) +} + +func (lf *Symlink) Seek(offset int64, whence int) (int64, error) { + if s, ok := lf.reader.(io.Seeker); ok { + return s.Seek(offset, whence) + } + + return 0, ErrNotSupported } -func (f *Symlink) Read(b []byte) (int, error) { - return f.reader.Read(b) +func (lf *Symlink) Size() (int64, error) { + return 0, ErrNotSupported } diff --git a/files/multifilereader.go b/files/multifilereader.go index 4833e8d18..7863c9d61 100644 --- a/files/multifilereader.go +++ b/files/multifilereader.go @@ -7,6 +7,7 @@ import ( "mime/multipart" "net/textproto" "net/url" + "path" "sync" ) @@ -15,7 +16,10 @@ import ( type MultiFileReader struct { io.Reader - files []File + // directory stack for NextFile + files []File + path []string + currentFile io.Reader buf bytes.Buffer mpWriter *multipart.Writer @@ -33,6 +37,7 @@ type MultiFileReader struct { func NewMultiFileReader(file File, form bool) *MultiFileReader { mfr := &MultiFileReader{ files: []File{file}, + path: []string{""}, form: form, mutex: &sync.Mutex{}, } @@ -53,6 +58,8 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { // if the current file isn't set, advance to the next file if mfr.currentFile == nil { var file File + var name string + for file == nil { if len(mfr.files) == 0 { mfr.mpWriter.Close() @@ -60,40 +67,43 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { return mfr.buf.Read(buf) } - nextfile, err := mfr.files[len(mfr.files)-1].NextFile() + nextName, nextFile, err := mfr.files[len(mfr.files)-1].NextFile() if err == io.EOF { mfr.files = mfr.files[:len(mfr.files)-1] + mfr.path = mfr.path[:len(mfr.path)-1] continue } else if err != nil { return 0, err } - file = nextfile + file = nextFile + name = nextName } // handle starting a new file part if !mfr.closed { + mfr.currentFile = file + + // write the boundary and headers + header := make(textproto.MIMEHeader) + filename := url.QueryEscape(path.Join(path.Join(mfr.path...), name)) + header.Set("Content-Disposition", fmt.Sprintf("file; filename=\"%s\"", filename)) + var contentType string if _, ok := file.(*Symlink); ok { contentType = "application/symlink" } else if file.IsDirectory() { mfr.files = append(mfr.files, file) + mfr.path = append(mfr.path, name) contentType = "application/x-directory" } else { // otherwise, use the file as a reader to read its contents contentType = "application/octet-stream" } - mfr.currentFile = file - - // write the boundary and headers - header := make(textproto.MIMEHeader) - filename := url.QueryEscape(file.FileName()) - header.Set("Content-Disposition", fmt.Sprintf("file; filename=\"%s\"", filename)) - header.Set("Content-Type", contentType) - if rf, ok := file.(*ReaderFile); ok { + if rf, ok := file.(FileInfo); ok { header.Set("abspath", rf.AbsPath()) } diff --git a/files/multifilereader_test.go b/files/multifilereader_test.go index 3d2c97892..18876ef00 100644 --- a/files/multifilereader_test.go +++ b/files/multifilereader_test.go @@ -8,35 +8,93 @@ import ( "testing" ) -func TestOutput(t *testing.T) { - text := "Some text! :)" - fileset := []File{ - NewReaderFile("file.txt", "file.txt", ioutil.NopCloser(strings.NewReader(text)), nil), - NewSliceFile("boop", "boop", []File{ - NewReaderFile("boop/a.txt", "boop/a.txt", ioutil.NopCloser(strings.NewReader("bleep")), nil), - NewReaderFile("boop/b.txt", "boop/b.txt", ioutil.NopCloser(strings.NewReader("bloop")), nil), - }), - NewReaderFile("beep.txt", "beep.txt", ioutil.NopCloser(strings.NewReader("beep")), nil), - } - sf := NewSliceFile("", "", fileset) - buf := make([]byte, 20) +var text = "Some text! :)" + +func getTestMultiFileReader() *MultiFileReader { + fileset := []FileEntry{ + {NewReaderFile(ioutil.NopCloser(strings.NewReader(text)), nil), "file.txt"}, + {NewSliceFile([]FileEntry{ + {NewReaderFile(ioutil.NopCloser(strings.NewReader("bleep")), nil), "a.txt"}, + {NewReaderFile(ioutil.NopCloser(strings.NewReader("bloop")), nil), "b.txt"}, + }), "boop"}, + {NewReaderFile(ioutil.NopCloser(strings.NewReader("beep")), nil), "beep.txt"}, + } + sf := NewSliceFile(fileset) // testing output by reading it with the go stdlib "mime/multipart" Reader - mfr := NewMultiFileReader(sf, true) + return NewMultiFileReader(sf, true) +} + +func TestMultiFileReaderToMultiFile(t *testing.T) { + mfr := getTestMultiFileReader() mpReader := multipart.NewReader(mfr, mfr.Boundary()) + mf, err := NewFileFromPartReader(mpReader, multipartFormdataType) + if err != nil { + t.Fatal(err) + } + + if !mf.IsDirectory() { + t.Fatal("Expected a directory") + } + + fn, f, err := mf.NextFile() + if fn != "file.txt" || f == nil || err != nil { + t.Fatal("NextFile returned unexpected data") + } + + dn, d, err := mf.NextFile() + if dn != "boop" || d == nil || err != nil { + t.Fatal("NextFile returned unexpected data") + } + + if !d.IsDirectory() { + t.Fatal("Expected a directory") + } + + cfn, cf, err := d.NextFile() + if cfn != "a.txt" || cf == nil || err != nil { + t.Fatal("NextFile returned unexpected data") + } + + cfn, cf, err = d.NextFile() + if cfn != "b.txt" || cf == nil || err != nil { + t.Fatal("NextFile returned unexpected data") + } + + cfn, cf, err = d.NextFile() + if cfn != "" || cf != nil || err != io.EOF { + t.Fatal("NextFile returned unexpected data") + } + + // try to break internal state + cfn, cf, err = d.NextFile() + if cfn != "" || cf != nil || err != io.EOF { + t.Fatal("NextFile returned unexpected data") + } + + fn, f, err = mf.NextFile() + if fn != "beep.txt" || f == nil || err != nil { + t.Fatal("NextFile returned unexpected data") + } +} + +func TestOutput(t *testing.T) { + mfr := getTestMultiFileReader() + mpReader := &peekReader{r: multipart.NewReader(mfr, mfr.Boundary())} + buf := make([]byte, 20) part, err := mpReader.NextPart() if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - mpf, err := NewFileFromPart(part) + mpname, mpf, err := newFileFromPart("", part, mpReader) if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } if mpf.IsDirectory() { t.Fatal("Expected file to not be a directory") } - if mpf.FileName() != "file.txt" { + if mpname != "file.txt" { t.Fatal("Expected filename to be \"file.txt\"") } if n, err := mpf.Read(buf); n != len(text) || err != nil { @@ -50,14 +108,14 @@ func TestOutput(t *testing.T) { if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - mpf, err = NewFileFromPart(part) + mpname, mpf, err = newFileFromPart("", part, mpReader) if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } if !mpf.IsDirectory() { t.Fatal("Expected file to be a directory") } - if mpf.FileName() != "boop" { + if mpname != "boop" { t.Fatal("Expected filename to be \"boop\"") } @@ -65,33 +123,33 @@ func TestOutput(t *testing.T) { if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - child, err := NewFileFromPart(part) + cname, child, err := newFileFromPart("boop", part, mpReader) if child == nil || err != nil { t.Fatal("Expected to be able to read a child file") } if child.IsDirectory() { t.Fatal("Expected file to not be a directory") } - if child.FileName() != "boop/a.txt" { - t.Fatal("Expected filename to be \"some/file/path\"") + if cname != "a.txt" { + t.Fatal("Expected filename to be \"a.txt\"") } part, err = mpReader.NextPart() if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - child, err = NewFileFromPart(part) + cname, child, err = newFileFromPart("boop", part, mpReader) if child == nil || err != nil { t.Fatal("Expected to be able to read a child file") } if child.IsDirectory() { t.Fatal("Expected file to not be a directory") } - if child.FileName() != "boop/b.txt" { - t.Fatal("Expected filename to be \"some/file/path\"") + if cname != "b.txt" { + t.Fatal("Expected filename to be \"b.txt\"") } - child, err = mpf.NextFile() + cname, child, err = mpf.NextFile() if child != nil || err != io.EOF { t.Fatal("Expected to get (nil, io.EOF)") } @@ -100,10 +158,13 @@ func TestOutput(t *testing.T) { if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - mpf, err = NewFileFromPart(part) + mpname, mpf, err = newFileFromPart("", part, mpReader) if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } + if mpname != "beep.txt" { + t.Fatal("Expected filename to be \"b.txt\"") + } part, err = mpReader.NextPart() if part != nil || err != io.EOF { diff --git a/files/multipartfile.go b/files/multipartfile.go index 8a4b0b2a5..a3041638b 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -1,11 +1,13 @@ package files import ( + "errors" "io" "io/ioutil" "mime" "mime/multipart" "net/url" + "path" ) const ( @@ -19,19 +21,36 @@ const ( contentTypeHeader = "Content-Type" ) +var ErrPartOutsideParent = errors.New("file outside parent dir") + // MultipartFile implements File, and is created from a `multipart.Part`. // It can be either a directory or file (checked by calling `IsDirectory()`). type MultipartFile struct { File Part *multipart.Part - Reader *multipart.Reader + Reader PartReader Mediatype string } -func NewFileFromPart(part *multipart.Part) (File, error) { +func NewFileFromPartReader(reader *multipart.Reader, mediatype string) (File, error) { + f := &MultipartFile{ + Reader: &peekReader{r: reader}, + Mediatype: mediatype, + } + + return f, nil +} + +func newFileFromPart(parent string, part *multipart.Part, reader PartReader) (string, File, error) { f := &MultipartFile{ - Part: part, + Part: part, + Reader: reader, + } + + dir, base := path.Split(f.fileName()) + if path.Clean(dir) != path.Clean(parent) { + return "", nil, ErrPartOutsideParent } contentType := part.Header.Get(contentTypeHeader) @@ -39,54 +58,64 @@ func NewFileFromPart(part *multipart.Part) (File, error) { case applicationSymlink: out, err := ioutil.ReadAll(part) if err != nil { - return nil, err + return "", nil, err } - return &Symlink{ + return base, &Symlink{ Target: string(out), - name: f.FileName(), }, nil case "": // default to application/octet-stream fallthrough case applicationFile: - return &ReaderFile{ - reader: part, - filename: f.FileName(), - abspath: part.Header.Get("abspath"), - fullpath: f.FullPath(), + return base, &ReaderFile{ + reader: part, + abspath: part.Header.Get("abspath"), }, nil } var err error f.Mediatype, _, err = mime.ParseMediaType(contentType) if err != nil { - return nil, err + return "", nil, err } - return f, nil + return base, f, nil } func (f *MultipartFile) IsDirectory() bool { return f.Mediatype == multipartFormdataType || f.Mediatype == applicationDirectory } -func (f *MultipartFile) NextFile() (File, error) { +func (f *MultipartFile) NextFile() (string, File, error) { if !f.IsDirectory() { - return nil, ErrNotDirectory + return "", nil, ErrNotDirectory + } + if f.Reader == nil { + return "", nil, io.EOF + } + part, err := f.Reader.NextPart() + if err != nil { + return "", nil, err } - if f.Reader != nil { - part, err := f.Reader.NextPart() - if err != nil { - return nil, err - } - return NewFileFromPart(part) + name, cf, err := newFileFromPart(f.fileName(), part, f.Reader) + if err != ErrPartOutsideParent { + return name, cf, err } - return nil, io.EOF + // we read too much, try to fix this + pr, ok := f.Reader.(*peekReader) + if !ok { + return "", nil, errors.New("cannot undo NextPart") + } + + if err := pr.put(part); err != nil { + return "", nil, err + } + return "", nil, io.EOF } -func (f *MultipartFile) FileName() string { +func (f *MultipartFile) fileName() string { if f == nil || f.Part == nil { return "" } @@ -99,10 +128,6 @@ func (f *MultipartFile) FileName() string { return filename } -func (f *MultipartFile) FullPath() string { - return f.FileName() -} - func (f *MultipartFile) Read(p []byte) (int, error) { if f.IsDirectory() { return 0, ErrNotReader @@ -116,3 +141,41 @@ func (f *MultipartFile) Close() error { } return f.Part.Close() } + +func (f *MultipartFile) Seek(offset int64, whence int) (int64, error) { + if f.IsDirectory() { + return 0, ErrNotReader + } + return 0, ErrNotReader +} + +func (f *MultipartFile) Size() (int64, error) { + return 0, ErrNotReader +} + +type PartReader interface { + NextPart() (*multipart.Part, error) +} + +type peekReader struct { + r PartReader + next *multipart.Part +} + +func (pr *peekReader) NextPart() (*multipart.Part, error) { + if pr.next != nil { + p := pr.next + pr.next = nil + return p, nil + } + + return pr.r.NextPart() +} + +func (pr *peekReader) put(p *multipart.Part) error { + if pr.next != nil { + return errors.New("cannot put multiple parts") + } + pr.next = p + return nil +} diff --git a/files/readerfile.go b/files/readerfile.go index 863641479..e40032c8c 100644 --- a/files/readerfile.go +++ b/files/readerfile.go @@ -10,40 +10,30 @@ import ( // ReaderFile is a implementation of File created from an `io.Reader`. // ReaderFiles are never directories, and can be read from and closed. type ReaderFile struct { - filename string - fullpath string - abspath string - reader io.ReadCloser - stat os.FileInfo + abspath string + reader io.ReadCloser + stat os.FileInfo } -func NewReaderFile(filename, path string, reader io.ReadCloser, stat os.FileInfo) *ReaderFile { - return &ReaderFile{filename, path, path, reader, stat} +func NewReaderFile(reader io.ReadCloser, stat os.FileInfo) File { + return &ReaderFile{"", reader, stat} } -func NewReaderPathFile(filename, path string, reader io.ReadCloser, stat os.FileInfo) (*ReaderFile, error) { +func NewReaderPathFile(path string, reader io.ReadCloser, stat os.FileInfo) (*ReaderFile, error) { abspath, err := filepath.Abs(path) if err != nil { return nil, err } - return &ReaderFile{filename, path, abspath, reader, stat}, nil + return &ReaderFile{abspath, reader, stat}, nil } func (f *ReaderFile) IsDirectory() bool { return false } -func (f *ReaderFile) NextFile() (File, error) { - return nil, ErrNotDirectory -} - -func (f *ReaderFile) FileName() string { - return f.filename -} - -func (f *ReaderFile) FullPath() string { - return f.fullpath +func (f *ReaderFile) NextFile() (string, File, error) { + return "", nil, ErrNotDirectory } func (f *ReaderFile) AbsPath() string { @@ -64,7 +54,15 @@ func (f *ReaderFile) Stat() os.FileInfo { func (f *ReaderFile) Size() (int64, error) { if f.stat == nil { - return 0, errors.New("File size unknown") + return 0, errors.New("file size unknown") } return f.stat.Size(), nil } + +func (f *ReaderFile) Seek(offset int64, whence int) (int64, error) { + if s, ok := f.reader.(io.Seeker); ok { + return s.Seek(offset, whence) + } + + return 0, ErrNotSupported +} diff --git a/files/serialfile.go b/files/serialfile.go index 15e6c9051..eb4197674 100644 --- a/files/serialfile.go +++ b/files/serialfile.go @@ -7,29 +7,26 @@ import ( "os" "path/filepath" "strings" - "syscall" ) // serialFile implements File, and reads from a path on the OS filesystem. // No more than one file will be opened at a time (directories will advance // to the next file when NextFile() is called). type serialFile struct { - name string path string files []os.FileInfo stat os.FileInfo - current *File handleHiddenFiles bool } -func NewSerialFile(name, path string, hidden bool, stat os.FileInfo) (File, error) { +func NewSerialFile(path string, hidden bool, stat os.FileInfo) (File, error) { switch mode := stat.Mode(); { case mode.IsRegular(): file, err := os.Open(path) if err != nil { return nil, err } - return NewReaderPathFile(name, path, file, stat) + return NewReaderPathFile(path, file, stat) case mode.IsDir(): // for directories, stat all of the contents first, so we know what files to // open when NextFile() is called @@ -37,15 +34,15 @@ func NewSerialFile(name, path string, hidden bool, stat os.FileInfo) (File, erro if err != nil { return nil, err } - return &serialFile{name, path, contents, stat, nil, hidden}, nil + return &serialFile{path, contents, stat, hidden}, nil case mode&os.ModeSymlink != 0: target, err := os.Readlink(path) if err != nil { return nil, err } - return NewLinkFile(name, path, target, stat), nil + return NewLinkFile(path, target, stat), nil default: - return nil, fmt.Errorf("Unrecognized file type for %s: %s", name, mode.String()) + return nil, fmt.Errorf("unrecognized file type for %s: %s", path, mode.String()) } } @@ -55,23 +52,23 @@ func (f *serialFile) IsDirectory() bool { return true } -func (f *serialFile) NextFile() (File, error) { +func (f *serialFile) NextFile() (string, File, error) { // if a file was opened previously, close it err := f.Close() if err != nil { switch err2 := err.(type) { case *os.PathError: if err2.Err != os.ErrClosed { - return nil, err + return "", nil, err } default: - return nil, err + return "", nil, err } } // if there aren't any files left in the root directory, we're done if len(f.files) == 0 { - return nil, io.EOF + return "", nil, io.EOF } stat := f.files[0] @@ -79,7 +76,7 @@ func (f *serialFile) NextFile() (File, error) { for !f.handleHiddenFiles && strings.HasPrefix(stat.Name(), ".") { if len(f.files) == 0 { - return nil, io.EOF + return "", nil, io.EOF } stat = f.files[0] @@ -87,28 +84,17 @@ func (f *serialFile) NextFile() (File, error) { } // open the next file - fileName := filepath.ToSlash(filepath.Join(f.name, stat.Name())) filePath := filepath.ToSlash(filepath.Join(f.path, stat.Name())) // recursively call the constructor on the next file // if it's a regular file, we will open it as a ReaderFile // if it's a directory, files in it will be opened serially - sf, err := NewSerialFile(fileName, filePath, f.handleHiddenFiles, stat) + sf, err := NewSerialFile(filePath, f.handleHiddenFiles, stat) if err != nil { - return nil, err + return "", nil, err } - f.current = &sf - - return sf, nil -} - -func (f *serialFile) FileName() string { - return f.name -} - -func (f *serialFile) FullPath() string { - return f.path + return stat.Name(), sf, nil } func (f *serialFile) Read(p []byte) (int, error) { @@ -116,18 +102,13 @@ func (f *serialFile) Read(p []byte) (int, error) { } func (f *serialFile) Close() error { - // close the current file if there is one - if f.current != nil { - err := (*f.current).Close() - // ignore EINVAL error, the file might have already been closed - if err != nil && err != syscall.EINVAL { - return err - } - } - return nil } +func (f *serialFile) Seek(offset int64, whence int) (int64, error) { + return 0, ErrNotReader +} + func (f *serialFile) Stat() os.FileInfo { return f.stat } @@ -138,7 +119,7 @@ func (f *serialFile) Size() (int64, error) { } var du int64 - err := filepath.Walk(f.FullPath(), func(p string, fi os.FileInfo, err error) error { + err := filepath.Walk(f.path, func(p string, fi os.FileInfo, err error) error { if err != nil { return err } diff --git a/files/slicefile.go b/files/slicefile.go index 8d18dcaa3..d356465b5 100644 --- a/files/slicefile.go +++ b/files/slicefile.go @@ -1,43 +1,37 @@ package files import ( - "errors" "io" ) +type FileEntry struct { + File File + Name string +} + // SliceFile implements File, and provides simple directory handling. // It contains children files, and is created from a `[]File`. // SliceFiles are always directories, and can't be read from or closed. type SliceFile struct { - filename string - path string - files []File - n int + files []FileEntry + n int } -func NewSliceFile(filename, path string, files []File) *SliceFile { - return &SliceFile{filename, path, files, 0} +func NewSliceFile(files []FileEntry) File { + return &SliceFile{files, 0} } func (f *SliceFile) IsDirectory() bool { return true } -func (f *SliceFile) NextFile() (File, error) { +func (f *SliceFile) NextFile() (string, File, error) { if f.n >= len(f.files) { - return nil, io.EOF + return "", nil, io.EOF } file := f.files[f.n] f.n++ - return file, nil -} - -func (f *SliceFile) FileName() string { - return f.filename -} - -func (f *SliceFile) FullPath() string { - return f.path + return file.Name, file.File, nil } func (f *SliceFile) Read(p []byte) (int, error) { @@ -48,8 +42,8 @@ func (f *SliceFile) Close() error { return ErrNotReader } -func (f *SliceFile) Peek(n int) File { - return f.files[n] +func (f *SliceFile) Seek(offset int64, whence int) (int64, error) { + return 0, ErrNotReader } func (f *SliceFile) Length() int { @@ -60,12 +54,7 @@ func (f *SliceFile) Size() (int64, error) { var size int64 for _, file := range f.files { - sizeFile, ok := file.(SizeFile) - if !ok { - return 0, errors.New("Could not get size of child file") - } - - s, err := sizeFile.Size() + s, err := file.File.Size() if err != nil { return 0, err } diff --git a/files/webfile.go b/files/webfile.go index fcf4412ea..65d1c1ee0 100644 --- a/files/webfile.go +++ b/files/webfile.go @@ -4,7 +4,6 @@ import ( "io" "net/http" "net/url" - "path/filepath" ) // WebFile is an implementation of File which reads it @@ -46,17 +45,6 @@ func (wf *WebFile) Close() error { return wf.body.Close() } -// FullPath returns the "Host+Path" for this WebFile. -func (wf *WebFile) FullPath() string { - return wf.url.Host + wf.url.Path -} - -// FileName returns the last element of the URL -// path for this file. -func (wf *WebFile) FileName() string { - return filepath.Base(wf.url.Path) -} - // IsDirectory returns false. func (wf *WebFile) IsDirectory() bool { return false @@ -66,3 +54,8 @@ func (wf *WebFile) IsDirectory() bool { func (wf *WebFile) NextFile() (File, error) { return nil, ErrNotDirectory } + +// TODO: implement +func (wf *WebFile) Seek(offset int64, whence int) (int64, error) { + return 0, ErrNotSupported +} From 239810eb4d8cf609131665b7e70acfd9cd476b52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 24 Oct 2018 16:48:13 +0200 Subject: [PATCH 12/76] Allow Close on dirs, make sure errors make sense This commit was moved from ipfs/go-ipfs-files@cfdeccb3723af80fe98b9b3103eff1501527b6cc --- files/file.go | 19 ++++++++++++++----- files/file_test.go | 10 +++++----- files/multifilereader.go | 8 ++++++-- files/multipartfile.go | 7 ++----- files/readerfile.go | 3 +-- files/serialfile.go | 2 +- files/slicefile.go | 4 ++-- 7 files changed, 31 insertions(+), 22 deletions(-) diff --git a/files/file.go b/files/file.go index ceb60dded..daa30b817 100644 --- a/files/file.go +++ b/files/file.go @@ -15,11 +15,10 @@ var ( // File is an interface that provides functionality for handling // files/directories as values that can be supplied to commands. For -// directories, child files are accessed serially by calling `Files()` -// or `Walk()`. +// directories, child files are accessed serially by calling `NextFile()` // -// Read/Seek/Close methods are only valid for files -// Files/Walk methods are only valid for directories +// Read/Seek methods are only valid for files +// NextFile method is only valid for directories type File interface { io.Reader io.Closer @@ -37,6 +36,16 @@ type File interface { // directory). It will return io.EOF if no more files are // available. If the file is a regular file (not a directory), NextFile // will return a non-nil error. + // + // Note: + // - Some implementations may only allow reading in order - if a + // child directory is returned, you need to read all it's children + // first before calling NextFile on parent again. Before doing parallel + // reading or reading entire level at once, make sure the implementation + // you are using allows that + // - Returned files may not be sorted + // - Depending on implementation it may not be safe to iterate multiple + // children in parallel NextFile() (string, File, error) } @@ -44,7 +53,7 @@ type File interface { type FileInfo interface { File - // AbsPath returns full/real file path. + // AbsPath returns full real file path. AbsPath() string // Stat returns os.Stat of this file diff --git a/files/file_test.go b/files/file_test.go index 60d1bc482..e68540c84 100644 --- a/files/file_test.go +++ b/files/file_test.go @@ -22,12 +22,12 @@ func TestSliceFiles(t *testing.T) { t.Fatal("SliceFile should always be a directory") } - if n, err := sf.Read(buf); n > 0 || err != io.EOF { + if n, err := sf.Read(buf); n > 0 || err != ErrNotReader { t.Fatal("Shouldn't be able to read data from a SliceFile") } - if err := sf.Close(); err != ErrNotReader { - t.Fatal("Shouldn't be able to call `Close` on a SliceFile") + if err := sf.Close(); err != nil { + t.Fatal("Should be able to call `Close` on a SliceFile") } _, file, err := sf.NextFile() @@ -151,8 +151,8 @@ anotherfile if n, err := mpf.Read(buf); n > 0 || err != ErrNotReader { t.Fatal("Shouldn't be able to call `Read` on a directory") } - if err := mpf.Close(); err != ErrNotReader { - t.Fatal("Shouldn't be able to call `Close` on a directory") + if err := mpf.Close(); err != nil { + t.Fatal("Should be able to call `Close` on a directory") } // test properties of file created from third part (nested file) diff --git a/files/multifilereader.go b/files/multifilereader.go index 7863c9d61..85e66e717 100644 --- a/files/multifilereader.go +++ b/files/multifilereader.go @@ -20,7 +20,7 @@ type MultiFileReader struct { files []File path []string - currentFile io.Reader + currentFile File buf bytes.Buffer mpWriter *multipart.Writer closed bool @@ -121,7 +121,11 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { // otherwise, read from file data written, err = mfr.currentFile.Read(buf) - if err == io.EOF { + if err == io.EOF || err == ErrNotReader { + if err := mfr.currentFile.Close(); err != nil { + return written, err + } + mfr.currentFile = nil return written, nil } diff --git a/files/multipartfile.go b/files/multipartfile.go index a3041638b..cc94f1714 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -136,9 +136,6 @@ func (f *MultipartFile) Read(p []byte) (int, error) { } func (f *MultipartFile) Close() error { - if f.IsDirectory() { - return ErrNotReader - } return f.Part.Close() } @@ -146,11 +143,11 @@ func (f *MultipartFile) Seek(offset int64, whence int) (int64, error) { if f.IsDirectory() { return 0, ErrNotReader } - return 0, ErrNotReader + return 0, ErrNotSupported } func (f *MultipartFile) Size() (int64, error) { - return 0, ErrNotReader + return 0, ErrNotSupported } type PartReader interface { diff --git a/files/readerfile.go b/files/readerfile.go index e40032c8c..070ee97ef 100644 --- a/files/readerfile.go +++ b/files/readerfile.go @@ -1,7 +1,6 @@ package files import ( - "errors" "io" "os" "path/filepath" @@ -54,7 +53,7 @@ func (f *ReaderFile) Stat() os.FileInfo { func (f *ReaderFile) Size() (int64, error) { if f.stat == nil { - return 0, errors.New("file size unknown") + return 0, ErrNotSupported } return f.stat.Size(), nil } diff --git a/files/serialfile.go b/files/serialfile.go index eb4197674..71855d52a 100644 --- a/files/serialfile.go +++ b/files/serialfile.go @@ -98,7 +98,7 @@ func (f *serialFile) NextFile() (string, File, error) { } func (f *serialFile) Read(p []byte) (int, error) { - return 0, io.EOF + return 0, ErrNotReader } func (f *serialFile) Close() error { diff --git a/files/slicefile.go b/files/slicefile.go index d356465b5..5fea5a60f 100644 --- a/files/slicefile.go +++ b/files/slicefile.go @@ -35,11 +35,11 @@ func (f *SliceFile) NextFile() (string, File, error) { } func (f *SliceFile) Read(p []byte) (int, error) { - return 0, io.EOF + return 0, ErrNotReader } func (f *SliceFile) Close() error { - return ErrNotReader + return nil } func (f *SliceFile) Seek(offset int64, whence int) (int64, error) { From 49d7a0e0dd0320d2c8d1ba4d4a310d7515ce5585 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 5 Nov 2018 17:05:53 +0100 Subject: [PATCH 13/76] fix IsHidden on windows This commit was moved from ipfs/go-ipfs-files@b0b422eb2d7f4defd547e4cb93fbc0fe2023c0b7 --- files/is_hidden_windows.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/is_hidden_windows.go b/files/is_hidden_windows.go index 40f40ae62..6f9568af2 100644 --- a/files/is_hidden_windows.go +++ b/files/is_hidden_windows.go @@ -11,7 +11,7 @@ import ( func IsHidden(name string, f File) bool { - fName := filepath.Base(f.FileName()) + fName := filepath.Base(name) if strings.HasPrefix(fName, ".") && len(fName) > 1 { return true From 8ec0a67baee3c7e8345d8c732cb13af26f72d8f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 7 Nov 2018 15:49:42 +0100 Subject: [PATCH 14/76] Separate file/directory types This commit was moved from ipfs/go-ipfs-files@97852d87e92fd8484e9fcf0e9b2b1c942849e44d --- files/file.go | 29 +++++++++------ files/file_test.go | 66 +++++++++++++---------------------- files/linkfile.go | 14 ++------ files/multifilereader.go | 36 +++++++++++-------- files/multifilereader_test.go | 36 ++++++++++--------- files/multipartfile.go | 38 +++++++++----------- files/readerfile.go | 13 +++---- files/serialfile.go | 18 ++-------- files/slicefile.go | 16 ++------- files/webfile.go | 24 +++++++------ 10 files changed, 127 insertions(+), 163 deletions(-) diff --git a/files/file.go b/files/file.go index daa30b817..b2d9712b4 100644 --- a/files/file.go +++ b/files/file.go @@ -7,8 +7,8 @@ import ( ) var ( - ErrNotDirectory = errors.New("couldn't call NextFile(), this isn't a directory") - ErrNotReader = errors.New("this file is a directory, can't use Reader functions") + ErrNotDirectory = errors.New("file isn't a directory") + ErrNotReader = errors.New("file isn't a regular file") ErrNotSupported = errors.New("operation not supported") ) @@ -20,22 +20,29 @@ var ( // Read/Seek methods are only valid for files // NextFile method is only valid for directories type File interface { - io.Reader io.Closer - io.Seeker - // Size returns size of the + // Size returns size of this file (if this file is a directory, total size of + // all files stored in the tree should be returned). Some implementations may + // choose not to implement this Size() (int64, error) +} + +// Regular represents the regular Unix file +type Regular interface { + File - // IsDirectory returns true if the File is a directory (and therefore - // supports calling `Files`/`Walk`) and false if the File is a normal file - // (and therefore supports calling `Read`/`Close`/`Seek`) - IsDirectory() bool + io.Reader + io.Seeker +} + +// Directory is a special file which can link to any number of files +type Directory interface { + File // NextFile returns the next child file available (if the File is a // directory). It will return io.EOF if no more files are - // available. If the file is a regular file (not a directory), NextFile - // will return a non-nil error. + // available. // // Note: // - Some implementations may only allow reading in order - if a diff --git a/files/file_test.go b/files/file_test.go index e68540c84..9b1308270 100644 --- a/files/file_test.go +++ b/files/file_test.go @@ -18,23 +18,15 @@ func TestSliceFiles(t *testing.T) { sf := NewSliceFile(files) - if !sf.IsDirectory() { - t.Fatal("SliceFile should always be a directory") - } - - if n, err := sf.Read(buf); n > 0 || err != ErrNotReader { - t.Fatal("Shouldn't be able to read data from a SliceFile") - } - - if err := sf.Close(); err != nil { - t.Fatal("Should be able to call `Close` on a SliceFile") - } - _, file, err := sf.NextFile() if file == nil || err != nil { t.Fatal("Expected a file and nil error") } - read, err := file.Read(buf) + rf, ok := file.(Regular) + if !ok { + t.Fatal("Expected a regular file") + } + read, err := rf.Read(buf) if read != 11 || err != nil { t.Fatal("NextFile got a file in the wrong order") } @@ -52,6 +44,10 @@ func TestSliceFiles(t *testing.T) { if file != nil || err != io.EOF { t.Fatal("Expected a nil file and io.EOF") } + + if err := sf.Close(); err != nil { + t.Fatal("Should be able to call `Close` on a SliceFile") + } } func TestReaderFiles(t *testing.T) { @@ -59,14 +55,6 @@ func TestReaderFiles(t *testing.T) { rf := NewReaderFile(ioutil.NopCloser(strings.NewReader(message)), nil) buf := make([]byte, len(message)) - if rf.IsDirectory() { - t.Fatal("ReaderFile should never be a directory") - } - _, file, err := rf.NextFile() - if file != nil || err != ErrNotDirectory { - t.Fatal("Expected a nil file and ErrNotDirectory") - } - if n, err := rf.Read(buf); n == 0 || err != nil { t.Fatal("Expected to be able to read") } @@ -117,19 +105,17 @@ anotherfile if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } - if mpf.IsDirectory() { + mf, ok := mpf.(Regular) + if !ok { t.Fatal("Expected file to not be a directory") } if mpname != "name" { t.Fatal("Expected filename to be \"name\"") } - if _, file, err := mpf.NextFile(); file != nil || err != ErrNotDirectory { - t.Fatal("Expected a nil file and ErrNotDirectory") - } - if n, err := mpf.Read(buf); n != 4 || !(err == io.EOF || err == nil) { + if n, err := mf.Read(buf); n != 4 || !(err == io.EOF || err == nil) { t.Fatal("Expected to be able to read 4 bytes", n, err) } - if err := mpf.Close(); err != nil { + if err := mf.Close(); err != nil { t.Fatal("Expected to be able to close file") } @@ -142,16 +128,14 @@ anotherfile if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } - if !mpf.IsDirectory() { + md, ok := mpf.(Directory) + if !ok { t.Fatal("Expected file to be a directory") } if mpname != "dir" { t.Fatal("Expected filename to be \"dir\"") } - if n, err := mpf.Read(buf); n > 0 || err != ErrNotReader { - t.Fatal("Shouldn't be able to call `Read` on a directory") - } - if err := mpf.Close(); err != nil { + if err := md.Close(); err != nil { t.Fatal("Should be able to call `Close` on a directory") } @@ -164,13 +148,14 @@ anotherfile if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } - if mpf.IsDirectory() { - t.Fatal("Expected file, got directory") + mf, ok = mpf.(Regular) + if !ok { + t.Fatal("Expected file to not be a directory") } if mpname != "nested" { t.Fatalf("Expected filename to be \"nested\", got %s", mpname) } - if n, err := mpf.Read(buf); n != 12 || !(err == nil || err == io.EOF) { + if n, err := mf.Read(buf); n != 12 || !(err == nil || err == io.EOF) { t.Fatalf("expected to be able to read 12 bytes from file: %s (got %d)", err, n) } if err := mpf.Close(); err != nil { @@ -186,17 +171,14 @@ anotherfile if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } - if mpf.IsDirectory() { - t.Fatal("Expected file to be a symlink") + ms, ok := mpf.(*Symlink) + if !ok { + t.Fatal("Expected file to not be a directory") } if mpname != "simlynk" { t.Fatal("Expected filename to be \"dir/simlynk\"") } - slink, ok := mpf.(*Symlink) - if !ok { - t.Fatalf("expected file to be a symlink") - } - if slink.Target != "anotherfile" { + if ms.Target != "anotherfile" { t.Fatal("expected link to point to anotherfile") } } diff --git a/files/linkfile.go b/files/linkfile.go index 0182d3969..df334d574 100644 --- a/files/linkfile.go +++ b/files/linkfile.go @@ -7,30 +7,20 @@ import ( ) type Symlink struct { - path string Target string stat os.FileInfo reader io.Reader } -func NewLinkFile(path, target string, stat os.FileInfo) File { +func NewLinkFile(target string, stat os.FileInfo) Regular { return &Symlink{ - path: path, Target: target, stat: stat, reader: strings.NewReader(target), } } -func (lf *Symlink) IsDirectory() bool { - return false -} - -func (lf *Symlink) NextFile() (string, File, error) { - return "", nil, ErrNotDirectory -} - func (lf *Symlink) Close() error { if c, ok := lf.reader.(io.Closer); ok { return c.Close() @@ -54,3 +44,5 @@ func (lf *Symlink) Seek(offset int64, whence int) (int64, error) { func (lf *Symlink) Size() (int64, error) { return 0, ErrNotSupported } + +var _ Regular = &Symlink{} diff --git a/files/multifilereader.go b/files/multifilereader.go index 85e66e717..b0ce1d97d 100644 --- a/files/multifilereader.go +++ b/files/multifilereader.go @@ -17,7 +17,7 @@ type MultiFileReader struct { io.Reader // directory stack for NextFile - files []File + files []Directory path []string currentFile File @@ -31,12 +31,12 @@ type MultiFileReader struct { form bool } -// NewMultiFileReader constructs a MultiFileReader. `file` can be any `commands.File`. +// NewMultiFileReader constructs a MultiFileReader. `file` can be any `commands.Directory`. // If `form` is set to true, the multipart data will have a Content-Type of 'multipart/form-data', // if `form` is false, the Content-Type will be 'multipart/mixed'. -func NewMultiFileReader(file File, form bool) *MultiFileReader { +func NewMultiFileReader(file Directory, form bool) *MultiFileReader { mfr := &MultiFileReader{ - files: []File{file}, + files: []Directory{file}, path: []string{""}, form: form, mutex: &sync.Mutex{}, @@ -91,15 +91,19 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { header.Set("Content-Disposition", fmt.Sprintf("file; filename=\"%s\"", filename)) var contentType string - if _, ok := file.(*Symlink); ok { + + switch f := file.(type) { + case *Symlink: contentType = "application/symlink" - } else if file.IsDirectory() { - mfr.files = append(mfr.files, file) + case Directory: + mfr.files = append(mfr.files, f) mfr.path = append(mfr.path, name) contentType = "application/x-directory" - } else { + case Regular: // otherwise, use the file as a reader to read its contents contentType = "application/octet-stream" + default: + return 0, ErrNotSupported } header.Set("Content-Type", contentType) @@ -120,16 +124,20 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { } // otherwise, read from file data - written, err = mfr.currentFile.Read(buf) - if err == io.EOF || err == ErrNotReader { - if err := mfr.currentFile.Close(); err != nil { + switch f := mfr.currentFile.(type) { + case Regular: + written, err = f.Read(buf) + if err != io.EOF { return written, err } + } - mfr.currentFile = nil - return written, nil + if err := mfr.currentFile.Close(); err != nil { + return written, err } - return written, err + + mfr.currentFile = nil + return written, nil } // Boundary returns the boundary string to be used to separate files in the multipart data diff --git a/files/multifilereader_test.go b/files/multifilereader_test.go index 18876ef00..5f0cac50d 100644 --- a/files/multifilereader_test.go +++ b/files/multifilereader_test.go @@ -33,46 +33,48 @@ func TestMultiFileReaderToMultiFile(t *testing.T) { t.Fatal(err) } - if !mf.IsDirectory() { + md, ok := mf.(Directory) + if !ok { t.Fatal("Expected a directory") } - fn, f, err := mf.NextFile() + fn, f, err := md.NextFile() if fn != "file.txt" || f == nil || err != nil { t.Fatal("NextFile returned unexpected data") } - dn, d, err := mf.NextFile() + dn, d, err := md.NextFile() if dn != "boop" || d == nil || err != nil { t.Fatal("NextFile returned unexpected data") } - if !d.IsDirectory() { + df, ok := d.(Directory) + if !ok { t.Fatal("Expected a directory") } - cfn, cf, err := d.NextFile() + cfn, cf, err := df.NextFile() if cfn != "a.txt" || cf == nil || err != nil { t.Fatal("NextFile returned unexpected data") } - cfn, cf, err = d.NextFile() + cfn, cf, err = df.NextFile() if cfn != "b.txt" || cf == nil || err != nil { t.Fatal("NextFile returned unexpected data") } - cfn, cf, err = d.NextFile() + cfn, cf, err = df.NextFile() if cfn != "" || cf != nil || err != io.EOF { t.Fatal("NextFile returned unexpected data") } // try to break internal state - cfn, cf, err = d.NextFile() + cfn, cf, err = df.NextFile() if cfn != "" || cf != nil || err != io.EOF { t.Fatal("NextFile returned unexpected data") } - fn, f, err = mf.NextFile() + fn, f, err = md.NextFile() if fn != "beep.txt" || f == nil || err != nil { t.Fatal("NextFile returned unexpected data") } @@ -91,13 +93,14 @@ func TestOutput(t *testing.T) { if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } - if mpf.IsDirectory() { - t.Fatal("Expected file to not be a directory") + mpr, ok := mpf.(Regular) + if !ok { + t.Fatal("Expected file to be a regular file") } if mpname != "file.txt" { t.Fatal("Expected filename to be \"file.txt\"") } - if n, err := mpf.Read(buf); n != len(text) || err != nil { + if n, err := mpr.Read(buf); n != len(text) || err != nil { t.Fatal("Expected to read from file", n, err) } if string(buf[:len(text)]) != text { @@ -112,7 +115,8 @@ func TestOutput(t *testing.T) { if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } - if !mpf.IsDirectory() { + mpd, ok := mpf.(Directory) + if !ok { t.Fatal("Expected file to be a directory") } if mpname != "boop" { @@ -127,7 +131,7 @@ func TestOutput(t *testing.T) { if child == nil || err != nil { t.Fatal("Expected to be able to read a child file") } - if child.IsDirectory() { + if _, ok := child.(Regular); !ok { t.Fatal("Expected file to not be a directory") } if cname != "a.txt" { @@ -142,14 +146,14 @@ func TestOutput(t *testing.T) { if child == nil || err != nil { t.Fatal("Expected to be able to read a child file") } - if child.IsDirectory() { + if _, ok := child.(Regular); !ok { t.Fatal("Expected file to not be a directory") } if cname != "b.txt" { t.Fatal("Expected filename to be \"b.txt\"") } - cname, child, err = mpf.NextFile() + cname, child, err = mpd.NextFile() if child != nil || err != io.EOF { t.Fatal("Expected to get (nil, io.EOF)") } diff --git a/files/multipartfile.go b/files/multipartfile.go index cc94f1714..5c62c2cde 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -34,6 +34,10 @@ type MultipartFile struct { } func NewFileFromPartReader(reader *multipart.Reader, mediatype string) (File, error) { + if !isDirectory(mediatype) { + return nil, ErrNotDirectory + } + f := &MultipartFile{ Reader: &peekReader{r: reader}, Mediatype: mediatype, @@ -61,9 +65,7 @@ func newFileFromPart(parent string, part *multipart.Part, reader PartReader) (st return "", nil, err } - return base, &Symlink{ - Target: string(out), - }, nil + return base, NewLinkFile(string(out), nil), nil case "": // default to application/octet-stream fallthrough case applicationFile: @@ -79,17 +81,21 @@ func newFileFromPart(parent string, part *multipart.Part, reader PartReader) (st return "", nil, err } + if !isDirectory(f.Mediatype) { + return base, &ReaderFile{ + reader: part, + abspath: part.Header.Get("abspath"), + }, nil + } + return base, f, nil } -func (f *MultipartFile) IsDirectory() bool { - return f.Mediatype == multipartFormdataType || f.Mediatype == applicationDirectory +func isDirectory(mediatype string) bool { + return mediatype == multipartFormdataType || mediatype == applicationDirectory } func (f *MultipartFile) NextFile() (string, File, error) { - if !f.IsDirectory() { - return "", nil, ErrNotDirectory - } if f.Reader == nil { return "", nil, io.EOF } @@ -128,24 +134,10 @@ func (f *MultipartFile) fileName() string { return filename } -func (f *MultipartFile) Read(p []byte) (int, error) { - if f.IsDirectory() { - return 0, ErrNotReader - } - return f.Part.Read(p) -} - func (f *MultipartFile) Close() error { return f.Part.Close() } -func (f *MultipartFile) Seek(offset int64, whence int) (int64, error) { - if f.IsDirectory() { - return 0, ErrNotReader - } - return 0, ErrNotSupported -} - func (f *MultipartFile) Size() (int64, error) { return 0, ErrNotSupported } @@ -176,3 +168,5 @@ func (pr *peekReader) put(p *multipart.Part) error { pr.next = p return nil } + +var _ Directory = &MultipartFile{} diff --git a/files/readerfile.go b/files/readerfile.go index 070ee97ef..7db7e966c 100644 --- a/files/readerfile.go +++ b/files/readerfile.go @@ -14,7 +14,7 @@ type ReaderFile struct { stat os.FileInfo } -func NewReaderFile(reader io.ReadCloser, stat os.FileInfo) File { +func NewReaderFile(reader io.ReadCloser, stat os.FileInfo) Regular { return &ReaderFile{"", reader, stat} } @@ -27,14 +27,6 @@ func NewReaderPathFile(path string, reader io.ReadCloser, stat os.FileInfo) (*Re return &ReaderFile{abspath, reader, stat}, nil } -func (f *ReaderFile) IsDirectory() bool { - return false -} - -func (f *ReaderFile) NextFile() (string, File, error) { - return "", nil, ErrNotDirectory -} - func (f *ReaderFile) AbsPath() string { return f.abspath } @@ -65,3 +57,6 @@ func (f *ReaderFile) Seek(offset int64, whence int) (int64, error) { return 0, ErrNotSupported } + +var _ Regular = &ReaderFile{} +var _ FileInfo = &ReaderFile{} diff --git a/files/serialfile.go b/files/serialfile.go index 71855d52a..666399d29 100644 --- a/files/serialfile.go +++ b/files/serialfile.go @@ -40,18 +40,12 @@ func NewSerialFile(path string, hidden bool, stat os.FileInfo) (File, error) { if err != nil { return nil, err } - return NewLinkFile(path, target, stat), nil + return NewLinkFile(target, stat), nil default: return nil, fmt.Errorf("unrecognized file type for %s: %s", path, mode.String()) } } -func (f *serialFile) IsDirectory() bool { - // non-directories get created as a ReaderFile, so serialFiles should only - // represent directories - return true -} - func (f *serialFile) NextFile() (string, File, error) { // if a file was opened previously, close it err := f.Close() @@ -97,18 +91,10 @@ func (f *serialFile) NextFile() (string, File, error) { return stat.Name(), sf, nil } -func (f *serialFile) Read(p []byte) (int, error) { - return 0, ErrNotReader -} - func (f *serialFile) Close() error { return nil } -func (f *serialFile) Seek(offset int64, whence int) (int64, error) { - return 0, ErrNotReader -} - func (f *serialFile) Stat() os.FileInfo { return f.stat } @@ -132,3 +118,5 @@ func (f *serialFile) Size() (int64, error) { return du, err } + +var _ Directory = &serialFile{} diff --git a/files/slicefile.go b/files/slicefile.go index 5fea5a60f..0d78e16c6 100644 --- a/files/slicefile.go +++ b/files/slicefile.go @@ -17,14 +17,10 @@ type SliceFile struct { n int } -func NewSliceFile(files []FileEntry) File { +func NewSliceFile(files []FileEntry) Directory { return &SliceFile{files, 0} } -func (f *SliceFile) IsDirectory() bool { - return true -} - func (f *SliceFile) NextFile() (string, File, error) { if f.n >= len(f.files) { return "", nil, io.EOF @@ -34,18 +30,10 @@ func (f *SliceFile) NextFile() (string, File, error) { return file.Name, file.File, nil } -func (f *SliceFile) Read(p []byte) (int, error) { - return 0, ErrNotReader -} - func (f *SliceFile) Close() error { return nil } -func (f *SliceFile) Seek(offset int64, whence int) (int64, error) { - return 0, ErrNotReader -} - func (f *SliceFile) Length() int { return len(f.files) } @@ -63,3 +51,5 @@ func (f *SliceFile) Size() (int64, error) { return size, nil } + +var _ Directory = &SliceFile{} diff --git a/files/webfile.go b/files/webfile.go index 65d1c1ee0..5638f770a 100644 --- a/files/webfile.go +++ b/files/webfile.go @@ -1,6 +1,7 @@ package files import ( + "github.com/pkg/errors" "io" "net/http" "net/url" @@ -12,6 +13,7 @@ import ( type WebFile struct { body io.ReadCloser url *url.URL + contentLength int64 } // NewWebFile creates a WebFile with the given URL, which @@ -33,6 +35,7 @@ func (wf *WebFile) Read(b []byte) (int, error) { return 0, err } wf.body = resp.Body + wf.contentLength = resp.ContentLength } return wf.body.Read(b) } @@ -45,17 +48,18 @@ func (wf *WebFile) Close() error { return wf.body.Close() } -// IsDirectory returns false. -func (wf *WebFile) IsDirectory() bool { - return false -} - -// NextFile always returns an ErrNotDirectory error. -func (wf *WebFile) NextFile() (File, error) { - return nil, ErrNotDirectory -} - // TODO: implement func (wf *WebFile) Seek(offset int64, whence int) (int64, error) { return 0, ErrNotSupported } + +func (wf *WebFile) Size() (int64, error) { + if wf.contentLength < 0 { + return -1, errors.New("Content-Length hearer was not set") + } + + return wf.contentLength, nil +} + + +var _ Regular = &WebFile{} From 8e6f6dbff849fca44ebaacfa4264f71781490f0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 7 Nov 2018 18:53:12 +0100 Subject: [PATCH 15/76] Directory iterators This commit was moved from ipfs/go-ipfs-files@2b34e278cd80cdb924b4b28bf434c4c084ab61fc --- files/file.go | 75 ++++++++++++++++++++++----- files/file_test.go | 37 +++++++------ files/multifilereader.go | 46 +++++++++------- files/multifilereader_test.go | 77 +++++++++++++-------------- files/multipartfile.go | 69 ++++++++++++++++++------ files/serialfile.go | 98 ++++++++++++++++++++++++++++++----- files/slicefile.go | 80 +++++++++++++++++++++------- files/webfile.go | 5 +- 8 files changed, 348 insertions(+), 139 deletions(-) diff --git a/files/file.go b/files/file.go index b2d9712b4..693baab82 100644 --- a/files/file.go +++ b/files/file.go @@ -28,7 +28,7 @@ type File interface { Size() (int64, error) } -// Regular represents the regular Unix file +// Regular represents a regular Unix file type Regular interface { File @@ -36,24 +36,71 @@ type Regular interface { io.Seeker } -// Directory is a special file which can link to any number of files +// DirEntry exposes information about a directory entry +type DirEntry interface { + // Name returns the base name of this entry, which is the base name of + // the referenced file + Name() string + + // File returns the file referenced by this DirEntry + File() File + + // Regular is an alias for ent.File().(Regular). If the file isn't a regular + // file, nil value will be returned + Regular() Regular + + // Dir is an alias for ent.File().(directory). If the file isn't a directory, + // nil value will be returned + Dir() Directory +} + +// DirIterator is a iterator over directory entries. +// See Directory.Entries for more +type DirIterator interface { + // DirEntry holds information about current directory entry. + // Note that after creating new iterator you MUST call Next() at least once + // before accessing these methods. Calling these methods without prior calls + // to Next() and after Next() returned false may result in undefined behavior + DirEntry + + // Next advances the iterator to the next file. + Next() bool + + // Err may return an error after the previous call to Next() returned `false`. + // If the previous call to Next() returned `true`, Err() is guaranteed to + // return nil + Err() error +} + +// Directory is a special file which can link to any number of files. type Directory interface { File - // NextFile returns the next child file available (if the File is a - // directory). It will return io.EOF if no more files are - // available. + // Entries returns a stateful iterator over directory entries. + // + // Example usage: + // + // it := dir.Entries() + // for it.Next() { + // name := it.Name() + // file := it.File() + // [...] + // } + // if it.Err() != nil { + // return err + // } // // Note: - // - Some implementations may only allow reading in order - if a - // child directory is returned, you need to read all it's children - // first before calling NextFile on parent again. Before doing parallel - // reading or reading entire level at once, make sure the implementation - // you are using allows that - // - Returned files may not be sorted + // - Below limitations aren't applicable to all implementations, consult + // your implementations manual before using this interface in a way that + // doesn't meet these constraints + // - Some implementations may only allow reading in order - so if the iterator + // returns a directory you must iterate over it's entries first before + // calling Next again + // - Order is not guaranteed // - Depending on implementation it may not be safe to iterate multiple - // children in parallel - NextFile() (string, File, error) + // 'branches' in parallel + Entries() (DirIterator, error) } // FileInfo exposes information on files in local filesystem @@ -63,6 +110,6 @@ type FileInfo interface { // AbsPath returns full real file path. AbsPath() string - // Stat returns os.Stat of this file + // Stat returns os.Stat of this file, may be nil for some files Stat() os.FileInfo } diff --git a/files/file_test.go b/files/file_test.go index 9b1308270..cbf227c08 100644 --- a/files/file_test.go +++ b/files/file_test.go @@ -9,21 +9,24 @@ import ( ) func TestSliceFiles(t *testing.T) { - files := []FileEntry{ - {NewReaderFile(ioutil.NopCloser(strings.NewReader("Some text!\n")), nil), ""}, - {NewReaderFile(ioutil.NopCloser(strings.NewReader("beep")), nil), ""}, - {NewReaderFile(ioutil.NopCloser(strings.NewReader("boop")), nil), ""}, + files := []DirEntry{ + FileEntry("", NewReaderFile(ioutil.NopCloser(strings.NewReader("Some text!\n")), nil)), + FileEntry("", NewReaderFile(ioutil.NopCloser(strings.NewReader("beep")), nil)), + FileEntry("", NewReaderFile(ioutil.NopCloser(strings.NewReader("boop")), nil)), } buf := make([]byte, 20) sf := NewSliceFile(files) + it, err := sf.Entries() + if err != nil { + t.Fatal(err) + } - _, file, err := sf.NextFile() - if file == nil || err != nil { - t.Fatal("Expected a file and nil error") + if !it.Next() { + t.Fatal("Expected a file") } - rf, ok := file.(Regular) - if !ok { + rf := it.Regular() + if rf == nil { t.Fatal("Expected a regular file") } read, err := rf.Read(buf) @@ -31,18 +34,14 @@ func TestSliceFiles(t *testing.T) { t.Fatal("NextFile got a file in the wrong order") } - _, file, err = sf.NextFile() - if file == nil || err != nil { - t.Fatal("Expected a file and nil error") + if !it.Next() { + t.Fatal("Expected a file") } - _, file, err = sf.NextFile() - if file == nil || err != nil { - t.Fatal("Expected a file and nil error") + if !it.Next() { + t.Fatal("Expected a file") } - - _, file, err = sf.NextFile() - if file != nil || err != io.EOF { - t.Fatal("Expected a nil file and io.EOF") + if it.Next() { + t.Fatal("Wild file appeared!") } if err := sf.Close(); err != nil { diff --git a/files/multifilereader.go b/files/multifilereader.go index b0ce1d97d..8817ad815 100644 --- a/files/multifilereader.go +++ b/files/multifilereader.go @@ -17,7 +17,7 @@ type MultiFileReader struct { io.Reader // directory stack for NextFile - files []Directory + files []DirIterator path []string currentFile File @@ -34,16 +34,21 @@ type MultiFileReader struct { // NewMultiFileReader constructs a MultiFileReader. `file` can be any `commands.Directory`. // If `form` is set to true, the multipart data will have a Content-Type of 'multipart/form-data', // if `form` is false, the Content-Type will be 'multipart/mixed'. -func NewMultiFileReader(file Directory, form bool) *MultiFileReader { +func NewMultiFileReader(file Directory, form bool) (*MultiFileReader, error) { + it, err := file.Entries() + if err != nil { + return nil, err + } + mfr := &MultiFileReader{ - files: []Directory{file}, + files: []DirIterator{it}, path: []string{""}, form: form, mutex: &sync.Mutex{}, } mfr.mpWriter = multipart.NewWriter(&mfr.buf) - return mfr + return mfr, nil } func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { @@ -57,47 +62,50 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { // if the current file isn't set, advance to the next file if mfr.currentFile == nil { - var file File - var name string + var entry DirEntry - for file == nil { + for entry == nil { if len(mfr.files) == 0 { mfr.mpWriter.Close() mfr.closed = true return mfr.buf.Read(buf) } - nextName, nextFile, err := mfr.files[len(mfr.files)-1].NextFile() - if err == io.EOF { + if !mfr.files[len(mfr.files)-1].Next() { mfr.files = mfr.files[:len(mfr.files)-1] mfr.path = mfr.path[:len(mfr.path)-1] continue - } else if err != nil { - return 0, err + } + if mfr.files[len(mfr.files)-1].Err() != nil { + return 0, mfr.files[len(mfr.files)-1].Err() } - file = nextFile - name = nextName + entry = mfr.files[len(mfr.files)-1] } // handle starting a new file part if !mfr.closed { - mfr.currentFile = file + mfr.currentFile = entry.File() // write the boundary and headers header := make(textproto.MIMEHeader) - filename := url.QueryEscape(path.Join(path.Join(mfr.path...), name)) + filename := url.QueryEscape(path.Join(path.Join(mfr.path...), entry.Name())) header.Set("Content-Disposition", fmt.Sprintf("file; filename=\"%s\"", filename)) var contentType string - switch f := file.(type) { + switch f := entry.File().(type) { case *Symlink: contentType = "application/symlink" case Directory: - mfr.files = append(mfr.files, f) - mfr.path = append(mfr.path, name) + newIt, err := f.Entries() + if err != nil { + return 0, err + } + + mfr.files = append(mfr.files, newIt) + mfr.path = append(mfr.path, entry.Name()) contentType = "application/x-directory" case Regular: // otherwise, use the file as a reader to read its contents @@ -107,7 +115,7 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { } header.Set("Content-Type", contentType) - if rf, ok := file.(FileInfo); ok { + if rf, ok := entry.File().(FileInfo); ok { header.Set("abspath", rf.AbsPath()) } diff --git a/files/multifilereader_test.go b/files/multifilereader_test.go index 5f0cac50d..dd7dac9e4 100644 --- a/files/multifilereader_test.go +++ b/files/multifilereader_test.go @@ -10,23 +10,27 @@ import ( var text = "Some text! :)" -func getTestMultiFileReader() *MultiFileReader { - fileset := []FileEntry{ - {NewReaderFile(ioutil.NopCloser(strings.NewReader(text)), nil), "file.txt"}, - {NewSliceFile([]FileEntry{ - {NewReaderFile(ioutil.NopCloser(strings.NewReader("bleep")), nil), "a.txt"}, - {NewReaderFile(ioutil.NopCloser(strings.NewReader("bloop")), nil), "b.txt"}, - }), "boop"}, - {NewReaderFile(ioutil.NopCloser(strings.NewReader("beep")), nil), "beep.txt"}, +func getTestMultiFileReader(t *testing.T) *MultiFileReader { + fileset := []DirEntry{ + FileEntry("file.txt", NewReaderFile(ioutil.NopCloser(strings.NewReader(text)), nil)), + FileEntry("boop", NewSliceFile([]DirEntry{ + FileEntry("a.txt", NewReaderFile(ioutil.NopCloser(strings.NewReader("bleep")), nil)), + FileEntry("b.txt", NewReaderFile(ioutil.NopCloser(strings.NewReader("bloop")), nil)), + })), + FileEntry("beep.txt", NewReaderFile(ioutil.NopCloser(strings.NewReader("beep")), nil)), } sf := NewSliceFile(fileset) // testing output by reading it with the go stdlib "mime/multipart" Reader - return NewMultiFileReader(sf, true) + r, err := NewMultiFileReader(sf, true) + if err != nil { + t.Fatal(err) + } + return r } func TestMultiFileReaderToMultiFile(t *testing.T) { - mfr := getTestMultiFileReader() + mfr := getTestMultiFileReader(t) mpReader := multipart.NewReader(mfr, mfr.Boundary()) mf, err := NewFileFromPartReader(mpReader, multipartFormdataType) if err != nil { @@ -37,51 +41,48 @@ func TestMultiFileReaderToMultiFile(t *testing.T) { if !ok { t.Fatal("Expected a directory") } + it, err := md.Entries() + if err != nil { + t.Fatal(err) + } - fn, f, err := md.NextFile() - if fn != "file.txt" || f == nil || err != nil { - t.Fatal("NextFile returned unexpected data") + if !it.Next() || it.Name() != "file.txt" { + t.Fatal("iterator didn't work as expected") } - dn, d, err := md.NextFile() - if dn != "boop" || d == nil || err != nil { - t.Fatal("NextFile returned unexpected data") + if !it.Next() || it.Name() != "boop" || it.Dir() == nil { + t.Fatal("iterator didn't work as expected") } - df, ok := d.(Directory) - if !ok { - t.Fatal("Expected a directory") + subIt, err := it.Dir().Entries() + if err != nil { + t.Fatal(err) } - cfn, cf, err := df.NextFile() - if cfn != "a.txt" || cf == nil || err != nil { - t.Fatal("NextFile returned unexpected data") + if !subIt.Next() || subIt.Name() != "a.txt" || subIt.Dir() != nil { + t.Fatal("iterator didn't work as expected") } - cfn, cf, err = df.NextFile() - if cfn != "b.txt" || cf == nil || err != nil { - t.Fatal("NextFile returned unexpected data") + if !subIt.Next() || subIt.Name() != "b.txt" || subIt.Dir() != nil { + t.Fatal("iterator didn't work as expected") } - cfn, cf, err = df.NextFile() - if cfn != "" || cf != nil || err != io.EOF { - t.Fatal("NextFile returned unexpected data") + if subIt.Next() { + t.Fatal("iterator didn't work as expected") } // try to break internal state - cfn, cf, err = df.NextFile() - if cfn != "" || cf != nil || err != io.EOF { - t.Fatal("NextFile returned unexpected data") + if subIt.Next() { + t.Fatal("iterator didn't work as expected") } - fn, f, err = md.NextFile() - if fn != "beep.txt" || f == nil || err != nil { - t.Fatal("NextFile returned unexpected data") + if !it.Next() || it.Name() != "beep.txt" || it.Dir() != nil { + t.Fatal("iterator didn't work as expected") } } func TestOutput(t *testing.T) { - mfr := getTestMultiFileReader() + mfr := getTestMultiFileReader(t) mpReader := &peekReader{r: multipart.NewReader(mfr, mfr.Boundary())} buf := make([]byte, 20) @@ -153,9 +154,9 @@ func TestOutput(t *testing.T) { t.Fatal("Expected filename to be \"b.txt\"") } - cname, child, err = mpd.NextFile() - if child != nil || err != io.EOF { - t.Fatal("Expected to get (nil, io.EOF)") + it, err := mpd.Entries() + if it.Next() { + t.Fatal("Expected to get false") } part, err = mpReader.NextPart() diff --git a/files/multipartfile.go b/files/multipartfile.go index 5c62c2cde..be7e2219f 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -2,7 +2,6 @@ package files import ( "errors" - "io" "io/ioutil" "mime" "mime/multipart" @@ -39,7 +38,7 @@ func NewFileFromPartReader(reader *multipart.Reader, mediatype string) (File, er } f := &MultipartFile{ - Reader: &peekReader{r: reader}, + Reader: &peekReader{r: reader}, Mediatype: mediatype, } @@ -95,30 +94,65 @@ func isDirectory(mediatype string) bool { return mediatype == multipartFormdataType || mediatype == applicationDirectory } -func (f *MultipartFile) NextFile() (string, File, error) { - if f.Reader == nil { - return "", nil, io.EOF +type multipartIterator struct { + f *MultipartFile + + curFile File + curName string + err error +} + +func (it *multipartIterator) Name() string { + return it.curName +} + +func (it *multipartIterator) File() File { + return it.curFile +} + +func (it *multipartIterator) Regular() Regular { + return castRegular(it.File()) +} + +func (it *multipartIterator) Dir() Directory { + return castDir(it.File()) +} + +func (it *multipartIterator) Next() bool { + if it.f.Reader == nil { + return false } - part, err := f.Reader.NextPart() + part, err := it.f.Reader.NextPart() if err != nil { - return "", nil, err + it.err = err + return false } - name, cf, err := newFileFromPart(f.fileName(), part, f.Reader) + name, cf, err := newFileFromPart(it.f.fileName(), part, it.f.Reader) if err != ErrPartOutsideParent { - return name, cf, err + it.curFile = cf + it.curName = name + it.err = err + return err == nil } // we read too much, try to fix this - pr, ok := f.Reader.(*peekReader) + pr, ok := it.f.Reader.(*peekReader) if !ok { - return "", nil, errors.New("cannot undo NextPart") + it.err = errors.New("cannot undo NextPart") + return false } - if err := pr.put(part); err != nil { - return "", nil, err - } - return "", nil, io.EOF + it.err = pr.put(part) + return false +} + +func (it *multipartIterator) Err() error { + panic("implement me") +} + +func (f *MultipartFile) Entries() (DirIterator, error) { + return &multipartIterator{f: f}, nil } func (f *MultipartFile) fileName() string { @@ -135,7 +169,10 @@ func (f *MultipartFile) fileName() string { } func (f *MultipartFile) Close() error { - return f.Part.Close() + if f.Part != nil { + return f.Part.Close() + } + return nil } func (f *MultipartFile) Size() (int64, error) { diff --git a/files/serialfile.go b/files/serialfile.go index 666399d29..77afafa6f 100644 --- a/files/serialfile.go +++ b/files/serialfile.go @@ -1,6 +1,7 @@ package files import ( + "errors" "fmt" "io" "io/ioutil" @@ -19,6 +20,18 @@ type serialFile struct { handleHiddenFiles bool } +type serialIterator struct { + files []os.FileInfo + handleHiddenFiles bool + path string + + curName string + curFile File + + err error +} + +// TODO: test/document limitations func NewSerialFile(path string, hidden bool, stat os.FileInfo) (File, error) { switch mode := stat.Mode(); { case mode.IsRegular(): @@ -46,20 +59,69 @@ func NewSerialFile(path string, hidden bool, stat os.FileInfo) (File, error) { } } -func (f *serialFile) NextFile() (string, File, error) { - // if a file was opened previously, close it - err := f.Close() - if err != nil { - switch err2 := err.(type) { - case *os.PathError: - if err2.Err != os.ErrClosed { - return "", nil, err - } - default: - return "", nil, err +func (it *serialIterator) Name() string { + return it.curName +} + +func (it *serialIterator) File() File { + return it.curFile +} + +func (it *serialIterator) Regular() Regular { + return castRegular(it.File()) +} + +func (it *serialIterator) Dir() Directory { + return castDir(it.File()) +} + +func (it *serialIterator) Next() bool { + // if there aren't any files left in the root directory, we're done + if len(it.files) == 0 { + return false + } + + stat := it.files[0] + it.files = it.files[1:] + for !it.handleHiddenFiles && strings.HasPrefix(stat.Name(), ".") { + if len(it.files) == 0 { + return false } + + stat = it.files[0] + it.files = it.files[1:] + } + + // open the next file + filePath := filepath.ToSlash(filepath.Join(it.path, stat.Name())) + + // recursively call the constructor on the next file + // if it's a regular file, we will open it as a ReaderFile + // if it's a directory, files in it will be opened serially + sf, err := NewSerialFile(filePath, it.handleHiddenFiles, stat) + if err != nil { + it.err = err + return false } + it.curName = stat.Name() + it.curFile = sf + return true +} + +func (it *serialIterator) Err() error { + return it.err +} + +func (f *serialFile) Entries() (DirIterator, error) { + return &serialIterator{ + path: f.path, + files: f.files, + handleHiddenFiles: f.handleHiddenFiles, + }, nil +} + +func (f *serialFile) NextFile() (string, File, error) { // if there aren't any files left in the root directory, we're done if len(f.files) == 0 { return "", nil, io.EOF @@ -101,7 +163,8 @@ func (f *serialFile) Stat() os.FileInfo { func (f *serialFile) Size() (int64, error) { if !f.stat.IsDir() { - return f.stat.Size(), nil + //something went terribly, terribly wrong + return 0, errors.New("serialFile is not a directory") } var du int64 @@ -119,4 +182,15 @@ func (f *serialFile) Size() (int64, error) { return du, err } +func castRegular(f File) Regular { + r, _ := f.(Regular) + return r +} + +func castDir(f File) Directory { + d, _ := f.(Directory) + return d +} + var _ Directory = &serialFile{} +var _ DirIterator = &serialIterator{} diff --git a/files/slicefile.go b/files/slicefile.go index 0d78e16c6..62c299f81 100644 --- a/files/slicefile.go +++ b/files/slicefile.go @@ -1,33 +1,76 @@ package files -import ( - "io" -) +type fileEntry struct { + name string + file File +} + +func (e fileEntry) Name() string { + return e.name +} + +func (e fileEntry) File() File { + return e.file +} + +func (e fileEntry) Regular() Regular { + return castRegular(e.file) +} + +func (e fileEntry) Dir() Directory { + return castDir(e.file) +} + +func FileEntry(name string, file File) DirEntry { + return fileEntry{ + name: name, + file: file, + } +} + +type sliceIterator struct { + files []DirEntry + n int +} + +func (it *sliceIterator) Name() string { + return it.files[it.n].Name() +} + +func (it *sliceIterator) File() File { + return it.files[it.n].File() +} + +func (it *sliceIterator) Regular() Regular { + return it.files[it.n].Regular() +} -type FileEntry struct { - File File - Name string +func (it *sliceIterator) Dir() Directory { + return it.files[it.n].Dir() +} + +func (it *sliceIterator) Next() bool { + it.n++ + return it.n < len(it.files) +} + +func (it *sliceIterator) Err() error { + return nil } // SliceFile implements File, and provides simple directory handling. // It contains children files, and is created from a `[]File`. // SliceFiles are always directories, and can't be read from or closed. type SliceFile struct { - files []FileEntry - n int + files []DirEntry } -func NewSliceFile(files []FileEntry) Directory { - return &SliceFile{files, 0} +func NewSliceFile(files []DirEntry) Directory { + return &SliceFile{files} } -func (f *SliceFile) NextFile() (string, File, error) { - if f.n >= len(f.files) { - return "", nil, io.EOF - } - file := f.files[f.n] - f.n++ - return file.Name, file.File, nil +func (f *SliceFile) Entries() (DirIterator, error) { + return &sliceIterator{files: f.files, n: -1}, nil } func (f *SliceFile) Close() error { @@ -42,7 +85,7 @@ func (f *SliceFile) Size() (int64, error) { var size int64 for _, file := range f.files { - s, err := file.File.Size() + s, err := file.File().Size() if err != nil { return 0, err } @@ -53,3 +96,4 @@ func (f *SliceFile) Size() (int64, error) { } var _ Directory = &SliceFile{} +var _ DirEntry = fileEntry{} diff --git a/files/webfile.go b/files/webfile.go index 5638f770a..c8e930a81 100644 --- a/files/webfile.go +++ b/files/webfile.go @@ -11,8 +11,8 @@ import ( // from a Web URL (http). A GET request will be performed // against the source when calling Read(). type WebFile struct { - body io.ReadCloser - url *url.URL + body io.ReadCloser + url *url.URL contentLength int64 } @@ -61,5 +61,4 @@ func (wf *WebFile) Size() (int64, error) { return wf.contentLength, nil } - var _ Regular = &WebFile{} From e8aa7305ca2034f081709905e977fce31acbb97e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 10 Nov 2018 00:28:12 +0100 Subject: [PATCH 16/76] Rename File/Regular to Node/File This commit was moved from ipfs/go-ipfs-files@960ec1edab9061dc7b8d2ae9c86e2784706d748e --- files/file.go | 41 +++++++++++++++-------------------- files/file_test.go | 6 ++--- files/is_hidden.go | 2 +- files/is_hidden_windows.go | 2 +- files/linkfile.go | 4 ++-- files/multifilereader.go | 14 ++++++------ files/multifilereader_test.go | 6 ++--- files/multipartfile.go | 14 ++++++------ files/readerfile.go | 4 ++-- files/serialfile.go | 18 +++++++-------- files/slicefile.go | 22 +++++++++---------- files/webfile.go | 2 +- 12 files changed, 65 insertions(+), 70 deletions(-) diff --git a/files/file.go b/files/file.go index 693baab82..6e6a24983 100644 --- a/files/file.go +++ b/files/file.go @@ -13,13 +13,8 @@ var ( ErrNotSupported = errors.New("operation not supported") ) -// File is an interface that provides functionality for handling -// files/directories as values that can be supplied to commands. For -// directories, child files are accessed serially by calling `NextFile()` -// -// Read/Seek methods are only valid for files -// NextFile method is only valid for directories -type File interface { +// Node is a common interface for files, directories and other special files +type Node interface { io.Closer // Size returns size of this file (if this file is a directory, total size of @@ -28,9 +23,9 @@ type File interface { Size() (int64, error) } -// Regular represents a regular Unix file -type Regular interface { - File +// Node represents a regular Unix file +type File interface { + Node io.Reader io.Seeker @@ -38,18 +33,18 @@ type Regular interface { // DirEntry exposes information about a directory entry type DirEntry interface { - // Name returns the base name of this entry, which is the base name of - // the referenced file + // Name returns base name of this entry, which is the base name of referenced + // file Name() string - // File returns the file referenced by this DirEntry - File() File + // Node returns the file referenced by this DirEntry + Node() Node - // Regular is an alias for ent.File().(Regular). If the file isn't a regular + // File is an alias for ent.Node().(File). If the file isn't a regular // file, nil value will be returned - Regular() Regular + File() File - // Dir is an alias for ent.File().(directory). If the file isn't a directory, + // Dir is an alias for ent.Node().(directory). If the file isn't a directory, // nil value will be returned Dir() Directory } @@ -63,18 +58,18 @@ type DirIterator interface { // to Next() and after Next() returned false may result in undefined behavior DirEntry - // Next advances the iterator to the next file. + // Next advances iterator to the next file. Next() bool - // Err may return an error after the previous call to Next() returned `false`. - // If the previous call to Next() returned `true`, Err() is guaranteed to + // Err may return an error after previous call to Next() returned `false`. + // If previous call to Next() returned `true`, Err() is guaranteed to // return nil Err() error } // Directory is a special file which can link to any number of files. type Directory interface { - File + Node // Entries returns a stateful iterator over directory entries. // @@ -83,7 +78,7 @@ type Directory interface { // it := dir.Entries() // for it.Next() { // name := it.Name() - // file := it.File() + // file := it.Node() // [...] // } // if it.Err() != nil { @@ -105,7 +100,7 @@ type Directory interface { // FileInfo exposes information on files in local filesystem type FileInfo interface { - File + Node // AbsPath returns full real file path. AbsPath() string diff --git a/files/file_test.go b/files/file_test.go index cbf227c08..8fa2a87d9 100644 --- a/files/file_test.go +++ b/files/file_test.go @@ -25,7 +25,7 @@ func TestSliceFiles(t *testing.T) { if !it.Next() { t.Fatal("Expected a file") } - rf := it.Regular() + rf := it.File() if rf == nil { t.Fatal("Expected a regular file") } @@ -104,7 +104,7 @@ anotherfile if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } - mf, ok := mpf.(Regular) + mf, ok := mpf.(File) if !ok { t.Fatal("Expected file to not be a directory") } @@ -147,7 +147,7 @@ anotherfile if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } - mf, ok = mpf.(Regular) + mf, ok = mpf.(File) if !ok { t.Fatal("Expected file to not be a directory") } diff --git a/files/is_hidden.go b/files/is_hidden.go index d5bc88683..4ebca6008 100644 --- a/files/is_hidden.go +++ b/files/is_hidden.go @@ -7,7 +7,7 @@ import ( "strings" ) -func IsHidden(name string, f File) bool { +func IsHidden(name string, f Node) bool { fName := filepath.Base(name) if strings.HasPrefix(fName, ".") && len(fName) > 1 { diff --git a/files/is_hidden_windows.go b/files/is_hidden_windows.go index 6f9568af2..7419f932e 100644 --- a/files/is_hidden_windows.go +++ b/files/is_hidden_windows.go @@ -9,7 +9,7 @@ import ( windows "golang.org/x/sys/windows" ) -func IsHidden(name string, f File) bool { +func IsHidden(name string, f Node) bool { fName := filepath.Base(name) diff --git a/files/linkfile.go b/files/linkfile.go index df334d574..11f7c9243 100644 --- a/files/linkfile.go +++ b/files/linkfile.go @@ -13,7 +13,7 @@ type Symlink struct { reader io.Reader } -func NewLinkFile(target string, stat os.FileInfo) Regular { +func NewLinkFile(target string, stat os.FileInfo) File { return &Symlink{ Target: target, stat: stat, @@ -45,4 +45,4 @@ func (lf *Symlink) Size() (int64, error) { return 0, ErrNotSupported } -var _ Regular = &Symlink{} +var _ File = &Symlink{} diff --git a/files/multifilereader.go b/files/multifilereader.go index 8817ad815..f5f35b4db 100644 --- a/files/multifilereader.go +++ b/files/multifilereader.go @@ -11,7 +11,7 @@ import ( "sync" ) -// MultiFileReader reads from a `commands.File` (which can be a directory of files +// MultiFileReader reads from a `commands.Node` (which can be a directory of files // or a regular file) as HTTP multipart encoded data. type MultiFileReader struct { io.Reader @@ -20,7 +20,7 @@ type MultiFileReader struct { files []DirIterator path []string - currentFile File + currentFile Node buf bytes.Buffer mpWriter *multipart.Writer closed bool @@ -86,7 +86,7 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { // handle starting a new file part if !mfr.closed { - mfr.currentFile = entry.File() + mfr.currentFile = entry.Node() // write the boundary and headers header := make(textproto.MIMEHeader) @@ -95,7 +95,7 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { var contentType string - switch f := entry.File().(type) { + switch f := entry.Node().(type) { case *Symlink: contentType = "application/symlink" case Directory: @@ -107,7 +107,7 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { mfr.files = append(mfr.files, newIt) mfr.path = append(mfr.path, entry.Name()) contentType = "application/x-directory" - case Regular: + case File: // otherwise, use the file as a reader to read its contents contentType = "application/octet-stream" default: @@ -115,7 +115,7 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { } header.Set("Content-Type", contentType) - if rf, ok := entry.File().(FileInfo); ok { + if rf, ok := entry.Node().(FileInfo); ok { header.Set("abspath", rf.AbsPath()) } @@ -133,7 +133,7 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { // otherwise, read from file data switch f := mfr.currentFile.(type) { - case Regular: + case File: written, err = f.Read(buf) if err != io.EOF { return written, err diff --git a/files/multifilereader_test.go b/files/multifilereader_test.go index dd7dac9e4..a3cf1eeb5 100644 --- a/files/multifilereader_test.go +++ b/files/multifilereader_test.go @@ -94,7 +94,7 @@ func TestOutput(t *testing.T) { if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } - mpr, ok := mpf.(Regular) + mpr, ok := mpf.(File) if !ok { t.Fatal("Expected file to be a regular file") } @@ -132,7 +132,7 @@ func TestOutput(t *testing.T) { if child == nil || err != nil { t.Fatal("Expected to be able to read a child file") } - if _, ok := child.(Regular); !ok { + if _, ok := child.(File); !ok { t.Fatal("Expected file to not be a directory") } if cname != "a.txt" { @@ -147,7 +147,7 @@ func TestOutput(t *testing.T) { if child == nil || err != nil { t.Fatal("Expected to be able to read a child file") } - if _, ok := child.(Regular); !ok { + if _, ok := child.(File); !ok { t.Fatal("Expected file to not be a directory") } if cname != "b.txt" { diff --git a/files/multipartfile.go b/files/multipartfile.go index be7e2219f..1552543db 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -22,17 +22,17 @@ const ( var ErrPartOutsideParent = errors.New("file outside parent dir") -// MultipartFile implements File, and is created from a `multipart.Part`. +// MultipartFile implements Node, and is created from a `multipart.Part`. // It can be either a directory or file (checked by calling `IsDirectory()`). type MultipartFile struct { - File + Node Part *multipart.Part Reader PartReader Mediatype string } -func NewFileFromPartReader(reader *multipart.Reader, mediatype string) (File, error) { +func NewFileFromPartReader(reader *multipart.Reader, mediatype string) (Node, error) { if !isDirectory(mediatype) { return nil, ErrNotDirectory } @@ -45,7 +45,7 @@ func NewFileFromPartReader(reader *multipart.Reader, mediatype string) (File, er return f, nil } -func newFileFromPart(parent string, part *multipart.Part, reader PartReader) (string, File, error) { +func newFileFromPart(parent string, part *multipart.Part, reader PartReader) (string, Node, error) { f := &MultipartFile{ Part: part, Reader: reader, @@ -97,7 +97,7 @@ func isDirectory(mediatype string) bool { type multipartIterator struct { f *MultipartFile - curFile File + curFile Node curName string err error } @@ -106,11 +106,11 @@ func (it *multipartIterator) Name() string { return it.curName } -func (it *multipartIterator) File() File { +func (it *multipartIterator) File() Node { return it.curFile } -func (it *multipartIterator) Regular() Regular { +func (it *multipartIterator) Regular() File { return castRegular(it.File()) } diff --git a/files/readerfile.go b/files/readerfile.go index 7db7e966c..261273c78 100644 --- a/files/readerfile.go +++ b/files/readerfile.go @@ -14,7 +14,7 @@ type ReaderFile struct { stat os.FileInfo } -func NewReaderFile(reader io.ReadCloser, stat os.FileInfo) Regular { +func NewReaderFile(reader io.ReadCloser, stat os.FileInfo) File { return &ReaderFile{"", reader, stat} } @@ -58,5 +58,5 @@ func (f *ReaderFile) Seek(offset int64, whence int) (int64, error) { return 0, ErrNotSupported } -var _ Regular = &ReaderFile{} +var _ File = &ReaderFile{} var _ FileInfo = &ReaderFile{} diff --git a/files/serialfile.go b/files/serialfile.go index 77afafa6f..cbfecc2b8 100644 --- a/files/serialfile.go +++ b/files/serialfile.go @@ -10,7 +10,7 @@ import ( "strings" ) -// serialFile implements File, and reads from a path on the OS filesystem. +// serialFile implements Node, and reads from a path on the OS filesystem. // No more than one file will be opened at a time (directories will advance // to the next file when NextFile() is called). type serialFile struct { @@ -26,13 +26,13 @@ type serialIterator struct { path string curName string - curFile File + curFile Node err error } // TODO: test/document limitations -func NewSerialFile(path string, hidden bool, stat os.FileInfo) (File, error) { +func NewSerialFile(path string, hidden bool, stat os.FileInfo) (Node, error) { switch mode := stat.Mode(); { case mode.IsRegular(): file, err := os.Open(path) @@ -63,11 +63,11 @@ func (it *serialIterator) Name() string { return it.curName } -func (it *serialIterator) File() File { +func (it *serialIterator) File() Node { return it.curFile } -func (it *serialIterator) Regular() Regular { +func (it *serialIterator) Regular() File { return castRegular(it.File()) } @@ -121,7 +121,7 @@ func (f *serialFile) Entries() (DirIterator, error) { }, nil } -func (f *serialFile) NextFile() (string, File, error) { +func (f *serialFile) NextFile() (string, Node, error) { // if there aren't any files left in the root directory, we're done if len(f.files) == 0 { return "", nil, io.EOF @@ -182,12 +182,12 @@ func (f *serialFile) Size() (int64, error) { return du, err } -func castRegular(f File) Regular { - r, _ := f.(Regular) +func castRegular(f Node) File { + r, _ := f.(File) return r } -func castDir(f File) Directory { +func castDir(f Node) Directory { d, _ := f.(Directory) return d } diff --git a/files/slicefile.go b/files/slicefile.go index 62c299f81..ddc263274 100644 --- a/files/slicefile.go +++ b/files/slicefile.go @@ -2,18 +2,18 @@ package files type fileEntry struct { name string - file File + file Node } func (e fileEntry) Name() string { return e.name } -func (e fileEntry) File() File { +func (e fileEntry) File() Node { return e.file } -func (e fileEntry) Regular() Regular { +func (e fileEntry) Regular() File { return castRegular(e.file) } @@ -21,7 +21,7 @@ func (e fileEntry) Dir() Directory { return castDir(e.file) } -func FileEntry(name string, file File) DirEntry { +func FileEntry(name string, file Node) DirEntry { return fileEntry{ name: name, file: file, @@ -37,12 +37,12 @@ func (it *sliceIterator) Name() string { return it.files[it.n].Name() } -func (it *sliceIterator) File() File { - return it.files[it.n].File() +func (it *sliceIterator) File() Node { + return it.files[it.n].Node() } -func (it *sliceIterator) Regular() Regular { - return it.files[it.n].Regular() +func (it *sliceIterator) Regular() File { + return it.files[it.n].File() } func (it *sliceIterator) Dir() Directory { @@ -58,8 +58,8 @@ func (it *sliceIterator) Err() error { return nil } -// SliceFile implements File, and provides simple directory handling. -// It contains children files, and is created from a `[]File`. +// SliceFile implements Node, and provides simple directory handling. +// It contains children files, and is created from a `[]Node`. // SliceFiles are always directories, and can't be read from or closed. type SliceFile struct { files []DirEntry @@ -85,7 +85,7 @@ func (f *SliceFile) Size() (int64, error) { var size int64 for _, file := range f.files { - s, err := file.File().Size() + s, err := file.Node().Size() if err != nil { return 0, err } diff --git a/files/webfile.go b/files/webfile.go index c8e930a81..6c1cc3ff0 100644 --- a/files/webfile.go +++ b/files/webfile.go @@ -61,4 +61,4 @@ func (wf *WebFile) Size() (int64, error) { return wf.contentLength, nil } -var _ Regular = &WebFile{} +var _ File = &WebFile{} From 5a00a2662f7428ab49a4b40dad57f5aefc342271 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 14 Nov 2018 18:26:22 +0100 Subject: [PATCH 17/76] move the MultipartFile iterator note This commit was moved from ipfs/go-ipfs-files@401cce734f3daaff69f97ebfd10460624dc3a6b5 --- files/file.go | 11 ++--------- files/multipartfile.go | 7 +++++-- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/files/file.go b/files/file.go index 6e6a24983..e416650e6 100644 --- a/files/file.go +++ b/files/file.go @@ -86,15 +86,8 @@ type Directory interface { // } // // Note: - // - Below limitations aren't applicable to all implementations, consult - // your implementations manual before using this interface in a way that - // doesn't meet these constraints - // - Some implementations may only allow reading in order - so if the iterator - // returns a directory you must iterate over it's entries first before - // calling Next again - // - Order is not guaranteed - // - Depending on implementation it may not be safe to iterate multiple - // 'branches' in parallel + // - Some implementations of this functions may define some constraints in how + // it can be used Entries() (DirIterator, error) } diff --git a/files/multipartfile.go b/files/multipartfile.go index 1552543db..b79efa54a 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -23,7 +23,10 @@ const ( var ErrPartOutsideParent = errors.New("file outside parent dir") // MultipartFile implements Node, and is created from a `multipart.Part`. -// It can be either a directory or file (checked by calling `IsDirectory()`). +// +// Note: iterating entries can be done only once and must be done in order, +// meaning that when iterator returns a directory, you MUST read all it's +// children before calling Next again type MultipartFile struct { Node @@ -32,7 +35,7 @@ type MultipartFile struct { Mediatype string } -func NewFileFromPartReader(reader *multipart.Reader, mediatype string) (Node, error) { +func NewFileFromPartReader(reader *multipart.Reader, mediatype string) (Directory, error) { if !isDirectory(mediatype) { return nil, ErrNotDirectory } From aa310c6586adf73228c8ded694c6e67d388ab936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 16 Nov 2018 01:31:38 +0100 Subject: [PATCH 18/76] Fix some iterators after rename This commit was moved from ipfs/go-ipfs-files@fbb870de8d52b2ea0d9388334c7ccd3603199319 --- files/multipartfile.go | 8 ++++---- files/serialfile.go | 8 ++++---- files/slicefile.go | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/files/multipartfile.go b/files/multipartfile.go index b79efa54a..fc5610ba5 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -109,16 +109,16 @@ func (it *multipartIterator) Name() string { return it.curName } -func (it *multipartIterator) File() Node { +func (it *multipartIterator) Node() Node { return it.curFile } -func (it *multipartIterator) Regular() File { - return castRegular(it.File()) +func (it *multipartIterator) File() File { + return castRegular(it.Node()) } func (it *multipartIterator) Dir() Directory { - return castDir(it.File()) + return castDir(it.Node()) } func (it *multipartIterator) Next() bool { diff --git a/files/serialfile.go b/files/serialfile.go index cbfecc2b8..f4e62b504 100644 --- a/files/serialfile.go +++ b/files/serialfile.go @@ -63,16 +63,16 @@ func (it *serialIterator) Name() string { return it.curName } -func (it *serialIterator) File() Node { +func (it *serialIterator) Node() Node { return it.curFile } -func (it *serialIterator) Regular() File { - return castRegular(it.File()) +func (it *serialIterator) File() File { + return castRegular(it.Node()) } func (it *serialIterator) Dir() Directory { - return castDir(it.File()) + return castDir(it.Node()) } func (it *serialIterator) Next() bool { diff --git a/files/slicefile.go b/files/slicefile.go index ddc263274..6400d4701 100644 --- a/files/slicefile.go +++ b/files/slicefile.go @@ -9,11 +9,11 @@ func (e fileEntry) Name() string { return e.name } -func (e fileEntry) File() Node { +func (e fileEntry) Node() Node { return e.file } -func (e fileEntry) Regular() File { +func (e fileEntry) File() File { return castRegular(e.file) } @@ -37,11 +37,11 @@ func (it *sliceIterator) Name() string { return it.files[it.n].Name() } -func (it *sliceIterator) File() Node { +func (it *sliceIterator) Node() Node { return it.files[it.n].Node() } -func (it *sliceIterator) Regular() File { +func (it *sliceIterator) File() File { return it.files[it.n].File() } From 5a89821589dc4faa25fd66ab21ce5f977688134d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 19 Nov 2018 03:54:49 +0100 Subject: [PATCH 19/76] Fix errors import This commit was moved from ipfs/go-ipfs-files@a557fc4681d48dabfced24efedb2bd7456ba8c6a --- files/webfile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/webfile.go b/files/webfile.go index 6c1cc3ff0..dbc813bd6 100644 --- a/files/webfile.go +++ b/files/webfile.go @@ -1,7 +1,7 @@ package files import ( - "github.com/pkg/errors" + "errors" "io" "net/http" "net/url" From ee6b055a93fca090411a982037ebb89f8321e90d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 24 Nov 2018 03:51:16 +0100 Subject: [PATCH 20/76] Don't return error from Entries This commit was moved from ipfs/go-ipfs-files@89a3f6de696e8d4c170dfcc5157c8e0187f9cc5a --- files/file.go | 2 +- files/file_test.go | 5 +---- files/multifilereader.go | 17 +++++------------ files/multifilereader_test.go | 12 +++--------- files/multipartfile.go | 4 ++-- files/serialfile.go | 4 ++-- files/slicefile.go | 4 ++-- 7 files changed, 16 insertions(+), 32 deletions(-) diff --git a/files/file.go b/files/file.go index e416650e6..d975f110c 100644 --- a/files/file.go +++ b/files/file.go @@ -88,7 +88,7 @@ type Directory interface { // Note: // - Some implementations of this functions may define some constraints in how // it can be used - Entries() (DirIterator, error) + Entries() DirIterator } // FileInfo exposes information on files in local filesystem diff --git a/files/file_test.go b/files/file_test.go index 8fa2a87d9..81bcabeb1 100644 --- a/files/file_test.go +++ b/files/file_test.go @@ -17,10 +17,7 @@ func TestSliceFiles(t *testing.T) { buf := make([]byte, 20) sf := NewSliceFile(files) - it, err := sf.Entries() - if err != nil { - t.Fatal(err) - } + it := sf.Entries() if !it.Next() { t.Fatal("Expected a file") diff --git a/files/multifilereader.go b/files/multifilereader.go index f5f35b4db..52d9f4c4f 100644 --- a/files/multifilereader.go +++ b/files/multifilereader.go @@ -35,10 +35,7 @@ type MultiFileReader struct { // If `form` is set to true, the multipart data will have a Content-Type of 'multipart/form-data', // if `form` is false, the Content-Type will be 'multipart/mixed'. func NewMultiFileReader(file Directory, form bool) (*MultiFileReader, error) { - it, err := file.Entries() - if err != nil { - return nil, err - } + it := file.Entries() mfr := &MultiFileReader{ files: []DirIterator{it}, @@ -72,13 +69,13 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { } if !mfr.files[len(mfr.files)-1].Next() { + if mfr.files[len(mfr.files)-1].Err() != nil { + return 0, err + } mfr.files = mfr.files[:len(mfr.files)-1] mfr.path = mfr.path[:len(mfr.path)-1] continue } - if mfr.files[len(mfr.files)-1].Err() != nil { - return 0, mfr.files[len(mfr.files)-1].Err() - } entry = mfr.files[len(mfr.files)-1] } @@ -99,11 +96,7 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { case *Symlink: contentType = "application/symlink" case Directory: - newIt, err := f.Entries() - if err != nil { - return 0, err - } - + newIt := f.Entries() mfr.files = append(mfr.files, newIt) mfr.path = append(mfr.path, entry.Name()) contentType = "application/x-directory" diff --git a/files/multifilereader_test.go b/files/multifilereader_test.go index a3cf1eeb5..8df39a85e 100644 --- a/files/multifilereader_test.go +++ b/files/multifilereader_test.go @@ -41,10 +41,7 @@ func TestMultiFileReaderToMultiFile(t *testing.T) { if !ok { t.Fatal("Expected a directory") } - it, err := md.Entries() - if err != nil { - t.Fatal(err) - } + it := md.Entries() if !it.Next() || it.Name() != "file.txt" { t.Fatal("iterator didn't work as expected") @@ -54,10 +51,7 @@ func TestMultiFileReaderToMultiFile(t *testing.T) { t.Fatal("iterator didn't work as expected") } - subIt, err := it.Dir().Entries() - if err != nil { - t.Fatal(err) - } + subIt := it.Dir().Entries() if !subIt.Next() || subIt.Name() != "a.txt" || subIt.Dir() != nil { t.Fatal("iterator didn't work as expected") @@ -154,7 +148,7 @@ func TestOutput(t *testing.T) { t.Fatal("Expected filename to be \"b.txt\"") } - it, err := mpd.Entries() + it := mpd.Entries() if it.Next() { t.Fatal("Expected to get false") } diff --git a/files/multipartfile.go b/files/multipartfile.go index fc5610ba5..18bf12c91 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -154,8 +154,8 @@ func (it *multipartIterator) Err() error { panic("implement me") } -func (f *MultipartFile) Entries() (DirIterator, error) { - return &multipartIterator{f: f}, nil +func (f *MultipartFile) Entries() DirIterator { + return &multipartIterator{f: f} } func (f *MultipartFile) fileName() string { diff --git a/files/serialfile.go b/files/serialfile.go index f4e62b504..e37957472 100644 --- a/files/serialfile.go +++ b/files/serialfile.go @@ -113,12 +113,12 @@ func (it *serialIterator) Err() error { return it.err } -func (f *serialFile) Entries() (DirIterator, error) { +func (f *serialFile) Entries() DirIterator { return &serialIterator{ path: f.path, files: f.files, handleHiddenFiles: f.handleHiddenFiles, - }, nil + } } func (f *serialFile) NextFile() (string, Node, error) { diff --git a/files/slicefile.go b/files/slicefile.go index 6400d4701..82ce22319 100644 --- a/files/slicefile.go +++ b/files/slicefile.go @@ -69,8 +69,8 @@ func NewSliceFile(files []DirEntry) Directory { return &SliceFile{files} } -func (f *SliceFile) Entries() (DirIterator, error) { - return &sliceIterator{files: f.files, n: -1}, nil +func (f *SliceFile) Entries() DirIterator { + return &sliceIterator{files: f.files, n: -1} } func (f *SliceFile) Close() error { From 9956a2bc8f7237d16cd434505581bba41572d725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 26 Nov 2018 09:10:51 +0100 Subject: [PATCH 21/76] Fix error return in multipartfile This commit was moved from ipfs/go-ipfs-files@34b159078a3a709315c69c133f89638e8a7cbdf1 --- files/multipartfile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/multipartfile.go b/files/multipartfile.go index 18bf12c91..86cef0f27 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -151,7 +151,7 @@ func (it *multipartIterator) Next() bool { } func (it *multipartIterator) Err() error { - panic("implement me") + return it.err } func (f *MultipartFile) Entries() DirIterator { From e449707a6a81cf0ab989a9b1ba3c2a34ece5596d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 29 Nov 2018 20:19:04 +0100 Subject: [PATCH 22/76] Don't expose io.EOF in multifilereader This commit was moved from ipfs/go-ipfs-files@23e95656f2e4e8325b03e72df7b9f0ba6197b45a --- files/multifilereader.go | 2 +- files/multifilereader_test.go | 10 +++++++--- files/multipartfile.go | 14 +++++++++++++- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/files/multifilereader.go b/files/multifilereader.go index 52d9f4c4f..758b5013f 100644 --- a/files/multifilereader.go +++ b/files/multifilereader.go @@ -70,7 +70,7 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { if !mfr.files[len(mfr.files)-1].Next() { if mfr.files[len(mfr.files)-1].Err() != nil { - return 0, err + return 0, mfr.files[len(mfr.files)-1].Err() } mfr.files = mfr.files[:len(mfr.files)-1] mfr.path = mfr.path[:len(mfr.path)-1] diff --git a/files/multifilereader_test.go b/files/multifilereader_test.go index 8df39a85e..46efe0ea9 100644 --- a/files/multifilereader_test.go +++ b/files/multifilereader_test.go @@ -61,16 +61,20 @@ func TestMultiFileReaderToMultiFile(t *testing.T) { t.Fatal("iterator didn't work as expected") } - if subIt.Next() { + if subIt.Next() || it.Err() != nil { t.Fatal("iterator didn't work as expected") } // try to break internal state - if subIt.Next() { + if subIt.Next() || it.Err() != nil { t.Fatal("iterator didn't work as expected") } - if !it.Next() || it.Name() != "beep.txt" || it.Dir() != nil { + if !it.Next() || it.Name() != "beep.txt" || it.Dir() != nil || it.Err() != nil { + t.Fatal("iterator didn't work as expected") + } + + if it.Next() || it.Err() != nil { t.Fatal("iterator didn't work as expected") } } diff --git a/files/multipartfile.go b/files/multipartfile.go index 86cef0f27..651bfdb8c 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -2,6 +2,7 @@ package files import ( "errors" + "io" "io/ioutil" "mime" "mime/multipart" @@ -127,6 +128,9 @@ func (it *multipartIterator) Next() bool { } part, err := it.f.Reader.NextPart() if err != nil { + if err == io.EOF { + return false + } it.err = err return false } @@ -198,7 +202,15 @@ func (pr *peekReader) NextPart() (*multipart.Part, error) { return p, nil } - return pr.r.NextPart() + if pr.r == nil { + return nil, io.EOF + } + + p, err := pr.r.NextPart() + if err == io.EOF { + pr.r = nil + } + return p, err } func (pr *peekReader) put(p *multipart.Part) error { From a4e5147f57e02d466d49c38579a783d1c38a8016 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 3 Dec 2018 14:20:03 +0100 Subject: [PATCH 23/76] files2.0: Add some convenience functions This commit was moved from ipfs/go-ipfs-files@f732b393dcf123f84fdbceb88985e8e85466794a --- files/file.go | 8 ---- files/file_test.go | 16 +++---- files/linkfile.go | 5 +++ files/multifilereader_test.go | 39 ++++++++--------- files/multipartfile.go | 8 ---- files/serialfile.go | 18 -------- files/slicefile.go | 16 ------- files/util.go | 80 +++++++++++++++++++++++++++++++++++ 8 files changed, 110 insertions(+), 80 deletions(-) create mode 100644 files/util.go diff --git a/files/file.go b/files/file.go index d975f110c..41545be54 100644 --- a/files/file.go +++ b/files/file.go @@ -39,14 +39,6 @@ type DirEntry interface { // Node returns the file referenced by this DirEntry Node() Node - - // File is an alias for ent.Node().(File). If the file isn't a regular - // file, nil value will be returned - File() File - - // Dir is an alias for ent.Node().(directory). If the file isn't a directory, - // nil value will be returned - Dir() Directory } // DirIterator is a iterator over directory entries. diff --git a/files/file_test.go b/files/file_test.go index 81bcabeb1..ef93cf73d 100644 --- a/files/file_test.go +++ b/files/file_test.go @@ -2,27 +2,25 @@ package files import ( "io" - "io/ioutil" "mime/multipart" "strings" "testing" ) func TestSliceFiles(t *testing.T) { - files := []DirEntry{ - FileEntry("", NewReaderFile(ioutil.NopCloser(strings.NewReader("Some text!\n")), nil)), - FileEntry("", NewReaderFile(ioutil.NopCloser(strings.NewReader("beep")), nil)), - FileEntry("", NewReaderFile(ioutil.NopCloser(strings.NewReader("boop")), nil)), - } + sf := DirFrom(map[string]Node{ + "1": FileFrom([]byte("Some text!\n")), + "2": FileFrom([]byte("beep")), + "3": FileFrom([]byte("boop")), + }) buf := make([]byte, 20) - sf := NewSliceFile(files) it := sf.Entries() if !it.Next() { t.Fatal("Expected a file") } - rf := it.File() + rf := ToFile(it.Node()) if rf == nil { t.Fatal("Expected a regular file") } @@ -48,7 +46,7 @@ func TestSliceFiles(t *testing.T) { func TestReaderFiles(t *testing.T) { message := "beep boop" - rf := NewReaderFile(ioutil.NopCloser(strings.NewReader(message)), nil) + rf := FileFrom([]byte(message)) buf := make([]byte, len(message)) if n, err := rf.Read(buf); n == 0 || err != nil { diff --git a/files/linkfile.go b/files/linkfile.go index 11f7c9243..409309bca 100644 --- a/files/linkfile.go +++ b/files/linkfile.go @@ -45,4 +45,9 @@ func (lf *Symlink) Size() (int64, error) { return 0, ErrNotSupported } +func ToSymlink(n Node) *Symlink { + l, _ := n.(*Symlink) + return l +} + var _ File = &Symlink{} diff --git a/files/multifilereader_test.go b/files/multifilereader_test.go index 46efe0ea9..154c5b299 100644 --- a/files/multifilereader_test.go +++ b/files/multifilereader_test.go @@ -2,24 +2,21 @@ package files import ( "io" - "io/ioutil" "mime/multipart" - "strings" "testing" ) var text = "Some text! :)" func getTestMultiFileReader(t *testing.T) *MultiFileReader { - fileset := []DirEntry{ - FileEntry("file.txt", NewReaderFile(ioutil.NopCloser(strings.NewReader(text)), nil)), - FileEntry("boop", NewSliceFile([]DirEntry{ - FileEntry("a.txt", NewReaderFile(ioutil.NopCloser(strings.NewReader("bleep")), nil)), - FileEntry("b.txt", NewReaderFile(ioutil.NopCloser(strings.NewReader("bloop")), nil)), - })), - FileEntry("beep.txt", NewReaderFile(ioutil.NopCloser(strings.NewReader("beep")), nil)), - } - sf := NewSliceFile(fileset) + sf := DirFrom(map[string]Node{ + "file.txt": FileFrom([]byte(text)), + "boop": DirFrom(map[string]Node{ + "a.txt": FileFrom([]byte("bleep")), + "b.txt": FileFrom([]byte("bloop")), + }), + "beep.txt": FileFrom([]byte("beep")), + }) // testing output by reading it with the go stdlib "mime/multipart" Reader r, err := NewMultiFileReader(sf, true) @@ -43,21 +40,21 @@ func TestMultiFileReaderToMultiFile(t *testing.T) { } it := md.Entries() - if !it.Next() || it.Name() != "file.txt" { + if !it.Next() || it.Name() != "beep.txt" { t.Fatal("iterator didn't work as expected") } - if !it.Next() || it.Name() != "boop" || it.Dir() == nil { + if !it.Next() || it.Name() != "boop" || DirFrom(it) == nil { t.Fatal("iterator didn't work as expected") } - subIt := it.Dir().Entries() + subIt := DirFrom(it).Entries() - if !subIt.Next() || subIt.Name() != "a.txt" || subIt.Dir() != nil { + if !subIt.Next() || subIt.Name() != "a.txt" || DirFrom(subIt) != nil { t.Fatal("iterator didn't work as expected") } - if !subIt.Next() || subIt.Name() != "b.txt" || subIt.Dir() != nil { + if !subIt.Next() || subIt.Name() != "b.txt" || DirFrom(subIt) != nil { t.Fatal("iterator didn't work as expected") } @@ -70,7 +67,7 @@ func TestMultiFileReaderToMultiFile(t *testing.T) { t.Fatal("iterator didn't work as expected") } - if !it.Next() || it.Name() != "beep.txt" || it.Dir() != nil || it.Err() != nil { + if !it.Next() || it.Name() != "file.txt" || DirFrom(it) != nil || it.Err() != nil { t.Fatal("iterator didn't work as expected") } @@ -96,13 +93,13 @@ func TestOutput(t *testing.T) { if !ok { t.Fatal("Expected file to be a regular file") } - if mpname != "file.txt" { + if mpname != "beep.txt" { t.Fatal("Expected filename to be \"file.txt\"") } - if n, err := mpr.Read(buf); n != len(text) || err != nil { + if n, err := mpr.Read(buf); n != 4 || err != nil { t.Fatal("Expected to read from file", n, err) } - if string(buf[:len(text)]) != text { + if string(buf[:4]) != "beep" { t.Fatal("Data read was different than expected") } @@ -165,7 +162,7 @@ func TestOutput(t *testing.T) { if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } - if mpname != "beep.txt" { + if mpname != "file.txt" { t.Fatal("Expected filename to be \"b.txt\"") } diff --git a/files/multipartfile.go b/files/multipartfile.go index 651bfdb8c..14b4cba2d 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -114,14 +114,6 @@ func (it *multipartIterator) Node() Node { return it.curFile } -func (it *multipartIterator) File() File { - return castRegular(it.Node()) -} - -func (it *multipartIterator) Dir() Directory { - return castDir(it.Node()) -} - func (it *multipartIterator) Next() bool { if it.f.Reader == nil { return false diff --git a/files/serialfile.go b/files/serialfile.go index e37957472..e29752d66 100644 --- a/files/serialfile.go +++ b/files/serialfile.go @@ -67,14 +67,6 @@ func (it *serialIterator) Node() Node { return it.curFile } -func (it *serialIterator) File() File { - return castRegular(it.Node()) -} - -func (it *serialIterator) Dir() Directory { - return castDir(it.Node()) -} - func (it *serialIterator) Next() bool { // if there aren't any files left in the root directory, we're done if len(it.files) == 0 { @@ -182,15 +174,5 @@ func (f *serialFile) Size() (int64, error) { return du, err } -func castRegular(f Node) File { - r, _ := f.(File) - return r -} - -func castDir(f Node) Directory { - d, _ := f.(Directory) - return d -} - var _ Directory = &serialFile{} var _ DirIterator = &serialIterator{} diff --git a/files/slicefile.go b/files/slicefile.go index 82ce22319..715cee3d0 100644 --- a/files/slicefile.go +++ b/files/slicefile.go @@ -13,14 +13,6 @@ func (e fileEntry) Node() Node { return e.file } -func (e fileEntry) File() File { - return castRegular(e.file) -} - -func (e fileEntry) Dir() Directory { - return castDir(e.file) -} - func FileEntry(name string, file Node) DirEntry { return fileEntry{ name: name, @@ -41,14 +33,6 @@ func (it *sliceIterator) Node() Node { return it.files[it.n].Node() } -func (it *sliceIterator) File() File { - return it.files[it.n].File() -} - -func (it *sliceIterator) Dir() Directory { - return it.files[it.n].Dir() -} - func (it *sliceIterator) Next() bool { it.n++ return it.n < len(it.files) diff --git a/files/util.go b/files/util.go new file mode 100644 index 000000000..a294190fb --- /dev/null +++ b/files/util.go @@ -0,0 +1,80 @@ +package files + +import ( + "bytes" + "io" + "io/ioutil" + "sort" +) + +// ToFile is an alias for n.(File). If the file isn't a regular file, nil value +// will be returned +func ToFile(n Node) File { + f, _ := n.(File) + return f +} + +// ToDir is an alias for n.(Directory). If the file isn't directory, a nil value +// will be returned +func ToDir(n Node) Directory { + d, _ := n.(Directory) + return d +} + +// FileFrom is a convenience function which tries to extract or create new file +// from provided value. If a passed value can't be turned into a File, nil will +// be returned. +// +// Supported types: +// * files.File (cast from Node) +// * DirEntry / DirIterator (cast from e.Node()) +// * []byte (wrapped into NewReaderFile) +// * io.Reader / io.ReadCloser (wrapped into NewReaderFile) +func FileFrom(n interface{}) File { + switch f := n.(type) { + case File: + return f + case DirEntry: + return ToFile(f.Node()) + case []byte: + return NewReaderFile(ioutil.NopCloser(bytes.NewReader(f)), nil) + case io.ReadCloser: + return NewReaderFile(f, nil) + case io.Reader: + return NewReaderFile(ioutil.NopCloser(f), nil) + default: + return nil + } +} + +// DirFrom is a convenience function which tries to extract or create new +// directory from the provided value. If a passed value can't be turned into a +// Directory, nil will be returned. +// +// Supported types: +// * files.File (cast from Node) +// * DirEntry (cast from e.Node()) +// * DirIterator (current file, cast from e.Node()) +// * []DirEntry (wrapped into NewSliceFile) +// * map[string]Node (wrapped into NewSliceFile) +func DirFrom(n interface{}) Directory { + switch f := n.(type) { + case Directory: + return f + case DirEntry: + return ToDir(f.Node()) + case []DirEntry: + return NewSliceFile(f) + case map[string]Node: + ents := make([]DirEntry, 0, len(f)) + for name, nd := range f { + ents = append(ents, FileEntry(name, nd)) + } + sort.Slice(ents, func(i, j int) bool { + return ents[i].Name() < ents[j].Name() + }) + return NewSliceFile(ents) + default: + return nil + } +} From 9bdb3d2df13448c7dd98d27c277aaae1e5a8e872 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 3 Dec 2018 14:20:43 +0100 Subject: [PATCH 24/76] files2.0: no error from NewMultiFileReader This commit was moved from ipfs/go-ipfs-files@bc7a700c01a34b4cc17bceb706381060afb30868 --- files/multifilereader.go | 4 ++-- files/multifilereader_test.go | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/files/multifilereader.go b/files/multifilereader.go index 758b5013f..cf3d14c73 100644 --- a/files/multifilereader.go +++ b/files/multifilereader.go @@ -34,7 +34,7 @@ type MultiFileReader struct { // NewMultiFileReader constructs a MultiFileReader. `file` can be any `commands.Directory`. // If `form` is set to true, the multipart data will have a Content-Type of 'multipart/form-data', // if `form` is false, the Content-Type will be 'multipart/mixed'. -func NewMultiFileReader(file Directory, form bool) (*MultiFileReader, error) { +func NewMultiFileReader(file Directory, form bool) *MultiFileReader { it := file.Entries() mfr := &MultiFileReader{ @@ -45,7 +45,7 @@ func NewMultiFileReader(file Directory, form bool) (*MultiFileReader, error) { } mfr.mpWriter = multipart.NewWriter(&mfr.buf) - return mfr, nil + return mfr } func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { diff --git a/files/multifilereader_test.go b/files/multifilereader_test.go index 154c5b299..a7d642139 100644 --- a/files/multifilereader_test.go +++ b/files/multifilereader_test.go @@ -19,11 +19,7 @@ func getTestMultiFileReader(t *testing.T) *MultiFileReader { }) // testing output by reading it with the go stdlib "mime/multipart" Reader - r, err := NewMultiFileReader(sf, true) - if err != nil { - t.Fatal(err) - } - return r + return NewMultiFileReader(sf, true) } func TestMultiFileReaderToMultiFile(t *testing.T) { From 5d0cdac110352e7652a12ab3ba76eeae9f3d3261 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 9 Dec 2018 16:40:06 +0100 Subject: [PATCH 25/76] Allow skipping entries in multipartIterator This commit was moved from ipfs/go-ipfs-files@b925960e72275f9cf944e8965ee18838a3b7fc6e --- files/multifilereader_test.go | 31 +++++++++++++++++++++++ files/multipartfile.go | 47 +++++++++++++++++++++++++---------- 2 files changed, 65 insertions(+), 13 deletions(-) diff --git a/files/multifilereader_test.go b/files/multifilereader_test.go index a7d642139..31ef5c0a5 100644 --- a/files/multifilereader_test.go +++ b/files/multifilereader_test.go @@ -72,6 +72,37 @@ func TestMultiFileReaderToMultiFile(t *testing.T) { } } +func TestMultiFileReaderToMultiFileSkip(t *testing.T) { + mfr := getTestMultiFileReader(t) + mpReader := multipart.NewReader(mfr, mfr.Boundary()) + mf, err := NewFileFromPartReader(mpReader, multipartFormdataType) + if err != nil { + t.Fatal(err) + } + + md, ok := mf.(Directory) + if !ok { + t.Fatal("Expected a directory") + } + it := md.Entries() + + if !it.Next() || it.Name() != "beep.txt" { + t.Fatal("iterator didn't work as expected") + } + + if !it.Next() || it.Name() != "boop" || DirFrom(it) == nil { + t.Fatal("iterator didn't work as expected") + } + + if !it.Next() || it.Name() != "file.txt" || DirFrom(it) != nil || it.Err() != nil { + t.Fatal("iterator didn't work as expected") + } + + if it.Next() || it.Err() != nil { + t.Fatal("iterator didn't work as expected") + } +} + func TestOutput(t *testing.T) { mfr := getTestMultiFileReader(t) mpReader := &peekReader{r: multipart.NewReader(mfr, mfr.Boundary())} diff --git a/files/multipartfile.go b/files/multipartfile.go index 14b4cba2d..7bb535f31 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -8,6 +8,7 @@ import ( "mime/multipart" "net/url" "path" + "strings" ) const ( @@ -22,6 +23,7 @@ const ( ) var ErrPartOutsideParent = errors.New("file outside parent dir") +var ErrPartInChildTree = errors.New("file in child tree") // MultipartFile implements Node, and is created from a `multipart.Part`. // @@ -56,7 +58,19 @@ func newFileFromPart(parent string, part *multipart.Part, reader PartReader) (st } dir, base := path.Split(f.fileName()) - if path.Clean(dir) != path.Clean(parent) { + dir = path.Clean(dir) + parent = path.Clean(parent) + if dir == "." { + dir = "" + } + if parent == "." { + parent = "" + } + + if dir != parent { + if strings.HasPrefix(dir, parent) { + return "", nil, ErrPartInChildTree + } return "", nil, ErrPartOutsideParent } @@ -118,21 +132,28 @@ func (it *multipartIterator) Next() bool { if it.f.Reader == nil { return false } - part, err := it.f.Reader.NextPart() - if err != nil { - if err == io.EOF { + var part *multipart.Part + for { + var err error + part, err = it.f.Reader.NextPart() + if err != nil { + if err == io.EOF { + return false + } + it.err = err return false } - it.err = err - return false - } - name, cf, err := newFileFromPart(it.f.fileName(), part, it.f.Reader) - if err != ErrPartOutsideParent { - it.curFile = cf - it.curName = name - it.err = err - return err == nil + name, cf, err := newFileFromPart(it.f.fileName(), part, it.f.Reader) + if err == ErrPartOutsideParent { + break + } + if err != ErrPartInChildTree { + it.curFile = cf + it.curName = name + it.err = err + return err == nil + } } // we read too much, try to fix this From 2c1df9d16cf28020f0815d7e312b45ee64ad38b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 9 Dec 2018 17:28:11 +0100 Subject: [PATCH 26/76] Expand docstring for Directory.Entries This commit was moved from ipfs/go-ipfs-files@26bbce7b6144f0e87f50b04a2ac55d6d360796ea --- files/file.go | 12 ++++++++++++ files/multipartfile.go | 4 ---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/files/file.go b/files/file.go index 41545be54..15cf10867 100644 --- a/files/file.go +++ b/files/file.go @@ -80,6 +80,18 @@ type Directory interface { // Note: // - Some implementations of this functions may define some constraints in how // it can be used + // - Each implementation MUST support: + // - Pre-order sequential iteration: + // - Meaning that after calling `Next` you can call `Next` if the returned + // node is a directory or read the returned file + // - Skipping entries: + // - Meaning that if `Next` returns a directory, you can skip reading it's + // entries and skip to next entry. Files don't have to be read in full. + // Note that you can't go back to unread entries, this only allows + // skipping parts of a directory tree + // - This is to allow listing files in a directory without having to read + // the entire tree + // - Entries may not be sorted Entries() DirIterator } diff --git a/files/multipartfile.go b/files/multipartfile.go index 7bb535f31..c11cc82ae 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -26,10 +26,6 @@ var ErrPartOutsideParent = errors.New("file outside parent dir") var ErrPartInChildTree = errors.New("file in child tree") // MultipartFile implements Node, and is created from a `multipart.Part`. -// -// Note: iterating entries can be done only once and must be done in order, -// meaning that when iterator returns a directory, you MUST read all it's -// children before calling Next again type MultipartFile struct { Node From d2515d7ec84831b8e2c815e41122ad0b820e1738 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 10 Dec 2018 22:57:55 +0100 Subject: [PATCH 27/76] More type-safe utility functions This commit was moved from ipfs/go-ipfs-files@8d1df4d43f7be3059e4291fb55a0e7d47a5b2de5 --- files/file_test.go | 10 ++-- files/multifilereader_test.go | 26 ++++----- files/readerfile.go | 19 ++++++- files/{slicefile.go => slicedirectory.go} | 16 +++++- files/util.go | 67 ++--------------------- 5 files changed, 56 insertions(+), 82 deletions(-) rename files/{slicefile.go => slicedirectory.go} (79%) diff --git a/files/file_test.go b/files/file_test.go index ef93cf73d..af607612a 100644 --- a/files/file_test.go +++ b/files/file_test.go @@ -8,10 +8,10 @@ import ( ) func TestSliceFiles(t *testing.T) { - sf := DirFrom(map[string]Node{ - "1": FileFrom([]byte("Some text!\n")), - "2": FileFrom([]byte("beep")), - "3": FileFrom([]byte("boop")), + sf := NewMapDirectory(map[string]Node{ + "1": NewBytesFile([]byte("Some text!\n")), + "2": NewBytesFile([]byte("beep")), + "3": NewBytesFile([]byte("boop")), }) buf := make([]byte, 20) @@ -46,7 +46,7 @@ func TestSliceFiles(t *testing.T) { func TestReaderFiles(t *testing.T) { message := "beep boop" - rf := FileFrom([]byte(message)) + rf := NewBytesFile([]byte(message)) buf := make([]byte, len(message)) if n, err := rf.Read(buf); n == 0 || err != nil { diff --git a/files/multifilereader_test.go b/files/multifilereader_test.go index 31ef5c0a5..3357b23c9 100644 --- a/files/multifilereader_test.go +++ b/files/multifilereader_test.go @@ -9,13 +9,13 @@ import ( var text = "Some text! :)" func getTestMultiFileReader(t *testing.T) *MultiFileReader { - sf := DirFrom(map[string]Node{ - "file.txt": FileFrom([]byte(text)), - "boop": DirFrom(map[string]Node{ - "a.txt": FileFrom([]byte("bleep")), - "b.txt": FileFrom([]byte("bloop")), + sf := NewMapDirectory(map[string]Node{ + "file.txt": NewBytesFile([]byte(text)), + "boop": NewMapDirectory(map[string]Node{ + "a.txt": NewBytesFile([]byte("bleep")), + "b.txt": NewBytesFile([]byte("bloop")), }), - "beep.txt": FileFrom([]byte("beep")), + "beep.txt": NewBytesFile([]byte("beep")), }) // testing output by reading it with the go stdlib "mime/multipart" Reader @@ -40,17 +40,17 @@ func TestMultiFileReaderToMultiFile(t *testing.T) { t.Fatal("iterator didn't work as expected") } - if !it.Next() || it.Name() != "boop" || DirFrom(it) == nil { + if !it.Next() || it.Name() != "boop" || DirFromEntry(it) == nil { t.Fatal("iterator didn't work as expected") } - subIt := DirFrom(it).Entries() + subIt := DirFromEntry(it).Entries() - if !subIt.Next() || subIt.Name() != "a.txt" || DirFrom(subIt) != nil { + if !subIt.Next() || subIt.Name() != "a.txt" || DirFromEntry(subIt) != nil { t.Fatal("iterator didn't work as expected") } - if !subIt.Next() || subIt.Name() != "b.txt" || DirFrom(subIt) != nil { + if !subIt.Next() || subIt.Name() != "b.txt" || DirFromEntry(subIt) != nil { t.Fatal("iterator didn't work as expected") } @@ -63,7 +63,7 @@ func TestMultiFileReaderToMultiFile(t *testing.T) { t.Fatal("iterator didn't work as expected") } - if !it.Next() || it.Name() != "file.txt" || DirFrom(it) != nil || it.Err() != nil { + if !it.Next() || it.Name() != "file.txt" || DirFromEntry(it) != nil || it.Err() != nil { t.Fatal("iterator didn't work as expected") } @@ -90,11 +90,11 @@ func TestMultiFileReaderToMultiFileSkip(t *testing.T) { t.Fatal("iterator didn't work as expected") } - if !it.Next() || it.Name() != "boop" || DirFrom(it) == nil { + if !it.Next() || it.Name() != "boop" || DirFromEntry(it) == nil { t.Fatal("iterator didn't work as expected") } - if !it.Next() || it.Name() != "file.txt" || DirFrom(it) != nil || it.Err() != nil { + if !it.Next() || it.Name() != "file.txt" || DirFromEntry(it) != nil || it.Err() != nil { t.Fatal("iterator didn't work as expected") } diff --git a/files/readerfile.go b/files/readerfile.go index 261273c78..cc965c810 100644 --- a/files/readerfile.go +++ b/files/readerfile.go @@ -1,7 +1,9 @@ package files import ( + "bytes" "io" + "io/ioutil" "os" "path/filepath" ) @@ -14,8 +16,21 @@ type ReaderFile struct { stat os.FileInfo } -func NewReaderFile(reader io.ReadCloser, stat os.FileInfo) File { - return &ReaderFile{"", reader, stat} +func NewBytesFile(b []byte) File { + return NewReaderFile(bytes.NewReader(b)) +} + +func NewReaderFile(reader io.Reader) File { + return NewReaderStatFile(reader, nil) +} + +func NewReaderStatFile(reader io.Reader, stat os.FileInfo) File { + rc, ok := reader.(io.ReadCloser) + if !ok { + rc = ioutil.NopCloser(reader) + } + + return &ReaderFile{"", rc, stat} } func NewReaderPathFile(path string, reader io.ReadCloser, stat os.FileInfo) (*ReaderFile, error) { diff --git a/files/slicefile.go b/files/slicedirectory.go similarity index 79% rename from files/slicefile.go rename to files/slicedirectory.go index 715cee3d0..d11656261 100644 --- a/files/slicefile.go +++ b/files/slicedirectory.go @@ -1,5 +1,7 @@ package files +import "sort" + type fileEntry struct { name string file Node @@ -49,7 +51,19 @@ type SliceFile struct { files []DirEntry } -func NewSliceFile(files []DirEntry) Directory { +func NewMapDirectory(f map[string]Node) Directory { + ents := make([]DirEntry, 0, len(f)) + for name, nd := range f { + ents = append(ents, FileEntry(name, nd)) + } + sort.Slice(ents, func(i, j int) bool { + return ents[i].Name() < ents[j].Name() + }) + + return NewSliceDirectory(ents) +} + +func NewSliceDirectory(files []DirEntry) Directory { return &SliceFile{files} } diff --git a/files/util.go b/files/util.go index a294190fb..e727e7ae6 100644 --- a/files/util.go +++ b/files/util.go @@ -1,12 +1,5 @@ package files -import ( - "bytes" - "io" - "io/ioutil" - "sort" -) - // ToFile is an alias for n.(File). If the file isn't a regular file, nil value // will be returned func ToFile(n Node) File { @@ -21,60 +14,12 @@ func ToDir(n Node) Directory { return d } -// FileFrom is a convenience function which tries to extract or create new file -// from provided value. If a passed value can't be turned into a File, nil will -// be returned. -// -// Supported types: -// * files.File (cast from Node) -// * DirEntry / DirIterator (cast from e.Node()) -// * []byte (wrapped into NewReaderFile) -// * io.Reader / io.ReadCloser (wrapped into NewReaderFile) -func FileFrom(n interface{}) File { - switch f := n.(type) { - case File: - return f - case DirEntry: - return ToFile(f.Node()) - case []byte: - return NewReaderFile(ioutil.NopCloser(bytes.NewReader(f)), nil) - case io.ReadCloser: - return NewReaderFile(f, nil) - case io.Reader: - return NewReaderFile(ioutil.NopCloser(f), nil) - default: - return nil - } +// FileFromEntry calls ToFile on Node in the given entry +func FileFromEntry(e DirEntry) File { + return ToFile(e.Node()) } -// DirFrom is a convenience function which tries to extract or create new -// directory from the provided value. If a passed value can't be turned into a -// Directory, nil will be returned. -// -// Supported types: -// * files.File (cast from Node) -// * DirEntry (cast from e.Node()) -// * DirIterator (current file, cast from e.Node()) -// * []DirEntry (wrapped into NewSliceFile) -// * map[string]Node (wrapped into NewSliceFile) -func DirFrom(n interface{}) Directory { - switch f := n.(type) { - case Directory: - return f - case DirEntry: - return ToDir(f.Node()) - case []DirEntry: - return NewSliceFile(f) - case map[string]Node: - ents := make([]DirEntry, 0, len(f)) - for name, nd := range f { - ents = append(ents, FileEntry(name, nd)) - } - sort.Slice(ents, func(i, j int) bool { - return ents[i].Name() < ents[j].Name() - }) - return NewSliceFile(ents) - default: - return nil - } +// DirFromEntry calls ToDir on Node in the given entry +func DirFromEntry(e DirEntry) Directory { + return ToDir(e.Node()) } From 44f354a1ff609e4e6c0ced13c1ee8c792905bffd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 12 Dec 2018 13:42:15 +0100 Subject: [PATCH 28/76] Improve Directory docs This commit was moved from ipfs/go-ipfs-files@dfb493173af8f8b4fe7bd135b141ee350bfd12e6 --- files/file.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/files/file.go b/files/file.go index 15cf10867..d575345a9 100644 --- a/files/file.go +++ b/files/file.go @@ -78,17 +78,13 @@ type Directory interface { // } // // Note: - // - Some implementations of this functions may define some constraints in how - // it can be used // - Each implementation MUST support: // - Pre-order sequential iteration: - // - Meaning that after calling `Next` you can call `Next` if the returned + // - After calling `Next` you can call `Next` if the returned // node is a directory or read the returned file // - Skipping entries: - // - Meaning that if `Next` returns a directory, you can skip reading it's - // entries and skip to next entry. Files don't have to be read in full. - // Note that you can't go back to unread entries, this only allows - // skipping parts of a directory tree + // - You can skip a file/directory by calling Next without reading it + // - You can't use skipped files/directories // - This is to allow listing files in a directory without having to read // the entire tree // - Entries may not be sorted From 976a0443fce3ccd1d136907f1fa92e5ed07d3d85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 12 Dec 2018 13:48:33 +0100 Subject: [PATCH 29/76] More type-safety for multipartFile This commit was moved from ipfs/go-ipfs-files@a6bcde3c953235a2454a895be4519b5566cafb81 --- files/file_test.go | 16 ++++----- files/multifilereader_test.go | 6 ++-- files/multipartfile.go | 65 ++++++++++++++++------------------- 3 files changed, 40 insertions(+), 47 deletions(-) diff --git a/files/file_test.go b/files/file_test.go index af607612a..819238f01 100644 --- a/files/file_test.go +++ b/files/file_test.go @@ -95,9 +95,9 @@ anotherfile if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - mpname, mpf, err := newFileFromPart("", part, mpReader) + mpname, mpf, err := newFileFromPart("", part, &peekReader{r: mpReader}) if mpf == nil || err != nil { - t.Fatal("Expected non-nil MultipartFile, nil error") + t.Fatal("Expected non-nil multipartFile, nil error") } mf, ok := mpf.(File) if !ok { @@ -118,9 +118,9 @@ anotherfile if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - mpname, mpf, err = newFileFromPart("", part, mpReader) + mpname, mpf, err = newFileFromPart("", part, &peekReader{r: mpReader}) if mpf == nil || err != nil { - t.Fatal("Expected non-nil MultipartFile, nil error") + t.Fatal("Expected non-nil multipartFile, nil error") } md, ok := mpf.(Directory) if !ok { @@ -138,9 +138,9 @@ anotherfile if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - mpname, mpf, err = newFileFromPart("dir/", part, mpReader) + mpname, mpf, err = newFileFromPart("dir/", part, &peekReader{r: mpReader}) if mpf == nil || err != nil { - t.Fatal("Expected non-nil MultipartFile, nil error") + t.Fatal("Expected non-nil multipartFile, nil error") } mf, ok = mpf.(File) if !ok { @@ -161,9 +161,9 @@ anotherfile if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - mpname, mpf, err = newFileFromPart("dir/", part, mpReader) + mpname, mpf, err = newFileFromPart("dir/", part, &peekReader{r: mpReader}) if mpf == nil || err != nil { - t.Fatal("Expected non-nil MultipartFile, nil error") + t.Fatal("Expected non-nil multipartFile, nil error") } ms, ok := mpf.(*Symlink) if !ok { diff --git a/files/multifilereader_test.go b/files/multifilereader_test.go index 3357b23c9..fb4f749e5 100644 --- a/files/multifilereader_test.go +++ b/files/multifilereader_test.go @@ -114,7 +114,7 @@ func TestOutput(t *testing.T) { } mpname, mpf, err := newFileFromPart("", part, mpReader) if mpf == nil || err != nil { - t.Fatal("Expected non-nil MultipartFile, nil error") + t.Fatal("Expected non-nil multipartFile, nil error") } mpr, ok := mpf.(File) if !ok { @@ -136,7 +136,7 @@ func TestOutput(t *testing.T) { } mpname, mpf, err = newFileFromPart("", part, mpReader) if mpf == nil || err != nil { - t.Fatal("Expected non-nil MultipartFile, nil error") + t.Fatal("Expected non-nil multipartFile, nil error") } mpd, ok := mpf.(Directory) if !ok { @@ -187,7 +187,7 @@ func TestOutput(t *testing.T) { } mpname, mpf, err = newFileFromPart("", part, mpReader) if mpf == nil || err != nil { - t.Fatal("Expected non-nil MultipartFile, nil error") + t.Fatal("Expected non-nil multipartFile, nil error") } if mpname != "file.txt" { t.Fatal("Expected filename to be \"b.txt\"") diff --git a/files/multipartfile.go b/files/multipartfile.go index c11cc82ae..b083e7049 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -25,13 +25,13 @@ const ( var ErrPartOutsideParent = errors.New("file outside parent dir") var ErrPartInChildTree = errors.New("file in child tree") -// MultipartFile implements Node, and is created from a `multipart.Part`. -type MultipartFile struct { +// multipartFile implements Node, and is created from a `multipart.Part`. +type multipartFile struct { Node - Part *multipart.Part - Reader PartReader - Mediatype string + part *multipart.Part + reader *peekReader + mediatype string } func NewFileFromPartReader(reader *multipart.Reader, mediatype string) (Directory, error) { @@ -39,18 +39,18 @@ func NewFileFromPartReader(reader *multipart.Reader, mediatype string) (Director return nil, ErrNotDirectory } - f := &MultipartFile{ - Reader: &peekReader{r: reader}, - Mediatype: mediatype, + f := &multipartFile{ + reader: &peekReader{r: reader}, + mediatype: mediatype, } return f, nil } -func newFileFromPart(parent string, part *multipart.Part, reader PartReader) (string, Node, error) { - f := &MultipartFile{ - Part: part, - Reader: reader, +func newFileFromPart(parent string, part *multipart.Part, reader *peekReader) (string, Node, error) { + f := &multipartFile{ + part: part, + reader: reader, } dir, base := path.Split(f.fileName()) @@ -89,12 +89,12 @@ func newFileFromPart(parent string, part *multipart.Part, reader PartReader) (st } var err error - f.Mediatype, _, err = mime.ParseMediaType(contentType) + f.mediatype, _, err = mime.ParseMediaType(contentType) if err != nil { return "", nil, err } - if !isDirectory(f.Mediatype) { + if !isDirectory(f.mediatype) { return base, &ReaderFile{ reader: part, abspath: part.Header.Get("abspath"), @@ -109,7 +109,7 @@ func isDirectory(mediatype string) bool { } type multipartIterator struct { - f *MultipartFile + f *multipartFile curFile Node curName string @@ -125,13 +125,13 @@ func (it *multipartIterator) Node() Node { } func (it *multipartIterator) Next() bool { - if it.f.Reader == nil { + if it.f.reader == nil { return false } var part *multipart.Part for { var err error - part, err = it.f.Reader.NextPart() + part, err = it.f.reader.NextPart() if err != nil { if err == io.EOF { return false @@ -140,7 +140,7 @@ func (it *multipartIterator) Next() bool { return false } - name, cf, err := newFileFromPart(it.f.fileName(), part, it.f.Reader) + name, cf, err := newFileFromPart(it.f.fileName(), part, it.f.reader) if err == ErrPartOutsideParent { break } @@ -152,14 +152,7 @@ func (it *multipartIterator) Next() bool { } } - // we read too much, try to fix this - pr, ok := it.f.Reader.(*peekReader) - if !ok { - it.err = errors.New("cannot undo NextPart") - return false - } - - it.err = pr.put(part) + it.err = it.f.reader.put(part) return false } @@ -167,31 +160,31 @@ func (it *multipartIterator) Err() error { return it.err } -func (f *MultipartFile) Entries() DirIterator { +func (f *multipartFile) Entries() DirIterator { return &multipartIterator{f: f} } -func (f *MultipartFile) fileName() string { - if f == nil || f.Part == nil { +func (f *multipartFile) fileName() string { + if f == nil || f.part == nil { return "" } - filename, err := url.QueryUnescape(f.Part.FileName()) + filename, err := url.QueryUnescape(f.part.FileName()) if err != nil { // if there is a unescape error, just treat the name as unescaped - return f.Part.FileName() + return f.part.FileName() } return filename } -func (f *MultipartFile) Close() error { - if f.Part != nil { - return f.Part.Close() +func (f *multipartFile) Close() error { + if f.part != nil { + return f.part.Close() } return nil } -func (f *MultipartFile) Size() (int64, error) { +func (f *multipartFile) Size() (int64, error) { return 0, ErrNotSupported } @@ -230,4 +223,4 @@ func (pr *peekReader) put(p *multipart.Part) error { return nil } -var _ Directory = &MultipartFile{} +var _ Directory = &multipartFile{} From 584821cb7959583dd983cebb9057ae686d552b03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 13 Dec 2018 20:35:11 +0100 Subject: [PATCH 30/76] Reword note on Directory.Entries This commit was moved from ipfs/go-ipfs-files@58b4b6f71acc268144e439128d2d0dd281e1b641 --- files/file.go | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/files/file.go b/files/file.go index d575345a9..4d7ef1132 100644 --- a/files/file.go +++ b/files/file.go @@ -77,17 +77,8 @@ type Directory interface { // return err // } // - // Note: - // - Each implementation MUST support: - // - Pre-order sequential iteration: - // - After calling `Next` you can call `Next` if the returned - // node is a directory or read the returned file - // - Skipping entries: - // - You can skip a file/directory by calling Next without reading it - // - You can't use skipped files/directories - // - This is to allow listing files in a directory without having to read - // the entire tree - // - Entries may not be sorted + // Note that you can't store the result of it.Node() and use it after + // advancing the iterator Entries() DirIterator } From e52b4c0248abd12e2f2ed48fc70fb47efc63552a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 24 Jan 2019 20:26:07 +0100 Subject: [PATCH 31/76] TarWriter This commit was moved from ipfs/go-ipfs-files@41d028a04deb0bbb7dc8e930c8e74b7a7a33c80d --- files/tarwriter.go | 101 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 files/tarwriter.go diff --git a/files/tarwriter.go b/files/tarwriter.go new file mode 100644 index 000000000..382f93f03 --- /dev/null +++ b/files/tarwriter.go @@ -0,0 +1,101 @@ +package files + +import ( + "archive/tar" + "fmt" + "io" + "path" + "time" +) + +type Writer struct { + TarW *tar.Writer +} + +// NewTarWriter wraps given io.Writer into a new tar writer +func NewTarWriter(w io.Writer) (*Writer, error) { + return &Writer{ + TarW: tar.NewWriter(w), + }, nil +} + +func (w *Writer) writeDir(f Directory, fpath string) error { + if err := writeDirHeader(w.TarW, fpath); err != nil { + return err + } + + it := f.Entries() + for it.Next() { + if err := w.WriteFile(it.Node(), path.Join(fpath, it.Name())); err != nil { + return err + } + } + return it.Err() +} + +func (w *Writer) writeFile(f File, fpath string) error { + size, err := f.Size() + if err != nil { + return err + } + + if err := writeFileHeader(w.TarW, fpath, uint64(size)); err != nil { + return err + } + + if _, err := io.Copy(w.TarW, f); err != nil { + return err + } + w.TarW.Flush() + return nil +} + +// WriteNode adds a node to the archive. +func (w *Writer) WriteFile(nd Node, fpath string) error { + switch nd := nd.(type) { + case *Symlink: + return writeSymlinkHeader(w.TarW, nd.Target, fpath) + case File: + return w.writeFile(nd, fpath) + case Directory: + return w.writeDir(nd, fpath) + default: + return fmt.Errorf("file type %T is not supported", nd) + } +} + +// Close closes the tar writer. +func (w *Writer) Close() error { + return w.TarW.Close() +} + +func writeDirHeader(w *tar.Writer, fpath string) error { + return w.WriteHeader(&tar.Header{ + Name: fpath, + Typeflag: tar.TypeDir, + Mode: 0777, + ModTime: time.Now(), + // TODO: set mode, dates, etc. when added to unixFS + }) +} + +func writeFileHeader(w *tar.Writer, fpath string, size uint64) error { + return w.WriteHeader(&tar.Header{ + Name: fpath, + Size: int64(size), + Typeflag: tar.TypeReg, + Mode: 0644, + ModTime: time.Now(), + // TODO: set mode, dates, etc. when added to unixFS + }) +} + +func writeSymlinkHeader(w *tar.Writer, target, fpath string) error { + return w.WriteHeader(&tar.Header{ + Name: fpath, + Linkname: target, + Mode: 0777, + Typeflag: tar.TypeSymlink, + }) +} + From dbb8fc1570ac15bb4079e52d3c15b40a68fc0ba2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 24 Jan 2019 21:59:10 +0100 Subject: [PATCH 32/76] TarWriter test This commit was moved from ipfs/go-ipfs-files@abdd3ab5072ad06e7d7f5d20def095c66043db76 --- files/readerfile.go | 11 ++++-- files/tarwriter_test.go | 80 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 files/tarwriter_test.go diff --git a/files/readerfile.go b/files/readerfile.go index cc965c810..c23300173 100644 --- a/files/readerfile.go +++ b/files/readerfile.go @@ -14,10 +14,12 @@ type ReaderFile struct { abspath string reader io.ReadCloser stat os.FileInfo + + fsize int64 } func NewBytesFile(b []byte) File { - return NewReaderFile(bytes.NewReader(b)) + return &ReaderFile{"", NewReaderFile(bytes.NewReader(b)), nil, int64(len(b))} } func NewReaderFile(reader io.Reader) File { @@ -30,7 +32,7 @@ func NewReaderStatFile(reader io.Reader, stat os.FileInfo) File { rc = ioutil.NopCloser(reader) } - return &ReaderFile{"", rc, stat} + return &ReaderFile{"", rc, stat, -1} } func NewReaderPathFile(path string, reader io.ReadCloser, stat os.FileInfo) (*ReaderFile, error) { @@ -39,7 +41,7 @@ func NewReaderPathFile(path string, reader io.ReadCloser, stat os.FileInfo) (*Re return nil, err } - return &ReaderFile{abspath, reader, stat}, nil + return &ReaderFile{abspath, reader, stat, -1}, nil } func (f *ReaderFile) AbsPath() string { @@ -60,6 +62,9 @@ func (f *ReaderFile) Stat() os.FileInfo { func (f *ReaderFile) Size() (int64, error) { if f.stat == nil { + if f.fsize >= 0 { + return f.fsize, nil + } return 0, ErrNotSupported } return f.stat.Size(), nil diff --git a/files/tarwriter_test.go b/files/tarwriter_test.go new file mode 100644 index 000000000..38101d045 --- /dev/null +++ b/files/tarwriter_test.go @@ -0,0 +1,80 @@ +package files + +import ( + "archive/tar" + "io" + "testing" +) + +func TestTarWriter(t *testing.T) { + tf := NewMapDirectory(map[string]Node{ + "file.txt": NewBytesFile([]byte(text)), + "boop": NewMapDirectory(map[string]Node{ + "a.txt": NewBytesFile([]byte("bleep")), + "b.txt": NewBytesFile([]byte("bloop")), + }), + "beep.txt": NewBytesFile([]byte("beep")), + }) + + pr, pw := io.Pipe() + tw, err := NewTarWriter(pw) + if err != nil { + t.Fatal(err) + } + tr := tar.NewReader(pr) + + go func() { + defer tw.Close() + if err := tw.WriteFile(tf, ""); err != nil { + t.Fatal(err) + } + }() + + var cur *tar.Header + + checkHeader := func(name string, typ byte, size int64) { + if cur.Name != name { + t.Errorf("got wrong name: %s != %s", cur.Name, name) + } + if cur.Typeflag != typ { + t.Errorf("got wrong type: %d != %d", cur.Typeflag, typ) + } + if cur.Size != size { + t.Errorf("got wrong size: %d != %d", cur.Size, size) + } + } + + if cur, err = tr.Next(); err != nil { + t.Fatal(err) + } + checkHeader("", tar.TypeDir, 0) + + if cur, err = tr.Next(); err != nil { + t.Fatal(err) + } + checkHeader("beep.txt", tar.TypeReg, 4) + + if cur, err = tr.Next(); err != nil { + t.Fatal(err) + } + checkHeader("boop", tar.TypeDir, 0) + + if cur, err = tr.Next(); err != nil { + t.Fatal(err) + } + checkHeader("boop/a.txt", tar.TypeReg, 5) + + if cur, err = tr.Next(); err != nil { + t.Fatal(err) + } + checkHeader("boop/b.txt", tar.TypeReg, 5) + + if cur, err = tr.Next(); err != nil { + t.Fatal(err) + } + checkHeader("file.txt", tar.TypeReg, 13) + + if cur, err = tr.Next(); err != io.EOF { + t.Fatal(err) + } +} \ No newline at end of file From b6c323954bad3d283cc76a2d49184c7726c01693 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 25 Jan 2019 16:32:22 +0100 Subject: [PATCH 33/76] Writer -> TarWriter This commit was moved from ipfs/go-ipfs-files@e76e6073cad86b7a9504d3f42a8f0aa43c8044e5 --- files/tarwriter.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/files/tarwriter.go b/files/tarwriter.go index 382f93f03..101d4c844 100644 --- a/files/tarwriter.go +++ b/files/tarwriter.go @@ -8,18 +8,18 @@ import ( "time" ) -type Writer struct { +type TarWriter struct { TarW *tar.Writer } // NewTarWriter wraps given io.Writer into a new tar writer -func NewTarWriter(w io.Writer) (*Writer, error) { - return &Writer{ +func NewTarWriter(w io.Writer) (*TarWriter, error) { + return &TarWriter{ TarW: tar.NewWriter(w), }, nil } -func (w *Writer) writeDir(f Directory, fpath string) error { +func (w *TarWriter) writeDir(f Directory, fpath string) error { if err := writeDirHeader(w.TarW, fpath); err != nil { return err } @@ -33,7 +33,7 @@ func (w *Writer) writeDir(f Directory, fpath string) error { return it.Err() } -func (w *Writer) writeFile(f File, fpath string) error { +func (w *TarWriter) writeFile(f File, fpath string) error { size, err := f.Size() if err != nil { return err @@ -51,7 +51,7 @@ func (w *Writer) writeFile(f File, fpath string) error { } // WriteNode adds a node to the archive. -func (w *Writer) WriteFile(nd Node, fpath string) error { +func (w *TarWriter) WriteFile(nd Node, fpath string) error { switch nd := nd.(type) { case *Symlink: return writeSymlinkHeader(w.TarW, nd.Target, fpath) @@ -65,7 +65,7 @@ func (w *Writer) WriteFile(nd Node, fpath string) error { } // Close closes the tar writer. -func (w *Writer) Close() error { +func (w *TarWriter) Close() error { return w.TarW.Close() } From 96b049e85022297fd96b558a2e20c4c4432eb576 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Sat, 9 Feb 2019 12:15:03 -0800 Subject: [PATCH 34/76] create implicit directories from multipart requests Also, introduce a multipart walker to simplify some things. fixes #5 This commit was moved from ipfs/go-ipfs-files@a5b601693a30437983937ef57bd60746f34ae757 --- files/file_test.go | 196 +++++++++++++----------------- files/helpers_test.go | 126 ++++++++++++++++++++ files/multifilereader_test.go | 49 ++------ files/multipartfile.go | 217 ++++++++++++++++------------------ 4 files changed, 314 insertions(+), 274 deletions(-) create mode 100644 files/helpers_test.go diff --git a/files/file_test.go b/files/file_test.go index 819238f01..8c6c62229 100644 --- a/files/file_test.go +++ b/files/file_test.go @@ -13,35 +13,24 @@ func TestSliceFiles(t *testing.T) { "2": NewBytesFile([]byte("beep")), "3": NewBytesFile([]byte("boop")), }) - buf := make([]byte, 20) - it := sf.Entries() - - if !it.Next() { - t.Fatal("Expected a file") - } - rf := ToFile(it.Node()) - if rf == nil { - t.Fatal("Expected a regular file") - } - read, err := rf.Read(buf) - if read != 11 || err != nil { - t.Fatal("NextFile got a file in the wrong order") - } - - if !it.Next() { - t.Fatal("Expected a file") - } - if !it.Next() { - t.Fatal("Expected a file") - } - if it.Next() { - t.Fatal("Wild file appeared!") - } - - if err := sf.Close(); err != nil { - t.Fatal("Should be able to call `Close` on a SliceFile") - } + CheckDir(t, sf, []Event{ + { + kind: TFile, + name: "1", + value: "Some text!\n", + }, + { + kind: TFile, + name: "2", + value: "beep", + }, + { + kind: TFile, + name: "3", + value: "boop", + }, + }) } func TestReaderFiles(t *testing.T) { @@ -59,7 +48,6 @@ func TestReaderFiles(t *testing.T) { t.Fatal("Expected EOF when reading after close") } } - func TestMultipartFiles(t *testing.T) { data := ` --Boundary! @@ -82,97 +70,73 @@ Content-Type: application/symlink Content-Disposition: file; filename="dir/simlynk" anotherfile +--Boundary! +Content-Type: text/plain +Content-Disposition: file; filename="implicit1/implicit2/deep_implicit" + +implicit file1 +--Boundary! +Content-Type: text/plain +Content-Disposition: file; filename="implicit1/shallow_implicit" + +implicit file2 --Boundary!-- ` reader := strings.NewReader(data) mpReader := multipart.NewReader(reader, "Boundary!") - buf := make([]byte, 20) - - // test properties of a file created from the first part - part, err := mpReader.NextPart() - if part == nil || err != nil { - t.Fatal("Expected non-nil part, nil error") - } - mpname, mpf, err := newFileFromPart("", part, &peekReader{r: mpReader}) - if mpf == nil || err != nil { - t.Fatal("Expected non-nil multipartFile, nil error") - } - mf, ok := mpf.(File) - if !ok { - t.Fatal("Expected file to not be a directory") - } - if mpname != "name" { - t.Fatal("Expected filename to be \"name\"") - } - if n, err := mf.Read(buf); n != 4 || !(err == io.EOF || err == nil) { - t.Fatal("Expected to be able to read 4 bytes", n, err) - } - if err := mf.Close(); err != nil { - t.Fatal("Expected to be able to close file") - } - - // test properties of file created from second part (directory) - part, err = mpReader.NextPart() - if part == nil || err != nil { - t.Fatal("Expected non-nil part, nil error") - } - mpname, mpf, err = newFileFromPart("", part, &peekReader{r: mpReader}) - if mpf == nil || err != nil { - t.Fatal("Expected non-nil multipartFile, nil error") - } - md, ok := mpf.(Directory) - if !ok { - t.Fatal("Expected file to be a directory") - } - if mpname != "dir" { - t.Fatal("Expected filename to be \"dir\"") - } - if err := md.Close(); err != nil { - t.Fatal("Should be able to call `Close` on a directory") - } - - // test properties of file created from third part (nested file) - part, err = mpReader.NextPart() - if part == nil || err != nil { - t.Fatal("Expected non-nil part, nil error") - } - mpname, mpf, err = newFileFromPart("dir/", part, &peekReader{r: mpReader}) - if mpf == nil || err != nil { - t.Fatal("Expected non-nil multipartFile, nil error") - } - mf, ok = mpf.(File) - if !ok { - t.Fatal("Expected file to not be a directory") - } - if mpname != "nested" { - t.Fatalf("Expected filename to be \"nested\", got %s", mpname) - } - if n, err := mf.Read(buf); n != 12 || !(err == nil || err == io.EOF) { - t.Fatalf("expected to be able to read 12 bytes from file: %s (got %d)", err, n) - } - if err := mpf.Close(); err != nil { - t.Fatalf("should be able to close file: %s", err) - } - - // test properties of symlink created from fourth part (symlink) - part, err = mpReader.NextPart() - if part == nil || err != nil { - t.Fatal("Expected non-nil part, nil error") - } - mpname, mpf, err = newFileFromPart("dir/", part, &peekReader{r: mpReader}) - if mpf == nil || err != nil { - t.Fatal("Expected non-nil multipartFile, nil error") - } - ms, ok := mpf.(*Symlink) - if !ok { - t.Fatal("Expected file to not be a directory") - } - if mpname != "simlynk" { - t.Fatal("Expected filename to be \"dir/simlynk\"") - } - if ms.Target != "anotherfile" { - t.Fatal("expected link to point to anotherfile") - } + dir, err := NewFileFromPartReader(mpReader, multipartFormdataType) + if err != nil { + t.Fatal(err) + } + + CheckDir(t, dir, []Event{ + { + kind: TFile, + name: "name", + value: "beep", + }, + { + kind: TDirStart, + name: "dir", + }, + { + kind: TFile, + name: "nested", + value: "some content", + }, + { + kind: TSymlink, + name: "simlynk", + value: "anotherfile", + }, + { + kind: TDirEnd, + }, + { + kind: TDirStart, + name: "implicit1", + }, + { + kind: TDirStart, + name: "implicit2", + }, + { + kind: TFile, + name: "deep_implicit", + value: "implicit file1", + }, + { + kind: TDirEnd, + }, + { + kind: TFile, + name: "shallow_implicit", + value: "implicit file2", + }, + { + kind: TDirEnd, + }, + }) } diff --git a/files/helpers_test.go b/files/helpers_test.go new file mode 100644 index 000000000..ec420bdc2 --- /dev/null +++ b/files/helpers_test.go @@ -0,0 +1,126 @@ +package files + +import ( + "io/ioutil" + "testing" +) + +type Kind int + +const ( + TFile Kind = iota + TSymlink + TDirStart + TDirEnd +) + +type Event struct { + kind Kind + name string + value string +} + +func CheckDir(t *testing.T, dir Directory, expected []Event) { + expectedIndex := 0 + expect := func() (Event, int) { + t.Helper() + + if expectedIndex > len(expected) { + t.Fatal("no more expected entries") + } + i := expectedIndex + expectedIndex++ + + // Add an implicit "end" event at the end. It makes this + // function a bit easier to write. + next := Event{kind: TDirEnd} + if i < len(expected) { + next = expected[i] + } + return next, i + } + var check func(d Directory) + check = func(d Directory) { + it := d.Entries() + + for it.Next() { + next, i := expect() + + if it.Name() != next.name { + t.Fatalf("[%d] expected filename to be %q", i, next.name) + } + + switch next.kind { + case TFile: + mf, ok := it.Node().(File) + if !ok { + t.Fatalf("[%d] expected file to be a normal file: %T", i, it.Node()) + } + out, err := ioutil.ReadAll(mf) + if err != nil { + t.Errorf("[%d] failed to read file", i) + continue + } + if string(out) != next.value { + t.Errorf( + "[%d] while reading %q, expected %q, got %q", + i, + it.Name(), + next.value, + string(out), + ) + continue + } + case TSymlink: + mf, ok := it.Node().(*Symlink) + if !ok { + t.Errorf("[%d] expected file to be a symlink: %T", i, it.Node()) + continue + } + if mf.Target != next.value { + t.Errorf( + "[%d] target of symlink %q should have been %q but was %q", + i, + it.Name(), + next.value, + mf.Target, + ) + continue + } + case TDirStart: + mf, ok := it.Node().(Directory) + if !ok { + t.Fatalf( + "[%d] expected file to be a directory: %T", + i, + it.Node(), + ) + } + check(mf) + case TDirEnd: + t.Errorf( + "[%d] expected end of directory, found %#v at %q", + i, + it.Node(), + it.Name(), + ) + return + default: + t.Fatal("unhandled type", next.kind) + } + if err := it.Node().Close(); err != nil { + t.Fatalf("[%d] expected to be able to close node", i) + } + } + next, i := expect() + + if it.Err() != nil { + t.Fatalf("[%d] got error: %s", i, it.Err()) + } + + if next.kind != TDirEnd { + t.Fatalf("[%d] found end of directory, expected %#v", i, next) + } + } + check(dir) +} diff --git a/files/multifilereader_test.go b/files/multifilereader_test.go index fb4f749e5..21fd2eb6b 100644 --- a/files/multifilereader_test.go +++ b/files/multifilereader_test.go @@ -105,14 +105,10 @@ func TestMultiFileReaderToMultiFileSkip(t *testing.T) { func TestOutput(t *testing.T) { mfr := getTestMultiFileReader(t) - mpReader := &peekReader{r: multipart.NewReader(mfr, mfr.Boundary())} + walker := &multipartWalker{reader: multipart.NewReader(mfr, mfr.Boundary())} buf := make([]byte, 20) - part, err := mpReader.NextPart() - if part == nil || err != nil { - t.Fatal("Expected non-nil part, nil error") - } - mpname, mpf, err := newFileFromPart("", part, mpReader) + mpf, err := nextFile(walker) if mpf == nil || err != nil { t.Fatal("Expected non-nil multipartFile, nil error") } @@ -120,9 +116,6 @@ func TestOutput(t *testing.T) { if !ok { t.Fatal("Expected file to be a regular file") } - if mpname != "beep.txt" { - t.Fatal("Expected filename to be \"file.txt\"") - } if n, err := mpr.Read(buf); n != 4 || err != nil { t.Fatal("Expected to read from file", n, err) } @@ -130,11 +123,7 @@ func TestOutput(t *testing.T) { t.Fatal("Data read was different than expected") } - part, err = mpReader.NextPart() - if part == nil || err != nil { - t.Fatal("Expected non-nil part, nil error") - } - mpname, mpf, err = newFileFromPart("", part, mpReader) + mpf, err = nextFile(walker) if mpf == nil || err != nil { t.Fatal("Expected non-nil multipartFile, nil error") } @@ -142,58 +131,34 @@ func TestOutput(t *testing.T) { if !ok { t.Fatal("Expected file to be a directory") } - if mpname != "boop" { - t.Fatal("Expected filename to be \"boop\"") - } - part, err = mpReader.NextPart() - if part == nil || err != nil { - t.Fatal("Expected non-nil part, nil error") - } - cname, child, err := newFileFromPart("boop", part, mpReader) + child, err := nextFile(walker) if child == nil || err != nil { t.Fatal("Expected to be able to read a child file") } if _, ok := child.(File); !ok { t.Fatal("Expected file to not be a directory") } - if cname != "a.txt" { - t.Fatal("Expected filename to be \"a.txt\"") - } - part, err = mpReader.NextPart() - if part == nil || err != nil { - t.Fatal("Expected non-nil part, nil error") - } - cname, child, err = newFileFromPart("boop", part, mpReader) + child, err = nextFile(walker) if child == nil || err != nil { t.Fatal("Expected to be able to read a child file") } if _, ok := child.(File); !ok { t.Fatal("Expected file to not be a directory") } - if cname != "b.txt" { - t.Fatal("Expected filename to be \"b.txt\"") - } it := mpd.Entries() if it.Next() { t.Fatal("Expected to get false") } - part, err = mpReader.NextPart() - if part == nil || err != nil { - t.Fatal("Expected non-nil part, nil error") - } - mpname, mpf, err = newFileFromPart("", part, mpReader) + mpf, err = nextFile(walker) if mpf == nil || err != nil { t.Fatal("Expected non-nil multipartFile, nil error") } - if mpname != "file.txt" { - t.Fatal("Expected filename to be \"b.txt\"") - } - part, err = mpReader.NextPart() + part, err := walker.getPart() if part != nil || err != io.EOF { t.Fatal("Expected to get (nil, io.EOF)") } diff --git a/files/multipartfile.go b/files/multipartfile.go index b083e7049..0572e6b81 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -1,7 +1,6 @@ package files import ( - "errors" "io" "io/ioutil" "mime" @@ -22,86 +21,103 @@ const ( contentTypeHeader = "Content-Type" ) -var ErrPartOutsideParent = errors.New("file outside parent dir") -var ErrPartInChildTree = errors.New("file in child tree") +type multipartDirectory struct { + path string + walker *multipartWalker -// multipartFile implements Node, and is created from a `multipart.Part`. -type multipartFile struct { - Node + // part is the part describing the directory. It's nil when implicit. + part *multipart.Part +} - part *multipart.Part - reader *peekReader - mediatype string +type multipartWalker struct { + part *multipart.Part + reader *multipart.Reader } -func NewFileFromPartReader(reader *multipart.Reader, mediatype string) (Directory, error) { - if !isDirectory(mediatype) { - return nil, ErrNotDirectory - } +func (m *multipartWalker) consumePart() { + m.part = nil +} - f := &multipartFile{ - reader: &peekReader{r: reader}, - mediatype: mediatype, +func (m *multipartWalker) getPart() (*multipart.Part, error) { + if m.part != nil { + return m.part, nil + } + if m.reader == nil { + return nil, io.EOF } - return f, nil + var err error + m.part, err = m.reader.NextPart() + if err == io.EOF { + m.reader = nil + } + return m.part, err } -func newFileFromPart(parent string, part *multipart.Part, reader *peekReader) (string, Node, error) { - f := &multipartFile{ - part: part, - reader: reader, +func NewFileFromPartReader(reader *multipart.Reader, mediatype string) (Directory, error) { + if !isDirectory(mediatype) { + return nil, ErrNotDirectory } - dir, base := path.Split(f.fileName()) - dir = path.Clean(dir) - parent = path.Clean(parent) - if dir == "." { - dir = "" - } - if parent == "." { - parent = "" - } + return &multipartDirectory{ + path: "/", + walker: &multipartWalker{ + reader: reader, + }, + }, nil +} - if dir != parent { - if strings.HasPrefix(dir, parent) { - return "", nil, ErrPartInChildTree - } - return "", nil, ErrPartOutsideParent +func nextFile(w *multipartWalker) (Node, error) { + part, err := w.getPart() + if err != nil { + return nil, err } + w.consumePart() contentType := part.Header.Get(contentTypeHeader) switch contentType { case applicationSymlink: out, err := ioutil.ReadAll(part) if err != nil { - return "", nil, err + return nil, err } - return base, NewLinkFile(string(out), nil), nil + return NewLinkFile(string(out), nil), nil case "": // default to application/octet-stream fallthrough case applicationFile: - return base, &ReaderFile{ + return &ReaderFile{ reader: part, abspath: part.Header.Get("abspath"), }, nil } - var err error - f.mediatype, _, err = mime.ParseMediaType(contentType) + mediatype, _, err := mime.ParseMediaType(contentType) if err != nil { - return "", nil, err + return nil, err } - if !isDirectory(f.mediatype) { - return base, &ReaderFile{ + if !isDirectory(mediatype) { + return &ReaderFile{ reader: part, abspath: part.Header.Get("abspath"), }, nil } - return base, f, nil + return &multipartDirectory{ + part: part, + path: fileName(part), + walker: w, + }, nil +} + +func fileName(part *multipart.Part) string { + filename := part.FileName() + if escaped, err := url.QueryUnescape(filename); err == nil { + filename = escaped + } // if there is a unescape error, just treat the name as unescaped + + return path.Clean("/" + filename) } func isDirectory(mediatype string) bool { @@ -109,7 +125,7 @@ func isDirectory(mediatype string) bool { } type multipartIterator struct { - f *multipartFile + f *multipartDirectory curFile Node curName string @@ -125,102 +141,71 @@ func (it *multipartIterator) Node() Node { } func (it *multipartIterator) Next() bool { - if it.f.reader == nil { + if it.f.walker.reader == nil || it.err != nil { return false } var part *multipart.Part for { - var err error - part, err = it.f.reader.NextPart() - if err != nil { - if err == io.EOF { - return false - } - it.err = err + part, it.err = it.f.walker.getPart() + if it.err != nil { + return false + } + + name := fileName(part) + + // Is the file in a different directory? + if !strings.HasPrefix(name, it.f.path) { return false } - name, cf, err := newFileFromPart(it.f.fileName(), part, it.f.reader) - if err == ErrPartOutsideParent { - break + // Have we already entered this directory? + if it.curName != "" && strings.HasPrefix(name, path.Join(it.f.path, it.curName)) { + it.f.walker.consumePart() + continue } - if err != ErrPartInChildTree { - it.curFile = cf - it.curName = name - it.err = err - return err == nil + + // Make the path relative to the current directory. + name = strings.TrimLeft(name[len(it.f.path):], "/") + + // Check if we need to create a fake directory (more than one + // path component). + if idx := strings.IndexByte(name, '/'); idx >= 0 { + it.curName = name[:idx] + it.curFile = &multipartDirectory{ + path: path.Join(it.f.path, it.curName), + walker: it.f.walker, + } + return true } - } + it.curName = name + + // Finally, advance to the next file. + it.curFile, it.err = nextFile(it.f.walker) - it.err = it.f.reader.put(part) - return false + return it.err == nil + } } func (it *multipartIterator) Err() error { + if it.err == io.EOF { + return nil + } return it.err } -func (f *multipartFile) Entries() DirIterator { +func (f *multipartDirectory) Entries() DirIterator { return &multipartIterator{f: f} } -func (f *multipartFile) fileName() string { - if f == nil || f.part == nil { - return "" - } - - filename, err := url.QueryUnescape(f.part.FileName()) - if err != nil { - // if there is a unescape error, just treat the name as unescaped - return f.part.FileName() - } - return filename -} - -func (f *multipartFile) Close() error { +func (f *multipartDirectory) Close() error { if f.part != nil { return f.part.Close() } return nil } -func (f *multipartFile) Size() (int64, error) { +func (f *multipartDirectory) Size() (int64, error) { return 0, ErrNotSupported } -type PartReader interface { - NextPart() (*multipart.Part, error) -} - -type peekReader struct { - r PartReader - next *multipart.Part -} - -func (pr *peekReader) NextPart() (*multipart.Part, error) { - if pr.next != nil { - p := pr.next - pr.next = nil - return p, nil - } - - if pr.r == nil { - return nil, io.EOF - } - - p, err := pr.r.NextPart() - if err == io.EOF { - pr.r = nil - } - return p, err -} - -func (pr *peekReader) put(p *multipart.Part) error { - if pr.next != nil { - return errors.New("cannot put multiple parts") - } - pr.next = p - return nil -} - -var _ Directory = &multipartFile{} +var _ Directory = &multipartDirectory{} From 93e77eede86d98bbd8ae530d2050dc545c78e0fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 11 Feb 2019 08:24:57 -0800 Subject: [PATCH 35/76] multipart: attach nextFile to multipartWalker Co-Authored-By: Stebalien This commit was moved from ipfs/go-ipfs-files@8df416970ffed933e416601f5125fbd20d79128e --- files/multifilereader_test.go | 10 +++++----- files/multipartfile.go | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/files/multifilereader_test.go b/files/multifilereader_test.go index 21fd2eb6b..f038f5368 100644 --- a/files/multifilereader_test.go +++ b/files/multifilereader_test.go @@ -108,7 +108,7 @@ func TestOutput(t *testing.T) { walker := &multipartWalker{reader: multipart.NewReader(mfr, mfr.Boundary())} buf := make([]byte, 20) - mpf, err := nextFile(walker) + mpf, err := walker.nextFile() if mpf == nil || err != nil { t.Fatal("Expected non-nil multipartFile, nil error") } @@ -123,7 +123,7 @@ func TestOutput(t *testing.T) { t.Fatal("Data read was different than expected") } - mpf, err = nextFile(walker) + mpf, err = walker.nextFile() if mpf == nil || err != nil { t.Fatal("Expected non-nil multipartFile, nil error") } @@ -132,7 +132,7 @@ func TestOutput(t *testing.T) { t.Fatal("Expected file to be a directory") } - child, err := nextFile(walker) + child, err := walker.nextFile() if child == nil || err != nil { t.Fatal("Expected to be able to read a child file") } @@ -140,7 +140,7 @@ func TestOutput(t *testing.T) { t.Fatal("Expected file to not be a directory") } - child, err = nextFile(walker) + child, err = walker.nextFile() if child == nil || err != nil { t.Fatal("Expected to be able to read a child file") } @@ -153,7 +153,7 @@ func TestOutput(t *testing.T) { t.Fatal("Expected to get false") } - mpf, err = nextFile(walker) + mpf, err = walker.nextFile() if mpf == nil || err != nil { t.Fatal("Expected non-nil multipartFile, nil error") } diff --git a/files/multipartfile.go b/files/multipartfile.go index 0572e6b81..8f3ee25a6 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -67,7 +67,7 @@ func NewFileFromPartReader(reader *multipart.Reader, mediatype string) (Director }, nil } -func nextFile(w *multipartWalker) (Node, error) { +func (w *multipartWalker) nextFile() (Node, error) { part, err := w.getPart() if err != nil { return nil, err @@ -180,7 +180,7 @@ func (it *multipartIterator) Next() bool { it.curName = name // Finally, advance to the next file. - it.curFile, it.err = nextFile(it.f.walker) + it.curFile, it.err = it.f.walker.nextFile() return it.err == nil } From 4e2b29f378ec0f7efa593f612ebdf0f614dee0ff Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 11 Feb 2019 08:29:21 -0800 Subject: [PATCH 36/76] multipart: comment on why we stash EOF in the iterator This commit was moved from ipfs/go-ipfs-files@57067a822243a6f5b10cabf50d49c59d88a600cb --- files/multipartfile.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/files/multipartfile.go b/files/multipartfile.go index 8f3ee25a6..e91beb48a 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -187,6 +187,8 @@ func (it *multipartIterator) Next() bool { } func (it *multipartIterator) Err() error { + // We use EOF to signal that this iterator is done. That way, we don't + // need to check every time `Next` is called. if it.err == io.EOF { return nil } From 85db9ec3991fc4d4006014a7b4785fe4152efd31 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 11 Feb 2019 09:35:31 -0800 Subject: [PATCH 37/76] multipart: fix handling of common prefixes This commit was moved from ipfs/go-ipfs-files@da001e456ff5b147ee3098ad1d3c07e06e4e9e1c --- files/multifilereader_test.go | 40 +++++++++++++++++++++++++++++++++++ files/multipartfile.go | 29 ++++++++++++++++++++++--- 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/files/multifilereader_test.go b/files/multifilereader_test.go index f038f5368..34cdd151e 100644 --- a/files/multifilereader_test.go +++ b/files/multifilereader_test.go @@ -163,3 +163,43 @@ func TestOutput(t *testing.T) { t.Fatal("Expected to get (nil, io.EOF)") } } + +func TestCommonPrefix(t *testing.T) { + sf := NewMapDirectory(map[string]Node{ + "boop": NewMapDirectory(map[string]Node{ + "a": NewBytesFile([]byte("bleep")), + "aa": NewBytesFile([]byte("bleep")), + "aaa": NewBytesFile([]byte("bleep")), + }), + }) + mfr := NewMultiFileReader(sf, true) + reader, err := NewFileFromPartReader(multipart.NewReader(mfr, mfr.Boundary()), multipartFormdataType) + if err != nil { + t.Fatal(err) + } + + CheckDir(t, reader, []Event{ + { + kind: TDirStart, + name: "boop", + }, + { + kind: TFile, + name: "a", + value: "bleep", + }, + { + kind: TFile, + name: "aa", + value: "bleep", + }, + { + kind: TFile, + name: "aaa", + value: "bleep", + }, + { + kind: TDirEnd, + }, + }) +} diff --git a/files/multipartfile.go b/files/multipartfile.go index e91beb48a..17681653f 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -111,6 +111,7 @@ func (w *multipartWalker) nextFile() (Node, error) { }, nil } +// fileName returns a normalized filename from a part. func fileName(part *multipart.Part) string { filename := part.FileName() if escaped, err := url.QueryUnescape(filename); err == nil { @@ -120,10 +121,32 @@ func fileName(part *multipart.Part) string { return path.Clean("/" + filename) } +// dirName appends a slash to the end of the filename, if not present. +// expects a _cleaned_ path. +func dirName(filename string) string { + if !strings.HasSuffix(filename, "/") { + filename += "/" + } + return filename +} + +// isDirectory checks if the media type is a valid directory media type. func isDirectory(mediatype string) bool { return mediatype == multipartFormdataType || mediatype == applicationDirectory } +// isChild checks if child is a child of parent directory. +// expects a _cleaned_ path. +func isChild(child, parent string) bool { + return strings.HasPrefix(child, dirName(parent)) +} + +// makeRelative makes the child path relative to the parent path. +// expects a _cleaned_ path. +func makeRelative(child, parent string) string { + return strings.TrimPrefix(child, dirName(parent)) +} + type multipartIterator struct { f *multipartDirectory @@ -154,18 +177,18 @@ func (it *multipartIterator) Next() bool { name := fileName(part) // Is the file in a different directory? - if !strings.HasPrefix(name, it.f.path) { + if !isChild(name, it.f.path) { return false } // Have we already entered this directory? - if it.curName != "" && strings.HasPrefix(name, path.Join(it.f.path, it.curName)) { + if it.curName != "" && isChild(name, path.Join(it.f.path, it.curName)) { it.f.walker.consumePart() continue } // Make the path relative to the current directory. - name = strings.TrimLeft(name[len(it.f.path):], "/") + name = makeRelative(name, it.f.path) // Check if we need to create a fake directory (more than one // path component). From 72615d98fc9878cb20937952aace2c72caf6d013 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 21 Feb 2019 09:58:14 -0800 Subject: [PATCH 38/76] simplify content type checking This commit was moved from ipfs/go-ipfs-files@35057144f74116c736474dd751a5f8f225879b36 --- files/multipartfile.go | 46 +++++++++++++++++------------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/files/multipartfile.go b/files/multipartfile.go index 17681653f..d4593ad6c 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -54,8 +54,11 @@ func (m *multipartWalker) getPart() (*multipart.Part, error) { return m.part, err } +// NewFileFromPartReader creates a Directory from a multipart reader. func NewFileFromPartReader(reader *multipart.Reader, mediatype string) (Directory, error) { - if !isDirectory(mediatype) { + switch mediatype { + case applicationDirectory, multipartFormdataType: + default: return nil, ErrNotDirectory } @@ -75,7 +78,21 @@ func (w *multipartWalker) nextFile() (Node, error) { w.consumePart() contentType := part.Header.Get(contentTypeHeader) + if contentType != "" { + var err error + contentType, _, err = mime.ParseMediaType(contentType) + if err != nil { + return nil, err + } + } + switch contentType { + case multipartFormdataType, applicationDirectory: + return &multipartDirectory{ + part: part, + path: fileName(part), + walker: w, + }, nil case applicationSymlink: out, err := ioutil.ReadAll(part) if err != nil { @@ -83,32 +100,12 @@ func (w *multipartWalker) nextFile() (Node, error) { } return NewLinkFile(string(out), nil), nil - case "": // default to application/octet-stream - fallthrough - case applicationFile: + default: return &ReaderFile{ reader: part, abspath: part.Header.Get("abspath"), }, nil } - - mediatype, _, err := mime.ParseMediaType(contentType) - if err != nil { - return nil, err - } - - if !isDirectory(mediatype) { - return &ReaderFile{ - reader: part, - abspath: part.Header.Get("abspath"), - }, nil - } - - return &multipartDirectory{ - part: part, - path: fileName(part), - walker: w, - }, nil } // fileName returns a normalized filename from a part. @@ -130,11 +127,6 @@ func dirName(filename string) string { return filename } -// isDirectory checks if the media type is a valid directory media type. -func isDirectory(mediatype string) bool { - return mediatype == multipartFormdataType || mediatype == applicationDirectory -} - // isChild checks if child is a child of parent directory. // expects a _cleaned_ path. func isChild(child, parent string) bool { From 270c79b87e4447ceadcb5967b10f262f86c58951 Mon Sep 17 00:00:00 2001 From: jmank88 Date: Sun, 17 Mar 2019 10:47:48 -0500 Subject: [PATCH 39/76] remove extra webfile test code This commit was moved from ipfs/go-ipfs-files@eeec4e804ef3cc2a9389ad47f38665edc69552d8 --- files/webfile_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/files/webfile_test.go b/files/webfile_test.go index 889cdc48d..ea8f0d7ae 100644 --- a/files/webfile_test.go +++ b/files/webfile_test.go @@ -10,10 +10,6 @@ import ( ) func TestWebFile(t *testing.T) { - http.HandleFunc("/my/url/content.txt", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, "Hello world!") - }) - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "Hello world!") })) From b0f20ff3bdde0e84b42ee589f1f3e7223439384b Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 20 Mar 2019 00:19:00 -0700 Subject: [PATCH 40/76] go format This commit was moved from ipfs/go-ipfs-files@f1ae2b139c75038e55090fb953b52802b8510e1a --- files/readerfile.go | 2 +- files/tarwriter.go | 1 - files/tarwriter_test.go | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/files/readerfile.go b/files/readerfile.go index c23300173..f98fec481 100644 --- a/files/readerfile.go +++ b/files/readerfile.go @@ -15,7 +15,7 @@ type ReaderFile struct { reader io.ReadCloser stat os.FileInfo - fsize int64 + fsize int64 } func NewBytesFile(b []byte) File { diff --git a/files/tarwriter.go b/files/tarwriter.go index 101d4c844..6d062726a 100644 --- a/files/tarwriter.go +++ b/files/tarwriter.go @@ -98,4 +98,3 @@ func writeSymlinkHeader(w *tar.Writer, target, fpath string) error { Typeflag: tar.TypeSymlink, }) } - diff --git a/files/tarwriter_test.go b/files/tarwriter_test.go index 38101d045..6b482912b 100644 --- a/files/tarwriter_test.go +++ b/files/tarwriter_test.go @@ -77,4 +77,4 @@ func TestTarWriter(t *testing.T) { if cur, err = tr.Next(); err != io.EOF { t.Fatal(err) } -} \ No newline at end of file +} From 8bf5a972711354ce0ba8c1c3b78ca92ea0eb940e Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 20 Mar 2019 00:00:49 -0700 Subject: [PATCH 41/76] fix the content disposition header * needs a name parameter * "file" isn't a valid disposition see https://github.com/ipfs/ipfs/issues/395 This commit was moved from ipfs/go-ipfs-files@cff97fc4a70883d3c924c37a21c8a4453d2fea08 --- files/multifilereader.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/files/multifilereader.go b/files/multifilereader.go index cf3d14c73..86867f68d 100644 --- a/files/multifilereader.go +++ b/files/multifilereader.go @@ -88,7 +88,12 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { // write the boundary and headers header := make(textproto.MIMEHeader) filename := url.QueryEscape(path.Join(path.Join(mfr.path...), entry.Name())) - header.Set("Content-Disposition", fmt.Sprintf("file; filename=\"%s\"", filename)) + dispositionPrefix := "attachment" + if mfr.form { + dispositionPrefix = "form-data; name=\"file\"" + } + + header.Set("Content-Disposition", fmt.Sprintf("%s; filename=\"%s\"", dispositionPrefix, filename)) var contentType string From 880ae9f42d89c7549c17fa74b771da7e9ba63345 Mon Sep 17 00:00:00 2001 From: jmank88 Date: Mon, 18 Mar 2019 21:16:44 -0500 Subject: [PATCH 42/76] return url as AbsPath from WebFile to implement FileInfo This commit was moved from ipfs/go-ipfs-files@14808cb938672991261b6ee5f7c89446f8c434c2 --- files/webfile.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/files/webfile.go b/files/webfile.go index dbc813bd6..0e26e277f 100644 --- a/files/webfile.go +++ b/files/webfile.go @@ -5,6 +5,7 @@ import ( "io" "net/http" "net/url" + "os" ) // WebFile is an implementation of File which reads it @@ -61,4 +62,13 @@ func (wf *WebFile) Size() (int64, error) { return wf.contentLength, nil } +func (wf *WebFile) AbsPath() string { + return wf.url.String() +} + +func (wf *WebFile) Stat() os.FileInfo { + return nil +} + var _ File = &WebFile{} +var _ FileInfo = &WebFile{} From 4fed0b87e40002a25868766c1d14ce74d1eaed69 Mon Sep 17 00:00:00 2001 From: jmank88 Date: Mon, 22 Apr 2019 16:25:20 -0500 Subject: [PATCH 43/76] check http status code during WebFile reads and return error for non-2XX This commit was moved from ipfs/go-ipfs-files@bb5d585a9e937dc20d6f15fd415c2d5e26f0ed3b --- files/webfile.go | 7 ++++++- files/webfile_test.go | 24 +++++++++++++++++++++--- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/files/webfile.go b/files/webfile.go index 0e26e277f..58208e391 100644 --- a/files/webfile.go +++ b/files/webfile.go @@ -2,6 +2,7 @@ package files import ( "errors" + "fmt" "io" "net/http" "net/url" @@ -31,10 +32,14 @@ func NewWebFile(url *url.URL) *WebFile { // reads will keep reading from the HTTP Request body. func (wf *WebFile) Read(b []byte) (int, error) { if wf.body == nil { - resp, err := http.Get(wf.url.String()) + s := wf.url.String() + resp, err := http.Get(s) if err != nil { return 0, err } + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return 0, fmt.Errorf("got non-2XX status code %d: %s", resp.StatusCode, s) + } wf.body = resp.Body wf.contentLength = resp.ContentLength } diff --git a/files/webfile_test.go b/files/webfile_test.go index ea8f0d7ae..11eaa2de2 100644 --- a/files/webfile_test.go +++ b/files/webfile_test.go @@ -10,8 +10,9 @@ import ( ) func TestWebFile(t *testing.T) { + const content = "Hello world!" s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, "Hello world!") + fmt.Fprintf(w, content) })) defer s.Close() @@ -24,7 +25,24 @@ func TestWebFile(t *testing.T) { if err != nil { t.Fatal(err) } - if string(body) != "Hello world!" { - t.Fatal("should have read the web file") + if string(body) != content { + t.Fatalf("expected %q but got %q", content, string(body)) + } +} + +func TestWebFile_notFound(t *testing.T) { + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "File not found.", http.StatusNotFound) + })) + defer s.Close() + + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + wf := NewWebFile(u) + _, err = ioutil.ReadAll(wf) + if err == nil { + t.Fatal("expected error") } } From 84f9621d4dc99ea8523c20a0ef2ee1ccd45b2d6a Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 25 Apr 2019 17:12:43 -0700 Subject: [PATCH 44/76] webfile: make Size() work before Read This commit was moved from ipfs/go-ipfs-files@f06ea3139d4f942e32e99debd1501e93470fd690 --- files/webfile.go | 24 ++++++++++++++------- files/webfile_test.go | 49 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 7 deletions(-) diff --git a/files/webfile.go b/files/webfile.go index 58208e391..594b81c82 100644 --- a/files/webfile.go +++ b/files/webfile.go @@ -26,23 +26,30 @@ func NewWebFile(url *url.URL) *WebFile { } } -// Read reads the File from it's web location. On the first -// call to Read, a GET request will be performed against the -// WebFile's URL, using Go's default HTTP client. Any further -// reads will keep reading from the HTTP Request body. -func (wf *WebFile) Read(b []byte) (int, error) { +func (wf *WebFile) start() error { if wf.body == nil { s := wf.url.String() resp, err := http.Get(s) if err != nil { - return 0, err + return err } if resp.StatusCode < 200 || resp.StatusCode > 299 { - return 0, fmt.Errorf("got non-2XX status code %d: %s", resp.StatusCode, s) + return fmt.Errorf("got non-2XX status code %d: %s", resp.StatusCode, s) } wf.body = resp.Body wf.contentLength = resp.ContentLength } + return nil +} + +// Read reads the File from it's web location. On the first +// call to Read, a GET request will be performed against the +// WebFile's URL, using Go's default HTTP client. Any further +// reads will keep reading from the HTTP Request body. +func (wf *WebFile) Read(b []byte) (int, error) { + if err := wf.start(); err != nil { + return 0, err + } return wf.body.Read(b) } @@ -60,6 +67,9 @@ func (wf *WebFile) Seek(offset int64, whence int) (int64, error) { } func (wf *WebFile) Size() (int64, error) { + if err := wf.start(); err != nil { + return 0, err + } if wf.contentLength < 0 { return -1, errors.New("Content-Length hearer was not set") } diff --git a/files/webfile_test.go b/files/webfile_test.go index 11eaa2de2..450dffc5b 100644 --- a/files/webfile_test.go +++ b/files/webfile_test.go @@ -46,3 +46,52 @@ func TestWebFile_notFound(t *testing.T) { t.Fatal("expected error") } } + +func TestWebFileSize(t *testing.T) { + body := "Hello world!" + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, body) + })) + defer s.Close() + + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + + // Read size before reading file. + + wf1 := NewWebFile(u) + if size, err := wf1.Size(); err != nil { + t.Error(err) + } else if int(size) != len(body) { + t.Errorf("expected size to be %d, got %d", len(body), size) + } + + actual, err := ioutil.ReadAll(wf1) + if err != nil { + t.Fatal(err) + } + if string(actual) != body { + t.Fatal("should have read the web file") + } + + wf1.Close() + + // Read size after reading file. + + wf2 := NewWebFile(u) + actual, err = ioutil.ReadAll(wf2) + if err != nil { + t.Fatal(err) + } + if string(actual) != body { + t.Fatal("should have read the web file") + } + + if size, err := wf2.Size(); err != nil { + t.Error(err) + } else if int(size) != len(body) { + t.Errorf("expected size to be %d, got %d", len(body), size) + } +} From 4b06448670ad391fb56582873c7935251e629420 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 9 Aug 2019 15:15:53 -0700 Subject: [PATCH 45/76] doc: fix formdata documentation This commit was moved from ipfs/go-ipfs-files@1320bcf01ed5d4cf1bd5b46892a5ec3271d7e982 --- files/multifilereader.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/files/multifilereader.go b/files/multifilereader.go index 86867f68d..f6f225a38 100644 --- a/files/multifilereader.go +++ b/files/multifilereader.go @@ -26,14 +26,14 @@ type MultiFileReader struct { closed bool mutex *sync.Mutex - // if true, the data will be type 'multipart/form-data' - // if false, the data will be type 'multipart/mixed' + // if true, the content disposition will be "form-data" + // if false, the content disposition will be "attachment" form bool } // NewMultiFileReader constructs a MultiFileReader. `file` can be any `commands.Directory`. -// If `form` is set to true, the multipart data will have a Content-Type of 'multipart/form-data', -// if `form` is false, the Content-Type will be 'multipart/mixed'. +// If `form` is set to true, the Content-Disposition will be "form-data". +// Otherwise, it will be "attachment". func NewMultiFileReader(file Directory, form bool) *MultiFileReader { it := file.Entries() From 264faca253483d4c9684246ef8882d9af6d9b617 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 15 Aug 2019 17:10:43 -0700 Subject: [PATCH 46/76] feat: add WriteTo function Writes a go-ipfs-files Node to a location in the filesystem. This commit was moved from ipfs/go-ipfs-files@d372278de0e798cced3b2b5b513fd2f8907b9756 --- files/filewriter.go | 43 +++++++++++++++++++++++ files/filewriter_test.go | 74 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 files/filewriter.go create mode 100644 files/filewriter_test.go diff --git a/files/filewriter.go b/files/filewriter.go new file mode 100644 index 000000000..c42b3c33e --- /dev/null +++ b/files/filewriter.go @@ -0,0 +1,43 @@ +package files + +import ( + "fmt" + "io" + "os" + "path/filepath" +) + +// WriteTo writes the given node to the local filesystem at fpath. +func WriteTo(nd Node, fpath string) error { + switch nd := nd.(type) { + case *Symlink: + return os.Symlink(nd.Target, fpath) + case File: + f, err := os.Create(fpath) + defer f.Close() + if err != nil { + return err + } + _, err = io.Copy(f, nd) + if err != nil { + return err + } + return nil + case Directory: + err := os.Mkdir(fpath, 0777) + if err != nil { + return err + } + + entries := nd.Entries() + for entries.Next() { + child := filepath.Join(fpath, entries.Name()) + if err := WriteTo(entries.Node(), child); err != nil { + return err + } + } + return entries.Err() + default: + return fmt.Errorf("file type %T at %q is not supported", nd, fpath) + } +} diff --git a/files/filewriter_test.go b/files/filewriter_test.go new file mode 100644 index 000000000..d80ac916d --- /dev/null +++ b/files/filewriter_test.go @@ -0,0 +1,74 @@ +package files + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +func TestWriteTo(t *testing.T) { + sf := NewMapDirectory(map[string]Node{ + "1": NewBytesFile([]byte("Some text!\n")), + "2": NewBytesFile([]byte("beep")), + "3": NewMapDirectory(nil), + "4": NewBytesFile([]byte("boop")), + "5": NewMapDirectory(map[string]Node{ + "a": NewBytesFile([]byte("foobar")), + }), + }) + tmppath, err := ioutil.TempDir("", "files-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmppath) + + path := tmppath + "/output" + + err = WriteTo(sf, path) + if err != nil { + t.Fatal(err) + } + expected := map[string]string{ + ".": "", + "1": "Some text!\n", + "2": "beep", + "3": "", + "4": "boop", + "5": "", + "5/a": "foobar", + } + err = filepath.Walk(path, func(cpath string, info os.FileInfo, err error) error { + rpath, err := filepath.Rel(path, cpath) + if err != nil { + return err + } + data, ok := expected[rpath] + if !ok { + return fmt.Errorf("expected something at %q", rpath) + } + delete(expected, rpath) + + if info.IsDir() { + if data != "" { + return fmt.Errorf("expected a directory at %q", rpath) + } + } else { + actual, err := ioutil.ReadFile(cpath) + if err != nil { + return err + } + if string(actual) != data { + return fmt.Errorf("expected %q, got %q", data, string(actual)) + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } + if len(expected) > 0 { + t.Fatalf("failed to find: %#v", expected) + } +} From fc2a1a210b1758cc89ffa71fbe0a234d7123b824 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 15 Aug 2019 17:42:13 -0700 Subject: [PATCH 47/76] serialfile: fix handling of hidden paths on windows fixes #11 This commit was moved from ipfs/go-ipfs-files@a7bc21a37b777217e30b332761593c7dfbab0216 --- files/is_hidden.go | 19 +++++++++---------- files/is_hidden_windows.go | 27 ++++++++++----------------- files/serialfile.go | 2 +- 3 files changed, 20 insertions(+), 28 deletions(-) diff --git a/files/is_hidden.go b/files/is_hidden.go index 4ebca6008..27960ac08 100644 --- a/files/is_hidden.go +++ b/files/is_hidden.go @@ -1,18 +1,17 @@ -// +build !windows +//+build !windows package files import ( - "path/filepath" - "strings" + "os" ) -func IsHidden(name string, f Node) bool { - fName := filepath.Base(name) - - if strings.HasPrefix(fName, ".") && len(fName) > 1 { - return true +func isHidden(fi os.FileInfo) bool { + fName := fi.Name() + switch fName { + case "", ".", "..": + return false + default: + return fName[0] == '.' } - - return false } diff --git a/files/is_hidden_windows.go b/files/is_hidden_windows.go index 7419f932e..6f8bd7870 100644 --- a/files/is_hidden_windows.go +++ b/files/is_hidden_windows.go @@ -3,33 +3,26 @@ package files import ( - "path/filepath" - "strings" + "os" windows "golang.org/x/sys/windows" ) -func IsHidden(name string, f Node) bool { - - fName := filepath.Base(name) +func isHidden(fi os.FileInfo) bool { + fName := fi.Name() + switch fName { + case "", ".", "..": + return false + } - if strings.HasPrefix(fName, ".") && len(fName) > 1 { + if fName[0] == '.' { return true } - fi, ok := f.(FileInfo) + wi, ok := fi.Sys().(*windows.Win32FileAttributeData) if !ok { return false } - p, e := windows.UTF16PtrFromString(fi.AbsPath()) - if e != nil { - return false - } - - attrs, e := windows.GetFileAttributes(p) - if e != nil { - return false - } - return attrs&windows.FILE_ATTRIBUTE_HIDDEN != 0 + return wi.FileAttributes&windows.FILE_ATTRIBUTE_HIDDEN != 0 } diff --git a/files/serialfile.go b/files/serialfile.go index e29752d66..75a73b57c 100644 --- a/files/serialfile.go +++ b/files/serialfile.go @@ -75,7 +75,7 @@ func (it *serialIterator) Next() bool { stat := it.files[0] it.files = it.files[1:] - for !it.handleHiddenFiles && strings.HasPrefix(stat.Name(), ".") { + for !it.handleHiddenFiles && isHidden(stat) { if len(it.files) == 0 { return false } From caa31c7633feecf8ccb9202ad975d7ef57232395 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 15 Aug 2019 19:19:21 -0700 Subject: [PATCH 48/76] test: fix an edge case This commit was moved from ipfs/go-ipfs-files@f3569282d35970223610ad3895b81e552df0d94c --- files/filewriter_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/files/filewriter_test.go b/files/filewriter_test.go index d80ac916d..5809aba57 100644 --- a/files/filewriter_test.go +++ b/files/filewriter_test.go @@ -40,6 +40,9 @@ func TestWriteTo(t *testing.T) { "5/a": "foobar", } err = filepath.Walk(path, func(cpath string, info os.FileInfo, err error) error { + if err != nil { + return err + } rpath, err := filepath.Rel(path, cpath) if err != nil { return err From 694bbf09769c7ac6df3c8d16922d2bfcf3fe3457 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 15 Aug 2019 19:19:33 -0700 Subject: [PATCH 49/76] walk: add a walk function This commit was moved from ipfs/go-ipfs-files@e1ba4f66e7ebec9f8744703808e4adc43afe0faf --- files/walk.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 files/walk.go diff --git a/files/walk.go b/files/walk.go new file mode 100644 index 000000000..f23e7e47f --- /dev/null +++ b/files/walk.go @@ -0,0 +1,27 @@ +package files + +import ( + "path/filepath" +) + +// Walk walks a file tree, like `os.Walk`. +func Walk(nd Node, cb func(fpath string, nd Node) error) error { + var helper func(string, Node) error + helper = func(path string, nd Node) error { + if err := cb(path, nd); err != nil { + return err + } + dir, ok := nd.(Directory) + if !ok { + return nil + } + iter := dir.Entries() + for iter.Next() { + if err := helper(filepath.Join(path, iter.Name()), iter.Node()); err != nil { + return err + } + } + return iter.Err() + } + return helper("", nd) +} From 626c7bf496687857fb8d1b41d1c1d78984ec12f9 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 15 Aug 2019 19:19:47 -0700 Subject: [PATCH 50/76] test: test walk, serialfile, and hidden This commit was moved from ipfs/go-ipfs-files@f0180f2755b56d79131e9062fcec7caccb886e5a --- files/serialfile_test.go | 126 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 files/serialfile_test.go diff --git a/files/serialfile_test.go b/files/serialfile_test.go new file mode 100644 index 000000000..748ba16fa --- /dev/null +++ b/files/serialfile_test.go @@ -0,0 +1,126 @@ +package files + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" +) + +func isPathHidden(p string) bool { + return strings.HasPrefix(p, ".") || strings.Contains(p, "/.") +} + +func TestSerialFile(t *testing.T) { + t.Run("Hidden", func(t *testing.T) { testSerialFile(t, true) }) + t.Run("NotHidden", func(t *testing.T) { testSerialFile(t, false) }) +} + +func testSerialFile(t *testing.T, hidden bool) { + tmppath, err := ioutil.TempDir("", "files-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmppath) + + expected := map[string]string{ + "1": "Some text!\n", + "2": "beep", + "3": "", + "4": "boop", + "5": "", + "5/a": "foobar", + ".6": "thing", + "7": "", + "7/.foo": "bla", + ".8": "", + ".8/foo": "bla", + } + + for p, c := range expected { + path := filepath.Join(tmppath, p) + if c != "" { + continue + } + if err := os.MkdirAll(path, 0777); err != nil { + t.Fatal(err) + } + } + + for p, c := range expected { + path := filepath.Join(tmppath, p) + if c == "" { + continue + } + if err := ioutil.WriteFile(path, []byte(c), 0666); err != nil { + t.Fatal(err) + } + } + + stat, err := os.Stat(tmppath) + if err != nil { + t.Fatal(err) + } + + sf, err := NewSerialFile(tmppath, hidden, stat) + if err != nil { + t.Fatal(err) + } + defer sf.Close() + + rootFound := false + err = Walk(sf, func(path string, nd Node) error { + defer nd.Close() + + // root node. + if path == "" { + if rootFound { + return fmt.Errorf("found root twice") + } + if sf != nd { + return fmt.Errorf("wrong root") + } + rootFound = true + return nil + } + + if !hidden && isPathHidden(path) { + return fmt.Errorf("found a hidden file") + } + + data, ok := expected[path] + if !ok { + return fmt.Errorf("expected something at %q", path) + } + delete(expected, path) + + switch nd := nd.(type) { + case *Symlink: + return fmt.Errorf("didn't expect a symlink") + case Directory: + if data != "" { + return fmt.Errorf("expected a directory at %q", path) + } + case File: + actual, err := ioutil.ReadAll(nd) + if err != nil { + return err + } + if string(actual) != data { + return fmt.Errorf("expected %q, got %q", data, string(actual)) + } + } + return nil + }) + if !rootFound { + t.Fatal("didn't find the root") + } + for p := range expected { + if !hidden && isPathHidden(p) { + continue + } + t.Errorf("missed %q", p) + } +} From 49818bad4473f8cbe2614fdeec5f3e95f773361e Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 18 Sep 2019 22:28:46 -0700 Subject: [PATCH 51/76] doc: add a lead maintainer This commit was moved from ipfs/go-ipfs-files@8d2cd18a0c1609ed78422660a29f1e461e60edc6 --- files/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/files/README.md b/files/README.md index 4f7046954..e0205d78a 100644 --- a/files/README.md +++ b/files/README.md @@ -7,6 +7,10 @@ > File interfaces and utils used in IPFS +## Lead Maintainer + +[Steven Allen](https://github.com/Stebalien) + ## Documentation https://godoc.org/github.com/ipfs/go-ipfs-files From 280ebca77993e3c8b19afb55b9fa342f8e5388d2 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 25 Sep 2019 22:49:21 -0700 Subject: [PATCH 52/76] feat: correctly report the size of symlinks Also, symlinks don't use stat. This commit was moved from ipfs/go-ipfs-files@777d1540bc2fa6c24aba7b7fd910e95e206f1667 --- files/linkfile.go | 27 +++++++-------------------- files/multipartfile.go | 2 +- files/serialfile.go | 2 +- 3 files changed, 9 insertions(+), 22 deletions(-) diff --git a/files/linkfile.go b/files/linkfile.go index 409309bca..64e87625c 100644 --- a/files/linkfile.go +++ b/files/linkfile.go @@ -1,31 +1,22 @@ package files import ( - "io" - "os" "strings" ) type Symlink struct { Target string - stat os.FileInfo - reader io.Reader + reader strings.Reader } -func NewLinkFile(target string, stat os.FileInfo) File { - return &Symlink{ - Target: target, - stat: stat, - reader: strings.NewReader(target), - } +func NewLinkFile(target string) File { + lf := &Symlink{Target: target} + lf.reader.Reset(lf.Target) + return lf } func (lf *Symlink) Close() error { - if c, ok := lf.reader.(io.Closer); ok { - return c.Close() - } - return nil } @@ -34,15 +25,11 @@ func (lf *Symlink) Read(b []byte) (int, error) { } func (lf *Symlink) Seek(offset int64, whence int) (int64, error) { - if s, ok := lf.reader.(io.Seeker); ok { - return s.Seek(offset, whence) - } - - return 0, ErrNotSupported + return lf.reader.Seek(offset, whence) } func (lf *Symlink) Size() (int64, error) { - return 0, ErrNotSupported + return lf.reader.Size(), nil } func ToSymlink(n Node) *Symlink { diff --git a/files/multipartfile.go b/files/multipartfile.go index d4593ad6c..0351e192b 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -99,7 +99,7 @@ func (w *multipartWalker) nextFile() (Node, error) { return nil, err } - return NewLinkFile(string(out), nil), nil + return NewLinkFile(string(out)), nil default: return &ReaderFile{ reader: part, diff --git a/files/serialfile.go b/files/serialfile.go index 75a73b57c..8e4bf592c 100644 --- a/files/serialfile.go +++ b/files/serialfile.go @@ -53,7 +53,7 @@ func NewSerialFile(path string, hidden bool, stat os.FileInfo) (Node, error) { if err != nil { return nil, err } - return NewLinkFile(target, stat), nil + return NewLinkFile(target), nil default: return nil, fmt.Errorf("unrecognized file type for %s: %s", path, mode.String()) } From dbc14ee03b49d5728a5e6a406a39283aacede2ae Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 26 Sep 2019 12:29:31 -0700 Subject: [PATCH 53/76] revert(symlink): keep stat argument I thought this wasn't used outside this package. Apparently, I didn't run grep from the right directory. It doesn't _hurt_ to keep this. This commit was moved from ipfs/go-ipfs-files@31f7f20851b4efcf15033ae56c876dcd8840cc4e --- files/linkfile.go | 6 ++++-- files/multipartfile.go | 2 +- files/serialfile.go | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/files/linkfile.go b/files/linkfile.go index 64e87625c..526998652 100644 --- a/files/linkfile.go +++ b/files/linkfile.go @@ -1,17 +1,19 @@ package files import ( + "os" "strings" ) type Symlink struct { Target string + stat os.FileInfo reader strings.Reader } -func NewLinkFile(target string) File { - lf := &Symlink{Target: target} +func NewLinkFile(target string, stat os.FileInfo) File { + lf := &Symlink{Target: target, stat: stat} lf.reader.Reset(lf.Target) return lf } diff --git a/files/multipartfile.go b/files/multipartfile.go index 0351e192b..d4593ad6c 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -99,7 +99,7 @@ func (w *multipartWalker) nextFile() (Node, error) { return nil, err } - return NewLinkFile(string(out)), nil + return NewLinkFile(string(out), nil), nil default: return &ReaderFile{ reader: part, diff --git a/files/serialfile.go b/files/serialfile.go index 8e4bf592c..75a73b57c 100644 --- a/files/serialfile.go +++ b/files/serialfile.go @@ -53,7 +53,7 @@ func NewSerialFile(path string, hidden bool, stat os.FileInfo) (Node, error) { if err != nil { return nil, err } - return NewLinkFile(target), nil + return NewLinkFile(target, stat), nil default: return nil, fmt.Errorf("unrecognized file type for %s: %s", path, mode.String()) } From 89a095148e2b5e3b64f255b23bec9c15777f278a Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 16 Mar 2020 19:09:48 -0700 Subject: [PATCH 54/76] chore: remove dead code This code was from a previous incarnation of this interface. This commit was moved from ipfs/go-ipfs-files@ec25a68e5dc2bd9acfd2eb6842adc73c2e0c575d --- files/serialfile.go | 39 ++------------------------------------- 1 file changed, 2 insertions(+), 37 deletions(-) diff --git a/files/serialfile.go b/files/serialfile.go index 75a73b57c..cd6016011 100644 --- a/files/serialfile.go +++ b/files/serialfile.go @@ -3,16 +3,13 @@ package files import ( "errors" "fmt" - "io" "io/ioutil" "os" "path/filepath" - "strings" ) // serialFile implements Node, and reads from a path on the OS filesystem. -// No more than one file will be opened at a time (directories will advance -// to the next file when NextFile() is called). +// No more than one file will be opened at a time. type serialFile struct { path string files []os.FileInfo @@ -42,7 +39,7 @@ func NewSerialFile(path string, hidden bool, stat os.FileInfo) (Node, error) { return NewReaderPathFile(path, file, stat) case mode.IsDir(): // for directories, stat all of the contents first, so we know what files to - // open when NextFile() is called + // open when Entries() is called contents, err := ioutil.ReadDir(path) if err != nil { return nil, err @@ -113,38 +110,6 @@ func (f *serialFile) Entries() DirIterator { } } -func (f *serialFile) NextFile() (string, Node, error) { - // if there aren't any files left in the root directory, we're done - if len(f.files) == 0 { - return "", nil, io.EOF - } - - stat := f.files[0] - f.files = f.files[1:] - - for !f.handleHiddenFiles && strings.HasPrefix(stat.Name(), ".") { - if len(f.files) == 0 { - return "", nil, io.EOF - } - - stat = f.files[0] - f.files = f.files[1:] - } - - // open the next file - filePath := filepath.ToSlash(filepath.Join(f.path, stat.Name())) - - // recursively call the constructor on the next file - // if it's a regular file, we will open it as a ReaderFile - // if it's a directory, files in it will be opened serially - sf, err := NewSerialFile(filePath, f.handleHiddenFiles, stat) - if err != nil { - return "", nil, err - } - - return stat.Name(), sf, nil -} - func (f *serialFile) Close() error { return nil } From 497fdd749af54b67f4d23e29b22ddecd2aa39705 Mon Sep 17 00:00:00 2001 From: Cornelius Toole Date: Sat, 15 Feb 2020 19:04:18 -0600 Subject: [PATCH 55/76] feat(file-ignore): add file ignore rules to serialfile - add a filter that defines rules for ignoring hidden and/or files listed explicitly or defined in a gitignore file - update SerialFile construct to accept a Filter feat(add-file-ignore): fix tests, slight refactor - add new SerialFile constructor with original signature for backward compatibility - update tests for new SerialFile behavior feat(file-ignore): cleanup code+tests - address PR comments - add more documentation - use existing function for cross-platform hidden file detection - be more consistent when checking for hidden fies - add more examples+test cases feat(file-ignore): rework `Filter` constructor - add filter tests feat(file-ignore): apply exclude rules lazily - apply exclude rules from filter when iterating over dir contents instead of at serialFile construction time This commit was moved from ipfs/go-ipfs-files@90aef3a9c0acfd15fa4e2ee07eda4c1e1b86817a --- files/filter.go | 49 +++++++++++++++++++++++++ files/filter_test.go | 50 ++++++++++++++++++++++++++ files/serialfile.go | 42 ++++++++++++++-------- files/serialfile_test.go | 77 ++++++++++++++++++++++++++++++++-------- 4 files changed, 188 insertions(+), 30 deletions(-) create mode 100644 files/filter.go create mode 100644 files/filter_test.go diff --git a/files/filter.go b/files/filter.go new file mode 100644 index 000000000..6b90f1f34 --- /dev/null +++ b/files/filter.go @@ -0,0 +1,49 @@ +package files + +import ( + "os" + + ignore "github.com/crackcomm/go-gitignore" +) + +// Filter represents a set of rules for determining if a file should be included or excluded. +// A rule follows the syntax for patterns used in .gitgnore files for specifying untracked files. +// Examples: +// foo.txt +// *.app +// bar/ +// **/baz +// fizz/** +type Filter struct { + // IncludeHidden - Include hidden files + IncludeHidden bool + // Rules - File filter rules + Rules *ignore.GitIgnore +} + +// NewFilter creates a new file filter from a .gitignore file and/or a list of ignore rules. +// An ignoreFile is a path to a file with .gitignore-style patterns to exclude, one per line +// rules is an array of strings representing .gitignore-style patterns +// For reference on ignore rule syntax, see https://git-scm.com/docs/gitignore +func NewFilter(ignoreFile string, rules []string, includeHidden bool) (*Filter, error) { + var ignoreRules *ignore.GitIgnore + var err error + if ignoreFile == "" { + ignoreRules, err = ignore.CompileIgnoreLines(rules...) + } else { + ignoreRules, err = ignore.CompileIgnoreFileAndLines(ignoreFile, rules...) + } + if err != nil { + return nil, err + } + return &Filter{IncludeHidden: includeHidden, Rules: ignoreRules}, nil +} + +// ShouldExclude takes an os.FileInfo object and applies rules to determine if its target should be excluded. +func (filter *Filter) ShouldExclude(fileInfo os.FileInfo) (result bool) { + path := fileInfo.Name() + if !filter.IncludeHidden && isHidden(fileInfo) { + return true + } + return filter.Rules.MatchesPath(path) +} diff --git a/files/filter_test.go b/files/filter_test.go new file mode 100644 index 000000000..d33b11429 --- /dev/null +++ b/files/filter_test.go @@ -0,0 +1,50 @@ +package files + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +type mockFileInfo struct { + os.FileInfo + name string +} + +func (m *mockFileInfo) Name() string { + return m.name +} + +var _ os.FileInfo = &mockFileInfo{} + +func TestFileFilter(t *testing.T) { + includeHidden := true + filter, err := NewFilter("", nil, includeHidden) + if err != nil { + t.Errorf("failed to create filter with empty rules") + } + if filter.IncludeHidden != includeHidden { + t.Errorf("new filter should include hidden files") + } + _, err = NewFilter("ignoreFileThatDoesNotExist", nil, false) + if err == nil { + t.Errorf("creating a filter without an invalid ignore file path should have failed") + } + tmppath, err := ioutil.TempDir("", "filter-test") + if err != nil { + t.Fatal(err) + } + ignoreFilePath := filepath.Join(tmppath, "ignoreFile") + ignoreFileContents := []byte("a.txt") + if err := ioutil.WriteFile(ignoreFilePath, ignoreFileContents, 0666); err != nil { + t.Fatal(err) + } + filterWithIgnoreFile, err := NewFilter(ignoreFilePath, nil, false) + if err != nil { + t.Errorf("failed to create filter with ignore file") + } + if !filterWithIgnoreFile.ShouldExclude(&mockFileInfo{name: "a.txt"}) { + t.Errorf("filter should've excluded expected file from ignoreFile: %s", "a.txt") + } +} diff --git a/files/serialfile.go b/files/serialfile.go index cd6016011..f56fcc6eb 100644 --- a/files/serialfile.go +++ b/files/serialfile.go @@ -11,16 +11,16 @@ import ( // serialFile implements Node, and reads from a path on the OS filesystem. // No more than one file will be opened at a time. type serialFile struct { - path string - files []os.FileInfo - stat os.FileInfo - handleHiddenFiles bool + path string + files []os.FileInfo + stat os.FileInfo + filter *Filter } type serialIterator struct { - files []os.FileInfo - handleHiddenFiles bool - path string + files []os.FileInfo + path string + filter *Filter curName string curFile Node @@ -28,8 +28,20 @@ type serialIterator struct { err error } -// TODO: test/document limitations -func NewSerialFile(path string, hidden bool, stat os.FileInfo) (Node, error) { +// NewSerialFile takes a filepath, a bool specifying if hidden files should be included, +// and a fileInfo and returns a Node representing file, directory or special file. +func NewSerialFile(path string, includeHidden bool, stat os.FileInfo) (Node, error) { + filter, err := NewFilter("", nil, includeHidden) + if err != nil { + return nil, err + } + return NewSerialFileWithFilter(path, filter, stat) +} + +// NewSerialFileWith takes a filepath, a filter for determining which files should be +// operated upon if the filepath is a directory, and a fileInfo and returns a +// Node representing file, directory or special file. +func NewSerialFileWithFilter(path string, filter *Filter, stat os.FileInfo) (Node, error) { switch mode := stat.Mode(); { case mode.IsRegular(): file, err := os.Open(path) @@ -44,7 +56,7 @@ func NewSerialFile(path string, hidden bool, stat os.FileInfo) (Node, error) { if err != nil { return nil, err } - return &serialFile{path, contents, stat, hidden}, nil + return &serialFile{path, contents, stat, filter}, nil case mode&os.ModeSymlink != 0: target, err := os.Readlink(path) if err != nil { @@ -72,7 +84,7 @@ func (it *serialIterator) Next() bool { stat := it.files[0] it.files = it.files[1:] - for !it.handleHiddenFiles && isHidden(stat) { + for it.filter.ShouldExclude(stat) { if len(it.files) == 0 { return false } @@ -87,7 +99,7 @@ func (it *serialIterator) Next() bool { // recursively call the constructor on the next file // if it's a regular file, we will open it as a ReaderFile // if it's a directory, files in it will be opened serially - sf, err := NewSerialFile(filePath, it.handleHiddenFiles, stat) + sf, err := NewSerialFileWithFilter(filePath, it.filter, stat) if err != nil { it.err = err return false @@ -104,9 +116,9 @@ func (it *serialIterator) Err() error { func (f *serialFile) Entries() DirIterator { return &serialIterator{ - path: f.path, - files: f.files, - handleHiddenFiles: f.handleHiddenFiles, + path: f.path, + files: f.files, + filter: f.filter, } } diff --git a/files/serialfile_test.go b/files/serialfile_test.go index 748ba16fa..edd5bb95d 100644 --- a/files/serialfile_test.go +++ b/files/serialfile_test.go @@ -5,27 +5,30 @@ import ( "io/ioutil" "os" "path/filepath" + "sort" "strings" "testing" ) -func isPathHidden(p string) bool { +func isFullPathHidden(p string) bool { return strings.HasPrefix(p, ".") || strings.Contains(p, "/.") } func TestSerialFile(t *testing.T) { - t.Run("Hidden", func(t *testing.T) { testSerialFile(t, true) }) - t.Run("NotHidden", func(t *testing.T) { testSerialFile(t, false) }) + t.Run("Hidden/NoFilter", func(t *testing.T) { testSerialFile(t, true, false) }) + t.Run("Hidden/Filter", func(t *testing.T) { testSerialFile(t, true, true) }) + t.Run("NotHidden/NoFilter", func(t *testing.T) { testSerialFile(t, false, false) }) + t.Run("NotHidden/Filter", func(t *testing.T) { testSerialFile(t, false, true) }) } -func testSerialFile(t *testing.T, hidden bool) { +func testSerialFile(t *testing.T, hidden, withIgnoreRules bool) { tmppath, err := ioutil.TempDir("", "files-test") if err != nil { t.Fatal(err) } defer os.RemoveAll(tmppath) - expected := map[string]string{ + testInputs := map[string]string{ "1": "Some text!\n", "2": "beep", "3": "", @@ -38,8 +41,18 @@ func testSerialFile(t *testing.T, hidden bool) { ".8": "", ".8/foo": "bla", } + fileFilter, err := NewFilter("", []string{"9", "10"}, hidden) + if err != nil { + t.Fatal(err) + } + if withIgnoreRules { + testInputs["9"] = "" + testInputs["9/b"] = "bebop" + testInputs["10"] = "" + testInputs["10/.c"] = "doowop" + } - for p, c := range expected { + for p, c := range testInputs { path := filepath.Join(tmppath, p) if c != "" { continue @@ -49,7 +62,7 @@ func testSerialFile(t *testing.T, hidden bool) { } } - for p, c := range expected { + for p, c := range testInputs { path := filepath.Join(tmppath, p) if c == "" { continue @@ -58,6 +71,22 @@ func testSerialFile(t *testing.T, hidden bool) { t.Fatal(err) } } + expectedHiddenPaths := make([]string, 0, 4) + expectedRegularPaths := make([]string, 0, 6) + for p := range testInputs { + path := filepath.Join(tmppath, p) + stat, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if !fileFilter.ShouldExclude(stat) { + if isFullPathHidden(path) { + expectedHiddenPaths = append(expectedHiddenPaths, p) + } else { + expectedRegularPaths = append(expectedRegularPaths, p) + } + } + } stat, err := os.Stat(tmppath) if err != nil { @@ -65,12 +94,17 @@ func testSerialFile(t *testing.T, hidden bool) { } sf, err := NewSerialFile(tmppath, hidden, stat) + if withIgnoreRules { + sf, err = NewSerialFileWithFilter(tmppath, fileFilter, stat) + } if err != nil { t.Fatal(err) } defer sf.Close() rootFound := false + actualRegularPaths := make([]string, 0, len(expectedRegularPaths)) + actualHiddenPaths := make([]string, 0, len(expectedHiddenPaths)) err = Walk(sf, func(path string, nd Node) error { defer nd.Close() @@ -85,16 +119,23 @@ func testSerialFile(t *testing.T, hidden bool) { rootFound = true return nil } - - if !hidden && isPathHidden(path) { + if isFullPathHidden(path) { + actualHiddenPaths = append(actualHiddenPaths, path) + } else { + actualRegularPaths = append(actualRegularPaths, path) + } + if !hidden && isFullPathHidden(path) { return fmt.Errorf("found a hidden file") } + if fileFilter.Rules.MatchesPath(path) { + return fmt.Errorf("found a file that should be excluded") + } - data, ok := expected[path] + data, ok := testInputs[path] if !ok { return fmt.Errorf("expected something at %q", path) } - delete(expected, path) + delete(testInputs, path) switch nd := nd.(type) { case *Symlink: @@ -117,10 +158,16 @@ func testSerialFile(t *testing.T, hidden bool) { if !rootFound { t.Fatal("didn't find the root") } - for p := range expected { - if !hidden && isPathHidden(p) { - continue + for _, regular := range expectedRegularPaths { + if idx := sort.SearchStrings(actualRegularPaths, regular); idx < 0 { + t.Errorf("missed regular path %q", regular) + } + } + if hidden && len(actualHiddenPaths) != len(expectedHiddenPaths) { + for _, missing := range expectedHiddenPaths { + if idx := sort.SearchStrings(actualHiddenPaths, missing); idx < 0 { + t.Errorf("missed hidden path %q", missing) + } } - t.Errorf("missed %q", p) } } From b865765ec8c293af08698e1506f219ed0828f063 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Sun, 29 Mar 2020 15:47:38 -0700 Subject: [PATCH 56/76] fix: skip ignored files when calculating size fixes https://github.com/ipfs/go-ipfs/issues/7052 This commit was moved from ipfs/go-ipfs-files@241cb6114844012a15042999b1cf8439bf7ccdb0 --- files/serialfile.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/files/serialfile.go b/files/serialfile.go index f56fcc6eb..86af30680 100644 --- a/files/serialfile.go +++ b/files/serialfile.go @@ -138,13 +138,18 @@ func (f *serialFile) Size() (int64, error) { var du int64 err := filepath.Walk(f.path, func(p string, fi os.FileInfo, err error) error { - if err != nil { + if err != nil || fi == nil { return err } - if fi != nil && fi.Mode().IsRegular() { + if f.filter.ShouldExclude(fi) { + if fi.Mode().IsDir() { + return filepath.SkipDir + } + } else if fi.Mode().IsRegular() { du += fi.Size() } + return nil }) From 88bf27406eef5b3c52e999fb8f4c9dbe7af1f26b Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Sun, 29 Mar 2020 17:21:06 -0700 Subject: [PATCH 57/76] test: test file size when ignoring Also, fix the file ignore tests. This commit was moved from ipfs/go-ipfs-files@cc3f8bdd392fa5ebaa5e2b77bffea67336bedf62 --- files/serialfile_test.go | 81 +++++++++++++++++++++++++--------------- 1 file changed, 51 insertions(+), 30 deletions(-) diff --git a/files/serialfile_test.go b/files/serialfile_test.go index edd5bb95d..ee8da3ad3 100644 --- a/files/serialfile_test.go +++ b/files/serialfile_test.go @@ -71,23 +71,32 @@ func testSerialFile(t *testing.T, hidden, withIgnoreRules bool) { t.Fatal(err) } } - expectedHiddenPaths := make([]string, 0, 4) - expectedRegularPaths := make([]string, 0, 6) + expectedPaths := make([]string, 0, 4) + expectedSize := int64(0) + +testInputs: for p := range testInputs { - path := filepath.Join(tmppath, p) - stat, err := os.Stat(path) - if err != nil { - t.Fatal(err) - } - if !fileFilter.ShouldExclude(stat) { - if isFullPathHidden(path) { - expectedHiddenPaths = append(expectedHiddenPaths, p) - } else { - expectedRegularPaths = append(expectedRegularPaths, p) + components := strings.Split(p, "/") + var stat os.FileInfo + for i := range components { + stat, err = os.Stat(filepath.Join( + append([]string{tmppath}, components[:i+1]...)..., + )) + if err != nil { + t.Fatal(err) } + if fileFilter.ShouldExclude(stat) { + continue testInputs + } + } + expectedPaths = append(expectedPaths, p) + if stat.Mode().IsRegular() { + expectedSize += stat.Size() } } + sort.Strings(expectedPaths) + stat, err := os.Stat(tmppath) if err != nil { t.Fatal(err) @@ -102,9 +111,14 @@ func testSerialFile(t *testing.T, hidden, withIgnoreRules bool) { } defer sf.Close() + if size, err := sf.Size(); err != nil { + t.Fatalf("failed to determine size: %s", err) + } else if size != expectedSize { + t.Fatalf("expected size %d, got size %d", expectedSize, size) + } + rootFound := false - actualRegularPaths := make([]string, 0, len(expectedRegularPaths)) - actualHiddenPaths := make([]string, 0, len(expectedHiddenPaths)) + actualPaths := make([]string, 0, len(expectedPaths)) err = Walk(sf, func(path string, nd Node) error { defer nd.Close() @@ -119,16 +133,15 @@ func testSerialFile(t *testing.T, hidden, withIgnoreRules bool) { rootFound = true return nil } - if isFullPathHidden(path) { - actualHiddenPaths = append(actualHiddenPaths, path) - } else { - actualRegularPaths = append(actualRegularPaths, path) - } + actualPaths = append(actualPaths, path) if !hidden && isFullPathHidden(path) { return fmt.Errorf("found a hidden file") } - if fileFilter.Rules.MatchesPath(path) { - return fmt.Errorf("found a file that should be excluded") + components := filepath.SplitList(path) + for i := range components { + if fileFilter.Rules.MatchesPath(filepath.Join(components[:i+1]...)) { + return fmt.Errorf("found a file that should be excluded") + } } data, ok := testInputs[path] @@ -155,19 +168,27 @@ func testSerialFile(t *testing.T, hidden, withIgnoreRules bool) { } return nil }) + if err != nil { + t.Fatal(err) + } if !rootFound { t.Fatal("didn't find the root") } - for _, regular := range expectedRegularPaths { - if idx := sort.SearchStrings(actualRegularPaths, regular); idx < 0 { - t.Errorf("missed regular path %q", regular) - } + + if len(expectedPaths) != len(actualPaths) { + t.Fatalf("expected %d paths, found %d", + len(expectedPaths), + len(actualPaths), + ) } - if hidden && len(actualHiddenPaths) != len(expectedHiddenPaths) { - for _, missing := range expectedHiddenPaths { - if idx := sort.SearchStrings(actualHiddenPaths, missing); idx < 0 { - t.Errorf("missed hidden path %q", missing) - } + + for i := range expectedPaths { + if expectedPaths[i] != actualPaths[i] { + t.Errorf( + "expected path %q does not match actual %q", + expectedPaths[i], + actualPaths[i], + ) } } } From e43b527b84d2221223e2ddb5ead808dfd39bcb0d Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 2 Jun 2021 09:22:10 -0700 Subject: [PATCH 58/76] fix go vet This commit was moved from ipfs/go-ipfs-files@8adf174193ce90f6da52523b85eed54f8a262639 --- files/tarwriter_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/tarwriter_test.go b/files/tarwriter_test.go index 6b482912b..02686b567 100644 --- a/files/tarwriter_test.go +++ b/files/tarwriter_test.go @@ -26,7 +26,7 @@ func TestTarWriter(t *testing.T) { go func() { defer tw.Close() if err := tw.WriteFile(tf, ""); err != nil { - t.Fatal(err) + t.Error(err) } }() From f8b03ed4aec10e0ea69d612c4bfd5bf3a7e39bd6 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 2 Jun 2021 09:22:34 -0700 Subject: [PATCH 59/76] fix staticcheck This commit was moved from ipfs/go-ipfs-files@7ebda61409d14004a11905724112f15f02c26dd8 --- files/webfile_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/webfile_test.go b/files/webfile_test.go index 450dffc5b..57a67fe87 100644 --- a/files/webfile_test.go +++ b/files/webfile_test.go @@ -12,7 +12,7 @@ import ( func TestWebFile(t *testing.T) { const content = "Hello world!" s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, content) + fmt.Fprint(w, content) })) defer s.Close() From bf03bd60b8f0dbdb399a861bb58d31272b44450c Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 2 Jun 2021 09:57:19 -0700 Subject: [PATCH 60/76] fix staticcheck This commit was moved from ipfs/go-ipfs-files@5dc5da514d6dcacee8cfba72ebaed811a47b306c --- files/multifilereader_test.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/files/multifilereader_test.go b/files/multifilereader_test.go index 34cdd151e..e36788a91 100644 --- a/files/multifilereader_test.go +++ b/files/multifilereader_test.go @@ -30,11 +30,7 @@ func TestMultiFileReaderToMultiFile(t *testing.T) { t.Fatal(err) } - md, ok := mf.(Directory) - if !ok { - t.Fatal("Expected a directory") - } - it := md.Entries() + it := mf.Entries() if !it.Next() || it.Name() != "beep.txt" { t.Fatal("iterator didn't work as expected") @@ -80,11 +76,7 @@ func TestMultiFileReaderToMultiFileSkip(t *testing.T) { t.Fatal(err) } - md, ok := mf.(Directory) - if !ok { - t.Fatal("Expected a directory") - } - it := md.Entries() + it := mf.Entries() if !it.Next() || it.Name() != "beep.txt" { t.Fatal("iterator didn't work as expected") From b9735651c8b38e951ed45b9bad42428598319a69 Mon Sep 17 00:00:00 2001 From: "Masih H. Derkani" Date: Thu, 22 Jul 2021 10:02:07 +0100 Subject: [PATCH 61/76] Fix test failure on Windows caused by nil `sys` in mock `FileInfo` Fix test failure caused by `panic` when checking if a file is hidden in Windows. The hidden check uses `FileInfo.Sys()`. In tests the object is a mock and that call returns `nil`. Regardless of mocking, we should check for `nil` since according to docs that function can return `nil`. Run `go mod tidy`. Relates to: - https://github.com/ipfs/go-ipfs-files/pull/34 This commit was moved from ipfs/go-ipfs-files@824686023dce36e338919a121b78b4695ab53684 --- files/is_hidden_windows.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/files/is_hidden_windows.go b/files/is_hidden_windows.go index 6f8bd7870..77ea34a70 100644 --- a/files/is_hidden_windows.go +++ b/files/is_hidden_windows.go @@ -19,7 +19,11 @@ func isHidden(fi os.FileInfo) bool { return true } - wi, ok := fi.Sys().(*windows.Win32FileAttributeData) + sys := fi.Sys() + if sys == nil { + return false + } + wi, ok := sys.(*windows.Win32FileAttributeData) if !ok { return false } From ecef908d30cf03808073e0ff21701a7e1c02c5be Mon Sep 17 00:00:00 2001 From: "Masih H. Derkani" Date: Thu, 22 Jul 2021 10:27:38 +0100 Subject: [PATCH 62/76] Mock `FileInfo.Sys()` to return `nil` explicitly Otherwise, this function call panics on windows. This commit was moved from ipfs/go-ipfs-files@73547d879f88980f233b3f929dbcaadabe233249 --- files/filter_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/files/filter_test.go b/files/filter_test.go index d33b11429..ad2e48cd9 100644 --- a/files/filter_test.go +++ b/files/filter_test.go @@ -16,6 +16,10 @@ func (m *mockFileInfo) Name() string { return m.name } +func (m *mockFileInfo) Sys() interface{} { + return nil +} + var _ os.FileInfo = &mockFileInfo{} func TestFileFilter(t *testing.T) { From 302094f964002d8d8058d566a67bf26175730c9a Mon Sep 17 00:00:00 2001 From: "Masih H. Derkani" Date: Thu, 22 Jul 2021 10:57:54 +0100 Subject: [PATCH 63/76] Normalize path separator in tests based in OS Use OS specific path separators in tests This commit was moved from ipfs/go-ipfs-files@ac10fa80a51edc0f32e9aa0a72de9fed5292a1ce --- files/filewriter_test.go | 16 ++++++++-------- files/serialfile_test.go | 28 ++++++++++++++-------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/files/filewriter_test.go b/files/filewriter_test.go index 5809aba57..4d2150359 100644 --- a/files/filewriter_test.go +++ b/files/filewriter_test.go @@ -24,20 +24,20 @@ func TestWriteTo(t *testing.T) { } defer os.RemoveAll(tmppath) - path := tmppath + "/output" + path := filepath.Join(tmppath, "output") err = WriteTo(sf, path) if err != nil { t.Fatal(err) } expected := map[string]string{ - ".": "", - "1": "Some text!\n", - "2": "beep", - "3": "", - "4": "boop", - "5": "", - "5/a": "foobar", + ".": "", + "1": "Some text!\n", + "2": "beep", + "3": "", + "4": "boop", + "5": "", + filepath.FromSlash("5/a"): "foobar", } err = filepath.Walk(path, func(cpath string, info os.FileInfo, err error) error { if err != nil { diff --git a/files/serialfile_test.go b/files/serialfile_test.go index ee8da3ad3..ae7639691 100644 --- a/files/serialfile_test.go +++ b/files/serialfile_test.go @@ -29,17 +29,17 @@ func testSerialFile(t *testing.T, hidden, withIgnoreRules bool) { defer os.RemoveAll(tmppath) testInputs := map[string]string{ - "1": "Some text!\n", - "2": "beep", - "3": "", - "4": "boop", - "5": "", - "5/a": "foobar", - ".6": "thing", - "7": "", - "7/.foo": "bla", - ".8": "", - ".8/foo": "bla", + "1": "Some text!\n", + "2": "beep", + "3": "", + "4": "boop", + "5": "", + filepath.FromSlash("5/a"): "foobar", + ".6": "thing", + "7": "", + filepath.FromSlash("7/.foo"): "bla", + ".8": "", + filepath.FromSlash(".8/foo"): "bla", } fileFilter, err := NewFilter("", []string{"9", "10"}, hidden) if err != nil { @@ -47,9 +47,9 @@ func testSerialFile(t *testing.T, hidden, withIgnoreRules bool) { } if withIgnoreRules { testInputs["9"] = "" - testInputs["9/b"] = "bebop" + testInputs[filepath.FromSlash("9/b")] = "bebop" testInputs["10"] = "" - testInputs["10/.c"] = "doowop" + testInputs[filepath.FromSlash("10/.c")] = "doowop" } for p, c := range testInputs { @@ -76,7 +76,7 @@ func testSerialFile(t *testing.T, hidden, withIgnoreRules bool) { testInputs: for p := range testInputs { - components := strings.Split(p, "/") + components := strings.Split(p, string(filepath.Separator)) var stat os.FileInfo for i := range components { stat, err = os.Stat(filepath.Join( From f2b9025d0c6974f9e50fe559bbdbd12416ff8b60 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Sun, 5 Sep 2021 21:52:38 +0200 Subject: [PATCH 64/76] fix: round timestamps down by truncating them to seconds Otherwise, we'll write timestamps in the future. fixes https://github.com/ipfs/go-ipfs/issues/8406 This commit was moved from ipfs/go-ipfs-files@3dadb7b27487873743ac6908295ff1959617d34c --- files/tarwriter.go | 4 ++-- files/tarwriter_test.go | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/files/tarwriter.go b/files/tarwriter.go index 6d062726a..4f4ee4e73 100644 --- a/files/tarwriter.go +++ b/files/tarwriter.go @@ -74,7 +74,7 @@ func writeDirHeader(w *tar.Writer, fpath string) error { Name: fpath, Typeflag: tar.TypeDir, Mode: 0777, - ModTime: time.Now(), + ModTime: time.Now().Truncate(time.Second), // TODO: set mode, dates, etc. when added to unixFS }) } @@ -85,7 +85,7 @@ func writeFileHeader(w *tar.Writer, fpath string, size uint64) error { Size: int64(size), Typeflag: tar.TypeReg, Mode: 0644, - ModTime: time.Now(), + ModTime: time.Now().Truncate(time.Second), // TODO: set mode, dates, etc. when added to unixFS }) } diff --git a/files/tarwriter_test.go b/files/tarwriter_test.go index 02686b567..f66d03549 100644 --- a/files/tarwriter_test.go +++ b/files/tarwriter_test.go @@ -4,6 +4,7 @@ import ( "archive/tar" "io" "testing" + "time" ) func TestTarWriter(t *testing.T) { @@ -42,6 +43,10 @@ func TestTarWriter(t *testing.T) { if cur.Size != size { t.Errorf("got wrong size: %d != %d", cur.Size, size) } + now := time.Now() + if cur.ModTime.After(now) { + t.Errorf("wrote timestamp in the future: %s (now) < %s", now, cur.ModTime) + } } if cur, err = tr.Next(); err != nil { From ecf58d57fec6aa0b0676b254e5f7edd211288662 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 21 Sep 2021 10:07:10 +0100 Subject: [PATCH 65/76] fix: manually parse the content disposition to preserve directories fixes https://github.com/ipfs/go-ipfs/issues/8445 This commit was moved from ipfs/go-ipfs-files@55e9e3fbacdfd1f4c99ca3638c03e1c92f23f9ae --- files/multipartfile.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/files/multipartfile.go b/files/multipartfile.go index d4593ad6c..24211cdc0 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -110,7 +110,12 @@ func (w *multipartWalker) nextFile() (Node, error) { // fileName returns a normalized filename from a part. func fileName(part *multipart.Part) string { - filename := part.FileName() + v := part.Header.Get("Content-Disposition") + _, params, err := mime.ParseMediaType(v) + if err != nil { + return "" + } + filename := params["filename"] if escaped, err := url.QueryUnescape(filename); err == nil { filename = escaped } // if there is a unescape error, just treat the name as unescaped From 589fb77387f9bc7bec4172510ea17502633d2ab9 Mon Sep 17 00:00:00 2001 From: web3-bot Date: Wed, 15 Sep 2021 22:20:08 +0000 Subject: [PATCH 66/76] run gofmt -s This commit was moved from ipfs/go-ipfs-files@e6e62a3006aea4ad35a9621d20509ac8a6d66955 --- files/is_hidden.go | 3 ++- files/is_hidden_windows.go | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/files/is_hidden.go b/files/is_hidden.go index 27960ac08..9ab08f7a4 100644 --- a/files/is_hidden.go +++ b/files/is_hidden.go @@ -1,4 +1,5 @@ -//+build !windows +//go:build !windows +// +build !windows package files diff --git a/files/is_hidden_windows.go b/files/is_hidden_windows.go index 77ea34a70..a8b95ca6b 100644 --- a/files/is_hidden_windows.go +++ b/files/is_hidden_windows.go @@ -1,3 +1,4 @@ +//go:build windows // +build windows package files From 221144f0b8cb00a8a1cb7d16ae1efbeb63ae1d38 Mon Sep 17 00:00:00 2001 From: Lucas Molas Date: Fri, 14 Jan 2022 12:55:08 -0300 Subject: [PATCH 67/76] chore(filewriter): cleanup writes (#43) * chore(filewriter): cleanup writes This commit was moved from ipfs/go-ipfs-files@447f558b9ee9d68a8a4bdd8fdf751d4623271520 --- files/filewriter.go | 20 ++++++++++++-- files/filewriter_test.go | 24 +++++++++++++++++ files/filewriter_unix.go | 20 ++++++++++++++ files/filewriter_unix_test.go | 35 ++++++++++++++++++++++++ files/filewriter_windows.go | 46 ++++++++++++++++++++++++++++++++ files/filewriter_windows_test.go | 37 +++++++++++++++++++++++++ 6 files changed, 180 insertions(+), 2 deletions(-) create mode 100644 files/filewriter_unix.go create mode 100644 files/filewriter_unix_test.go create mode 100644 files/filewriter_windows.go create mode 100644 files/filewriter_windows_test.go diff --git a/files/filewriter.go b/files/filewriter.go index c42b3c33e..bf4bcf649 100644 --- a/files/filewriter.go +++ b/files/filewriter.go @@ -1,19 +1,28 @@ package files import ( + "errors" "fmt" "io" "os" "path/filepath" ) +var ErrInvalidDirectoryEntry = errors.New("invalid directory entry name") +var ErrPathExistsOverwrite = errors.New("path already exists and overwriting is not allowed") + // WriteTo writes the given node to the local filesystem at fpath. func WriteTo(nd Node, fpath string) error { + if _, err := os.Lstat(fpath); err == nil { + return ErrPathExistsOverwrite + } else if !os.IsNotExist(err) { + return err + } switch nd := nd.(type) { case *Symlink: return os.Symlink(nd.Target, fpath) case File: - f, err := os.Create(fpath) + f, err := createNewFile(fpath) defer f.Close() if err != nil { return err @@ -31,7 +40,14 @@ func WriteTo(nd Node, fpath string) error { entries := nd.Entries() for entries.Next() { - child := filepath.Join(fpath, entries.Name()) + entryName := entries.Name() + if entryName == "" || + entryName == "." || + entryName == ".." || + !isValidFilename(entryName) { + return ErrInvalidDirectoryEntry + } + child := filepath.Join(fpath, entryName) if err := WriteTo(entries.Node(), child); err != nil { return err } diff --git a/files/filewriter_test.go b/files/filewriter_test.go index 4d2150359..c61e29f63 100644 --- a/files/filewriter_test.go +++ b/files/filewriter_test.go @@ -6,6 +6,8 @@ import ( "os" "path/filepath" "testing" + + "github.com/stretchr/testify/assert" ) func TestWriteTo(t *testing.T) { @@ -75,3 +77,25 @@ func TestWriteTo(t *testing.T) { t.Fatalf("failed to find: %#v", expected) } } + +func TestDontAllowOverwrite(t *testing.T) { + tmppath, err := ioutil.TempDir("", "files-test") + assert.NoError(t, err) + defer os.RemoveAll(tmppath) + + path := filepath.Join(tmppath, "output") + + // Check we can actually write to the output path before trying invalid entries + // and leave an existing entry to test overwrite protection. + assert.NoError(t, WriteTo(NewMapDirectory(map[string]Node{ + "exisiting-entry": NewBytesFile(nil), + }), path)) + + assert.Equal(t, ErrPathExistsOverwrite, WriteTo(NewBytesFile(nil), filepath.Join(path))) + assert.Equal(t, ErrPathExistsOverwrite, WriteTo(NewBytesFile(nil), filepath.Join(path, "exisiting-entry"))) + // The directory in `path` has already been created so this should fail too: + assert.Equal(t, ErrPathExistsOverwrite, WriteTo(NewMapDirectory(map[string]Node{ + "any-name": NewBytesFile(nil), + }), filepath.Join(path))) + os.RemoveAll(path) +} diff --git a/files/filewriter_unix.go b/files/filewriter_unix.go new file mode 100644 index 000000000..1589594a4 --- /dev/null +++ b/files/filewriter_unix.go @@ -0,0 +1,20 @@ +//go:build darwin || linux || netbsd || openbsd +// +build darwin linux netbsd openbsd + +package files + +import ( + "os" + "strings" + "syscall" +) + +var invalidChars = `/` + "\x00" + +func isValidFilename(filename string) bool { + return !strings.ContainsAny(filename, invalidChars) +} + +func createNewFile(path string) (*os.File, error) { + return os.OpenFile(path, os.O_EXCL|os.O_CREATE|os.O_WRONLY|syscall.O_NOFOLLOW, 0666) +} diff --git a/files/filewriter_unix_test.go b/files/filewriter_unix_test.go new file mode 100644 index 000000000..09aeb919f --- /dev/null +++ b/files/filewriter_unix_test.go @@ -0,0 +1,35 @@ +//go:build darwin || linux || netbsd || openbsd +// +build darwin linux netbsd openbsd + +package files + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestWriteToInvalidPaths(t *testing.T) { + tmppath, err := ioutil.TempDir("", "files-test") + assert.NoError(t, err) + defer os.RemoveAll(tmppath) + + path := filepath.Join(tmppath, "output") + + // Check we can actually write to the output path before trying invalid entries. + assert.NoError(t, WriteTo(NewMapDirectory(map[string]Node{ + "valid-entry": NewBytesFile(nil), + }), path)) + os.RemoveAll(path) + + // Now try all invalid entry names + for _, entryName := range []string{"", ".", "..", "/", "", "not/a/base/path"} { + assert.Equal(t, ErrInvalidDirectoryEntry, WriteTo(NewMapDirectory(map[string]Node{ + entryName: NewBytesFile(nil), + }), filepath.Join(path))) + os.RemoveAll(path) + } +} diff --git a/files/filewriter_windows.go b/files/filewriter_windows.go new file mode 100644 index 000000000..4392e0e2c --- /dev/null +++ b/files/filewriter_windows.go @@ -0,0 +1,46 @@ +//go:build windows +// +build windows + +package files + +import ( + "os" + "strings" +) + +var invalidChars = `<>:"/\|?*` + "\x00" + +var reservedNames = map[string]struct{}{ + "CON": {}, + "PRN": {}, + "AUX": {}, + "NUL": {}, + "COM1": {}, + "COM2": {}, + "COM3": {}, + "COM4": {}, + "COM5": {}, + "COM6": {}, + "COM7": {}, + "COM8": {}, + "COM9": {}, + "LPT1": {}, + "LPT2": {}, + "LPT3": {}, + "LPT4": {}, + "LPT5": {}, + "LPT6": {}, + "LPT7": {}, + "LPT8": {}, + "LPT9": {}, +} + +func isValidFilename(filename string) bool { + _, isReservedName := reservedNames[filename] + return !strings.ContainsAny(filename, invalidChars) && + !isReservedName +} + +func createNewFile(path string) (*os.File, error) { + return os.OpenFile(path, os.O_EXCL|os.O_CREATE|os.O_WRONLY, 0666) +} diff --git a/files/filewriter_windows_test.go b/files/filewriter_windows_test.go new file mode 100644 index 000000000..4193ac9ea --- /dev/null +++ b/files/filewriter_windows_test.go @@ -0,0 +1,37 @@ +//go:build windows +// +build windows + +package files + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestWriteToInvalidPaths(t *testing.T) { + tmppath, err := ioutil.TempDir("", "files-test") + assert.NoError(t, err) + defer os.RemoveAll(tmppath) + + path := filepath.Join(tmppath, "output") + + // Check we can actually write to the output path before trying invalid entries. + assert.NoError(t, WriteTo(NewMapDirectory(map[string]Node{ + "valid-entry": NewBytesFile(nil), + }), path)) + os.RemoveAll(path) + + // Now try all invalid entry names + for _, entryName := range []string{"", ".", "..", "/", "", "not/a/base/path", + "<", ">", ":", "\"", "\\", "|", "?", "*", "\x00", + "CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9"} { + assert.Equal(t, ErrInvalidDirectoryEntry, WriteTo(NewMapDirectory(map[string]Node{ + entryName: NewBytesFile(nil), + }), filepath.Join(path))) + os.RemoveAll(path) + } +} From 45fc9415e26db74742b8aae48f638803a80bebc3 Mon Sep 17 00:00:00 2001 From: Abdul Rauf Date: Fri, 14 Jan 2022 20:56:10 +0500 Subject: [PATCH 68/76] docs: fix community CONTRIBUTING.md link (#45) This commit was moved from ipfs/go-ipfs-files@5540d9265e382e4db39884294db35ba73f299393 --- files/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/README.md b/files/README.md index e0205d78a..f8be00c99 100644 --- a/files/README.md +++ b/files/README.md @@ -23,7 +23,7 @@ This repository falls under the IPFS [Code of Conduct](https://github.com/ipfs/c ### Want to hack on IPFS? -[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/contributing.md) +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) ## License From 38ce1dfd524370bba2e075f16d08e82406f12a01 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 18 Jan 2022 01:24:57 -0500 Subject: [PATCH 69/76] fix: add freebsd build option for filewriter flags This commit was moved from ipfs/go-ipfs-files@90b5617c775dfc96b950baec7d1c17e925c0afb6 --- files/filewriter_unix.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/files/filewriter_unix.go b/files/filewriter_unix.go index 1589594a4..c5199222c 100644 --- a/files/filewriter_unix.go +++ b/files/filewriter_unix.go @@ -1,5 +1,5 @@ -//go:build darwin || linux || netbsd || openbsd -// +build darwin linux netbsd openbsd +//go:build darwin || linux || netbsd || openbsd || freebsd +// +build darwin linux netbsd openbsd freebsd package files From 4ebe0905bd6a74e9d25c40537f2e60cbbc9aff98 Mon Sep 17 00:00:00 2001 From: Wayback Archiver <66856220+waybackarchiver@users.noreply.github.com> Date: Sun, 23 Jan 2022 21:20:25 +0800 Subject: [PATCH 70/76] fix: add dragonfly build option for filewriter flags This commit was moved from ipfs/go-ipfs-files@ad1c38162d9e3fa607b2dffba237b7773ed5eb71 --- files/filewriter_unix.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/files/filewriter_unix.go b/files/filewriter_unix.go index c5199222c..8252e21fb 100644 --- a/files/filewriter_unix.go +++ b/files/filewriter_unix.go @@ -1,5 +1,5 @@ -//go:build darwin || linux || netbsd || openbsd || freebsd -// +build darwin linux netbsd openbsd freebsd +//go:build darwin || linux || netbsd || openbsd || freebsd || dragonfly +// +build darwin linux netbsd openbsd freebsd dragonfly package files From 7418db2679442a910af275b2e899589d96e6a9fd Mon Sep 17 00:00:00 2001 From: Lucas Molas Date: Tue, 2 Aug 2022 13:01:32 -0300 Subject: [PATCH 71/76] chore(Directory): add DirIterator API restriction: iterate only once This commit was moved from ipfs/go-ipfs-files@0889edbf54bc298ba39dedc61ee5377bf5bfe39f --- files/file.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/files/file.go b/files/file.go index 4d7ef1132..7ac1fc98a 100644 --- a/files/file.go +++ b/files/file.go @@ -63,7 +63,9 @@ type DirIterator interface { type Directory interface { Node - // Entries returns a stateful iterator over directory entries. + // Entries returns a stateful iterator over directory entries. The iterator + // may consume the Directory state so it must be called only once (this + // applies specifically to the multipartIterator). // // Example usage: // From 494035465d40286f0a1b4953699cd7a34b02f73a Mon Sep 17 00:00:00 2001 From: web3-bot Date: Thu, 1 Sep 2022 09:14:34 +0000 Subject: [PATCH 72/76] bump go.mod to Go 1.18 and run go fix This commit was moved from ipfs/go-ipfs-files@e603bdf2737b12dbc7f8813c7818190cc5f0297e --- files/filewriter_unix.go | 1 - files/filewriter_unix_test.go | 1 - files/filewriter_windows.go | 1 - files/filewriter_windows_test.go | 1 - files/is_hidden.go | 1 - files/is_hidden_windows.go | 1 - 6 files changed, 6 deletions(-) diff --git a/files/filewriter_unix.go b/files/filewriter_unix.go index 8252e21fb..98d040018 100644 --- a/files/filewriter_unix.go +++ b/files/filewriter_unix.go @@ -1,5 +1,4 @@ //go:build darwin || linux || netbsd || openbsd || freebsd || dragonfly -// +build darwin linux netbsd openbsd freebsd dragonfly package files diff --git a/files/filewriter_unix_test.go b/files/filewriter_unix_test.go index 09aeb919f..14c1967dd 100644 --- a/files/filewriter_unix_test.go +++ b/files/filewriter_unix_test.go @@ -1,5 +1,4 @@ //go:build darwin || linux || netbsd || openbsd -// +build darwin linux netbsd openbsd package files diff --git a/files/filewriter_windows.go b/files/filewriter_windows.go index 4392e0e2c..a5d626199 100644 --- a/files/filewriter_windows.go +++ b/files/filewriter_windows.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package files diff --git a/files/filewriter_windows_test.go b/files/filewriter_windows_test.go index 4193ac9ea..586a9dbc8 100644 --- a/files/filewriter_windows_test.go +++ b/files/filewriter_windows_test.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package files diff --git a/files/is_hidden.go b/files/is_hidden.go index 9ab08f7a4..9842ca232 100644 --- a/files/is_hidden.go +++ b/files/is_hidden.go @@ -1,5 +1,4 @@ //go:build !windows -// +build !windows package files diff --git a/files/is_hidden_windows.go b/files/is_hidden_windows.go index a8b95ca6b..9a0703863 100644 --- a/files/is_hidden_windows.go +++ b/files/is_hidden_windows.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package files From 8520c194769caa446f7c484f47d340e811facd58 Mon Sep 17 00:00:00 2001 From: web3-bot Date: Thu, 1 Sep 2022 09:14:40 +0000 Subject: [PATCH 73/76] stop using the deprecated io/ioutil package This commit was moved from ipfs/go-ipfs-files@c2dbc997b17a8d211c9f51a18755827efdac0836 --- files/filewriter_test.go | 7 +++---- files/filewriter_unix_test.go | 3 +-- files/filewriter_windows_test.go | 3 +-- files/filter_test.go | 5 ++--- files/helpers_test.go | 4 ++-- files/multipartfile.go | 3 +-- files/readerfile.go | 3 +-- files/serialfile.go | 3 +-- files/serialfile_test.go | 8 ++++---- files/webfile_test.go | 10 +++++----- 10 files changed, 21 insertions(+), 28 deletions(-) diff --git a/files/filewriter_test.go b/files/filewriter_test.go index c61e29f63..00a0b1ce2 100644 --- a/files/filewriter_test.go +++ b/files/filewriter_test.go @@ -2,7 +2,6 @@ package files import ( "fmt" - "io/ioutil" "os" "path/filepath" "testing" @@ -20,7 +19,7 @@ func TestWriteTo(t *testing.T) { "a": NewBytesFile([]byte("foobar")), }), }) - tmppath, err := ioutil.TempDir("", "files-test") + tmppath, err := os.MkdirTemp("", "files-test") if err != nil { t.Fatal(err) } @@ -60,7 +59,7 @@ func TestWriteTo(t *testing.T) { return fmt.Errorf("expected a directory at %q", rpath) } } else { - actual, err := ioutil.ReadFile(cpath) + actual, err := os.ReadFile(cpath) if err != nil { return err } @@ -79,7 +78,7 @@ func TestWriteTo(t *testing.T) { } func TestDontAllowOverwrite(t *testing.T) { - tmppath, err := ioutil.TempDir("", "files-test") + tmppath, err := os.MkdirTemp("", "files-test") assert.NoError(t, err) defer os.RemoveAll(tmppath) diff --git a/files/filewriter_unix_test.go b/files/filewriter_unix_test.go index 14c1967dd..ffc33ce51 100644 --- a/files/filewriter_unix_test.go +++ b/files/filewriter_unix_test.go @@ -3,7 +3,6 @@ package files import ( - "io/ioutil" "os" "path/filepath" "testing" @@ -12,7 +11,7 @@ import ( ) func TestWriteToInvalidPaths(t *testing.T) { - tmppath, err := ioutil.TempDir("", "files-test") + tmppath, err := os.MkdirTemp("", "files-test") assert.NoError(t, err) defer os.RemoveAll(tmppath) diff --git a/files/filewriter_windows_test.go b/files/filewriter_windows_test.go index 586a9dbc8..ca0222ba3 100644 --- a/files/filewriter_windows_test.go +++ b/files/filewriter_windows_test.go @@ -3,7 +3,6 @@ package files import ( - "io/ioutil" "os" "path/filepath" "testing" @@ -12,7 +11,7 @@ import ( ) func TestWriteToInvalidPaths(t *testing.T) { - tmppath, err := ioutil.TempDir("", "files-test") + tmppath, err := os.MkdirTemp("", "files-test") assert.NoError(t, err) defer os.RemoveAll(tmppath) diff --git a/files/filter_test.go b/files/filter_test.go index ad2e48cd9..8ce25ee3b 100644 --- a/files/filter_test.go +++ b/files/filter_test.go @@ -1,7 +1,6 @@ package files import ( - "io/ioutil" "os" "path/filepath" "testing" @@ -35,13 +34,13 @@ func TestFileFilter(t *testing.T) { if err == nil { t.Errorf("creating a filter without an invalid ignore file path should have failed") } - tmppath, err := ioutil.TempDir("", "filter-test") + tmppath, err := os.MkdirTemp("", "filter-test") if err != nil { t.Fatal(err) } ignoreFilePath := filepath.Join(tmppath, "ignoreFile") ignoreFileContents := []byte("a.txt") - if err := ioutil.WriteFile(ignoreFilePath, ignoreFileContents, 0666); err != nil { + if err := os.WriteFile(ignoreFilePath, ignoreFileContents, 0666); err != nil { t.Fatal(err) } filterWithIgnoreFile, err := NewFilter(ignoreFilePath, nil, false) diff --git a/files/helpers_test.go b/files/helpers_test.go index ec420bdc2..0180b8f27 100644 --- a/files/helpers_test.go +++ b/files/helpers_test.go @@ -1,7 +1,7 @@ package files import ( - "io/ioutil" + "io" "testing" ) @@ -56,7 +56,7 @@ func CheckDir(t *testing.T, dir Directory, expected []Event) { if !ok { t.Fatalf("[%d] expected file to be a normal file: %T", i, it.Node()) } - out, err := ioutil.ReadAll(mf) + out, err := io.ReadAll(mf) if err != nil { t.Errorf("[%d] failed to read file", i) continue diff --git a/files/multipartfile.go b/files/multipartfile.go index 24211cdc0..27653982c 100644 --- a/files/multipartfile.go +++ b/files/multipartfile.go @@ -2,7 +2,6 @@ package files import ( "io" - "io/ioutil" "mime" "mime/multipart" "net/url" @@ -94,7 +93,7 @@ func (w *multipartWalker) nextFile() (Node, error) { walker: w, }, nil case applicationSymlink: - out, err := ioutil.ReadAll(part) + out, err := io.ReadAll(part) if err != nil { return nil, err } diff --git a/files/readerfile.go b/files/readerfile.go index f98fec481..a03dae23f 100644 --- a/files/readerfile.go +++ b/files/readerfile.go @@ -3,7 +3,6 @@ package files import ( "bytes" "io" - "io/ioutil" "os" "path/filepath" ) @@ -29,7 +28,7 @@ func NewReaderFile(reader io.Reader) File { func NewReaderStatFile(reader io.Reader, stat os.FileInfo) File { rc, ok := reader.(io.ReadCloser) if !ok { - rc = ioutil.NopCloser(reader) + rc = io.NopCloser(reader) } return &ReaderFile{"", rc, stat, -1} diff --git a/files/serialfile.go b/files/serialfile.go index 86af30680..77871d352 100644 --- a/files/serialfile.go +++ b/files/serialfile.go @@ -3,7 +3,6 @@ package files import ( "errors" "fmt" - "io/ioutil" "os" "path/filepath" ) @@ -52,7 +51,7 @@ func NewSerialFileWithFilter(path string, filter *Filter, stat os.FileInfo) (Nod case mode.IsDir(): // for directories, stat all of the contents first, so we know what files to // open when Entries() is called - contents, err := ioutil.ReadDir(path) + contents, err := os.ReadDir(path) if err != nil { return nil, err } diff --git a/files/serialfile_test.go b/files/serialfile_test.go index ae7639691..80c252a7e 100644 --- a/files/serialfile_test.go +++ b/files/serialfile_test.go @@ -2,7 +2,7 @@ package files import ( "fmt" - "io/ioutil" + "io" "os" "path/filepath" "sort" @@ -22,7 +22,7 @@ func TestSerialFile(t *testing.T) { } func testSerialFile(t *testing.T, hidden, withIgnoreRules bool) { - tmppath, err := ioutil.TempDir("", "files-test") + tmppath, err := os.MkdirTemp("", "files-test") if err != nil { t.Fatal(err) } @@ -67,7 +67,7 @@ func testSerialFile(t *testing.T, hidden, withIgnoreRules bool) { if c == "" { continue } - if err := ioutil.WriteFile(path, []byte(c), 0666); err != nil { + if err := os.WriteFile(path, []byte(c), 0666); err != nil { t.Fatal(err) } } @@ -158,7 +158,7 @@ testInputs: return fmt.Errorf("expected a directory at %q", path) } case File: - actual, err := ioutil.ReadAll(nd) + actual, err := io.ReadAll(nd) if err != nil { return err } diff --git a/files/webfile_test.go b/files/webfile_test.go index 57a67fe87..94cddb5d2 100644 --- a/files/webfile_test.go +++ b/files/webfile_test.go @@ -2,7 +2,7 @@ package files import ( "fmt" - "io/ioutil" + "io" "net/http" "net/http/httptest" "net/url" @@ -21,7 +21,7 @@ func TestWebFile(t *testing.T) { t.Fatal(err) } wf := NewWebFile(u) - body, err := ioutil.ReadAll(wf) + body, err := io.ReadAll(wf) if err != nil { t.Fatal(err) } @@ -41,7 +41,7 @@ func TestWebFile_notFound(t *testing.T) { t.Fatal(err) } wf := NewWebFile(u) - _, err = ioutil.ReadAll(wf) + _, err = io.ReadAll(wf) if err == nil { t.Fatal("expected error") } @@ -68,7 +68,7 @@ func TestWebFileSize(t *testing.T) { t.Errorf("expected size to be %d, got %d", len(body), size) } - actual, err := ioutil.ReadAll(wf1) + actual, err := io.ReadAll(wf1) if err != nil { t.Fatal(err) } @@ -81,7 +81,7 @@ func TestWebFileSize(t *testing.T) { // Read size after reading file. wf2 := NewWebFile(u) - actual, err = ioutil.ReadAll(wf2) + actual, err = io.ReadAll(wf2) if err != nil { t.Fatal(err) } From e6e3c80d0051f0b3f8478b7a878f2b01e47281d7 Mon Sep 17 00:00:00 2001 From: galargh Date: Thu, 1 Sep 2022 11:33:55 +0200 Subject: [PATCH 74/76] fix: type of contents in serialfile This commit was moved from ipfs/go-ipfs-files@263276c1f170d34e51279193ee44eb3f3d00d40b --- files/serialfile.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/files/serialfile.go b/files/serialfile.go index 77871d352..176038cde 100644 --- a/files/serialfile.go +++ b/files/serialfile.go @@ -3,6 +3,7 @@ package files import ( "errors" "fmt" + "io/fs" "os" "path/filepath" ) @@ -51,10 +52,18 @@ func NewSerialFileWithFilter(path string, filter *Filter, stat os.FileInfo) (Nod case mode.IsDir(): // for directories, stat all of the contents first, so we know what files to // open when Entries() is called - contents, err := os.ReadDir(path) + entries, err := os.ReadDir(path) if err != nil { return nil, err } + contents := make([]fs.FileInfo, 0, len(entries)) + for _, entry := range entries { + content, err := entry.Info() + if err != nil { + return nil, err + } + contents = append(contents, content) + } return &serialFile{path, contents, stat, filter}, nil case mode&os.ModeSymlink != 0: target, err := os.Readlink(path) From 83a75d27afc2e308383efc356f75792af1770b92 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 9 Nov 2022 16:53:07 +0100 Subject: [PATCH 75/76] fix: error when TAR has files outside of root (#56) See "Security" section of IPIP-288 (https://github.com/ipfs/specs/pull/288) This commit was moved from ipfs/go-ipfs-files@e8cf9a3f9b1bdd35c4568dc46d55df7e88f3487b --- files/tarwriter.go | 39 ++++++++++++++++++++++++- files/tarwriter_test.go | 64 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 1 deletion(-) diff --git a/files/tarwriter.go b/files/tarwriter.go index 4f4ee4e73..cecbcae42 100644 --- a/files/tarwriter.go +++ b/files/tarwriter.go @@ -2,14 +2,22 @@ package files import ( "archive/tar" + "errors" "fmt" "io" "path" + "strings" "time" ) +var ( + ErrUnixFSPathOutsideRoot = errors.New("relative UnixFS paths outside the root are now allowed, use CAR instead") +) + type TarWriter struct { - TarW *tar.Writer + TarW *tar.Writer + baseDirSet bool + baseDir string } // NewTarWriter wraps given io.Writer into a new tar writer @@ -50,8 +58,37 @@ func (w *TarWriter) writeFile(f File, fpath string) error { return nil } +func validateTarFilePath(baseDir, fpath string) bool { + // Ensure the filepath has no ".", "..", etc within the known root directory. + fpath = path.Clean(fpath) + + // If we have a non-empty baseDir, check if the filepath starts with baseDir. + // If not, we can exclude it immediately. For 'ipfs get' and for the gateway, + // the baseDir would be '{cid}.tar'. + if baseDir != "" && !strings.HasPrefix(path.Clean(fpath), baseDir) { + return false + } + + // Otherwise, check if the path starts with '..' which would make it fall + // outside the root path. This works since the path has already been cleaned. + if strings.HasPrefix(fpath, "..") { + return false + } + + return true +} + // WriteNode adds a node to the archive. func (w *TarWriter) WriteFile(nd Node, fpath string) error { + if !w.baseDirSet { + w.baseDirSet = true // Use a variable for this as baseDir may be an empty string. + w.baseDir = fpath + } + + if !validateTarFilePath(w.baseDir, fpath) { + return ErrUnixFSPathOutsideRoot + } + switch nd := nd.(type) { case *Symlink: return writeSymlinkHeader(w.TarW, nd.Target, fpath) diff --git a/files/tarwriter_test.go b/files/tarwriter_test.go index f66d03549..0e1488e7f 100644 --- a/files/tarwriter_test.go +++ b/files/tarwriter_test.go @@ -2,6 +2,7 @@ package files import ( "archive/tar" + "errors" "io" "testing" "time" @@ -83,3 +84,66 @@ func TestTarWriter(t *testing.T) { t.Fatal(err) } } + +func TestTarWriterRelativePathInsideRoot(t *testing.T) { + tf := NewMapDirectory(map[string]Node{ + "file.txt": NewBytesFile([]byte(text)), + "boop": NewMapDirectory(map[string]Node{ + "../a.txt": NewBytesFile([]byte("bleep")), + "b.txt": NewBytesFile([]byte("bloop")), + }), + "beep.txt": NewBytesFile([]byte("beep")), + }) + + tw, err := NewTarWriter(io.Discard) + if err != nil { + t.Fatal(err) + } + + defer tw.Close() + if err := tw.WriteFile(tf, ""); err != nil { + t.Error(err) + } +} + +func TestTarWriterFailsFileOutsideRoot(t *testing.T) { + tf := NewMapDirectory(map[string]Node{ + "file.txt": NewBytesFile([]byte(text)), + "boop": NewMapDirectory(map[string]Node{ + "../../a.txt": NewBytesFile([]byte("bleep")), + "b.txt": NewBytesFile([]byte("bloop")), + }), + "beep.txt": NewBytesFile([]byte("beep")), + }) + + tw, err := NewTarWriter(io.Discard) + if err != nil { + t.Fatal(err) + } + + defer tw.Close() + if err := tw.WriteFile(tf, ""); !errors.Is(err, ErrUnixFSPathOutsideRoot) { + t.Errorf("unexpected error, wanted: %v; got: %v", ErrUnixFSPathOutsideRoot, err) + } +} + +func TestTarWriterFailsFileOutsideRootWithBaseDir(t *testing.T) { + tf := NewMapDirectory(map[string]Node{ + "../file.txt": NewBytesFile([]byte(text)), + "boop": NewMapDirectory(map[string]Node{ + "a.txt": NewBytesFile([]byte("bleep")), + "b.txt": NewBytesFile([]byte("bloop")), + }), + "beep.txt": NewBytesFile([]byte("beep")), + }) + + tw, err := NewTarWriter(io.Discard) + if err != nil { + t.Fatal(err) + } + + defer tw.Close() + if err := tw.WriteFile(tf, "test.tar"); !errors.Is(err, ErrUnixFSPathOutsideRoot) { + t.Errorf("unexpected error, wanted: %v; got: %v", ErrUnixFSPathOutsideRoot, err) + } +} From ad29d57a4248fc37d5c797a35f569159922228eb Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 11 Jan 2023 13:46:59 +0100 Subject: [PATCH 76/76] go mod tidy --- go.mod | 3 ++- go.sum | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a04266b34..7f40a640f 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.19 require ( github.com/benbjohnson/clock v1.3.0 + github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 github.com/gorilla/mux v1.8.0 github.com/ipfs/go-cid v0.3.2 github.com/ipfs/go-ipns v0.3.0 @@ -16,6 +17,7 @@ require ( github.com/samber/lo v1.36.0 github.com/stretchr/testify v1.8.1 go.opencensus.io v0.23.0 + golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab ) require ( @@ -46,7 +48,6 @@ require ( go.uber.org/zap v1.23.0 // indirect golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b // indirect - golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect ) diff --git a/go.sum b/go.sum index d21fda2df..ab57d0a12 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZx github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= +github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=