Merge pull request #20 from ipfs/migrate-files

files/README.md

@@ -0,0 +1,31 @@
+# go-ipfs-files
+
+[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io)
+[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/)
+[![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs)
+[![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme)
+
+> File interfaces and utils used in IPFS
+
+## Lead Maintainer
+
+[Steven Allen](https://github.com/Stebalien)
+
+## Documentation
+
+https://godoc.org/github.com/ipfs/go-ipfs-files
+
+## Contribute
+
+Feel free to join in. All welcome. Open an [issue](https://github.com/ipfs/go-ipfs-files/issues)!
+
+This repository falls under the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md).
+
+### Want to hack on IPFS?
+
+[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md)
+
+## License
+
+MIT
+

files/file.go

@@ -0,0 +1,96 @@
+package files
+
+import (
+	"errors"
+	"io"
+	"os"
+)
+
+var (
+	ErrNotDirectory = errors.New("file isn't a directory")
+	ErrNotReader    = errors.New("file isn't a regular file")
+
+	ErrNotSupported = errors.New("operation not supported")
+)
+
+// Node is a common interface for files, directories and other special files
+type Node interface {
+	io.Closer
+
+	// Size returns size of this file (if this file is a directory, total size of
+	// all files stored in the tree should be returned). Some implementations may
+	// choose not to implement this
+	Size() (int64, error)
+}
+
+// Node represents a regular Unix file
+type File interface {
+	Node
+
+	io.Reader
+	io.Seeker
+}
+
+// DirEntry exposes information about a directory entry
+type DirEntry interface {
+	// Name returns base name of this entry, which is the base name of referenced
+	// file
+	Name() string
+
+	// Node returns the file referenced by this DirEntry
+	Node() Node
+}
+
+// DirIterator is a iterator over directory entries.
+// See Directory.Entries for more
+type DirIterator interface {
+	// DirEntry holds information about current directory entry.
+	// Note that after creating new iterator you MUST call Next() at least once
+	// before accessing these methods. Calling these methods without prior calls
+	// to Next() and after Next() returned false may result in undefined behavior
+	DirEntry
+
+	// Next advances iterator to the next file.
+	Next() bool
+
+	// Err may return an error after previous call to Next() returned `false`.
+	// If previous call to Next() returned `true`, Err() is guaranteed to
+	// return nil
+	Err() error
+}
+
+// Directory is a special file which can link to any number of files.
+type Directory interface {
+	Node
+
+	// Entries returns a stateful iterator over directory entries. The iterator
+	// may consume the Directory state so it must be called only once (this
+	// applies specifically to the multipartIterator).
+	//
+	// Example usage:
+	//
+	// it := dir.Entries()
+	// for it.Next() {
+	//   name := it.Name()
+	//   file := it.Node()
+	//   [...]
+	// }
+	// if it.Err() != nil {
+	//   return err
+	// }
+	//
+	// Note that you can't store the result of it.Node() and use it after
+	// advancing the iterator
+	Entries() DirIterator
+}
+
+// FileInfo exposes information on files in local filesystem
+type FileInfo interface {
+	Node
+
+	// AbsPath returns full real file path.
+	AbsPath() string
+
+	// Stat returns os.Stat of this file, may be nil for some files
+	Stat() os.FileInfo
+}

files/file_test.go

@@ -0,0 +1,142 @@
+package files
+
+import (
+	"io"
+	"mime/multipart"
+	"strings"
+	"testing"
+)
+
+func TestSliceFiles(t *testing.T) {
+	sf := NewMapDirectory(map[string]Node{
+		"1": NewBytesFile([]byte("Some text!\n")),
+		"2": NewBytesFile([]byte("beep")),
+		"3": NewBytesFile([]byte("boop")),
+	})
+
+	CheckDir(t, sf, []Event{
+		{
+			kind:  TFile,
+			name:  "1",
+			value: "Some text!\n",
+		},
+		{
+			kind:  TFile,
+			name:  "2",
+			value: "beep",
+		},
+		{
+			kind:  TFile,
+			name:  "3",
+			value: "boop",
+		},
+	})
+}
+
+func TestReaderFiles(t *testing.T) {
+	message := "beep boop"
+	rf := NewBytesFile([]byte(message))
+	buf := make([]byte, len(message))
+
+	if n, err := rf.Read(buf); n == 0 || err != nil {
+		t.Fatal("Expected to be able to read")
+	}
+	if err := rf.Close(); err != nil {
+		t.Fatal("Should be able to close")
+	}
+	if n, err := rf.Read(buf); n != 0 || err != io.EOF {
+		t.Fatal("Expected EOF when reading after close")
+	}
+}
+func TestMultipartFiles(t *testing.T) {
+	data := `
+--Boundary!
+Content-Type: text/plain
+Content-Disposition: file; filename="name"
+Some-Header: beep
+
+beep
+--Boundary!
+Content-Type: application/x-directory
+Content-Disposition: file; filename="dir"
+
+--Boundary!
+Content-Type: text/plain
+Content-Disposition: file; filename="dir/nested"
+
+some content
+--Boundary!
+Content-Type: application/symlink
+Content-Disposition: file; filename="dir/simlynk"
+
+anotherfile
+--Boundary!
+Content-Type: text/plain
+Content-Disposition: file; filename="implicit1/implicit2/deep_implicit"
+
+implicit file1
+--Boundary!
+Content-Type: text/plain
+Content-Disposition: file; filename="implicit1/shallow_implicit"
+
+implicit file2
+--Boundary!--
+
+`
+
+	reader := strings.NewReader(data)
+	mpReader := multipart.NewReader(reader, "Boundary!")
+	dir, err := NewFileFromPartReader(mpReader, multipartFormdataType)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	CheckDir(t, dir, []Event{
+		{
+			kind:  TFile,
+			name:  "name",
+			value: "beep",
+		},
+		{
+			kind: TDirStart,
+			name: "dir",
+		},
+		{
+			kind:  TFile,
+			name:  "nested",
+			value: "some content",
+		},
+		{
+			kind:  TSymlink,
+			name:  "simlynk",
+			value: "anotherfile",
+		},
+		{
+			kind: TDirEnd,
+		},
+		{
+			kind: TDirStart,
+			name: "implicit1",
+		},
+		{
+			kind: TDirStart,
+			name: "implicit2",
+		},
+		{
+			kind:  TFile,
+			name:  "deep_implicit",
+			value: "implicit file1",
+		},
+		{
+			kind: TDirEnd,
+		},
+		{
+			kind:  TFile,
+			name:  "shallow_implicit",
+			value: "implicit file2",
+		},
+		{
+			kind: TDirEnd,
+		},
+	})
+}

files/filewriter.go

@@ -0,0 +1,59 @@
+package files
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+var ErrInvalidDirectoryEntry = errors.New("invalid directory entry name")
+var ErrPathExistsOverwrite = errors.New("path already exists and overwriting is not allowed")
+
+// WriteTo writes the given node to the local filesystem at fpath.
+func WriteTo(nd Node, fpath string) error {
+	if _, err := os.Lstat(fpath); err == nil {
+		return ErrPathExistsOverwrite
+	} else if !os.IsNotExist(err) {
+		return err
+	}
+	switch nd := nd.(type) {
+	case *Symlink:
+		return os.Symlink(nd.Target, fpath)
+	case File:
+		f, err := createNewFile(fpath)
+		defer f.Close()
+		if err != nil {
+			return err
+		}
+		_, err = io.Copy(f, nd)
+		if err != nil {
+			return err
+		}
+		return nil
+	case Directory:
+		err := os.Mkdir(fpath, 0777)
+		if err != nil {
+			return err
+		}
+
+		entries := nd.Entries()
+		for entries.Next() {
+			entryName := entries.Name()
+			if entryName == "" ||
+				entryName == "." ||
+				entryName == ".." ||
+				!isValidFilename(entryName) {
+				return ErrInvalidDirectoryEntry
+			}
+			child := filepath.Join(fpath, entryName)
+			if err := WriteTo(entries.Node(), child); err != nil {
+				return err
+			}
+		}
+		return entries.Err()
+	default:
+		return fmt.Errorf("file type %T at %q is not supported", nd, fpath)
+	}
+}