Fix/separate migration bins

cmd/ipfs/daemon.go

@@ -26,7 +26,7 @@ import (
 	libp2p "github.com/ipfs/go-ipfs/core/node/libp2p"
 	nodeMount "github.com/ipfs/go-ipfs/fuse/node"
 	fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
-	migrate "github.com/ipfs/go-ipfs/repo/fsrepo/migrations"
+	"github.com/ipfs/go-ipfs/repo/fsrepo/migrations"
 	sockets "github.com/libp2p/go-socket-activation"
 
 	cmds "github.com/ipfs/go-ipfs-cmds"
@@ -288,7 +288,7 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
 			return fmt.Errorf("fs-repo requires migration")
 		}
 
-		err = migrate.RunMigration(fsrepo.RepoVersion)
+		err = migrations.RunMigration(cctx.Context(), fsrepo.RepoVersion, "")
 		if err != nil {
 			fmt.Println("The migrations of fs-repo failed:")
 			fmt.Printf("  %s\n", err)

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/bren2010/proquint v0.0.0-20160323162903-38337c27106d
 	github.com/cheggaaa/pb v1.0.29
+	github.com/coreos/go-semver v0.3.0
 	github.com/coreos/go-systemd/v22 v22.1.0
 	github.com/dustin/go-humanize v1.0.0
 	github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302
@@ -29,6 +30,7 @@ require (
 	github.com/ipfs/go-filestore v0.0.3
 	github.com/ipfs/go-fs-lock v0.0.6
 	github.com/ipfs/go-graphsync v0.6.0
+	github.com/ipfs/go-ipfs-api v0.2.0
 	github.com/ipfs/go-ipfs-blockstore v0.1.4
 	github.com/ipfs/go-ipfs-chunker v0.0.5
 	github.com/ipfs/go-ipfs-cmds v0.6.0

go.sum

@@ -87,6 +87,8 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
+github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764=
+github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U=
 github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo=
 github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
@@ -320,6 +322,8 @@ github.com/ipfs/go-fs-lock v0.0.6 h1:sn3TWwNVQqSeNjlWy6zQ1uUGAZrV3hPOyEA6y1/N2a0
 github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28L7zESmM=
 github.com/ipfs/go-graphsync v0.6.0 h1:x6UvDUGA7wjaKNqx5Vbo7FGT8aJ5ryYA0dMQ5jN3dF0=
 github.com/ipfs/go-graphsync v0.6.0/go.mod h1:e2ZxnClqBBYAtd901g9vXMJzS47labjAtOzsWtOzKNk=
+github.com/ipfs/go-ipfs-api v0.2.0 h1:BXRctUU8YOUOQT/jW1s56d9wLa85ntOqK6bptvCKb8c=
+github.com/ipfs/go-ipfs-api v0.2.0/go.mod h1:zCTyTl+BuyvUqoSmVb8vjezCJLVTW7G/HBZbCXpTgeM=
 github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08=
 github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw=
 github.com/ipfs/go-ipfs-blockstore v0.1.4 h1:2SGI6U1B44aODevza8Rde3+dY30Pb+lbcObe1LETxOQ=
@@ -1029,6 +1033,7 @@ github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9 h1:Y1/FEOpaCpD2
 github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4=
 github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 h1:E9S12nwJwEOXe2d6gT6qxdvqMnNq+VnSsKPgm2ZZNds=
 github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI=
+github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c/go.mod h1:xxcJeBb7SIUl/Wzkz1eVKJE/CB34YNrqX2TQI6jY9zs=
 github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b h1:wA3QeTsaAXybLL2kb2cKhCAQTHgYTMwuI8lBlJSv5V8=
 github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b/go.mod h1:xT1Y5p2JR2PfSZihE0s4mjdJaRGp1waCTf5JzhQLBck=
 github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow=

repo/fsrepo/fsrepo.go

@@ -14,7 +14,7 @@ import (
 	keystore "github.com/ipfs/go-ipfs/keystore"
 	repo "github.com/ipfs/go-ipfs/repo"
 	"github.com/ipfs/go-ipfs/repo/common"
-	mfsr "github.com/ipfs/go-ipfs/repo/fsrepo/migrations"
+	"github.com/ipfs/go-ipfs/repo/fsrepo/migrations"
 	dir "github.com/ipfs/go-ipfs/thirdparty/dir"
 
 	ds "github.com/ipfs/go-datastore"
@@ -142,7 +142,7 @@ func open(repoPath string) (repo.Repo, error) {
 	}()
 
 	// Check version, and error out if not matching
-	ver, err := mfsr.RepoPath(r.path).Version()
+	ver, err := migrations.RepoVersion(r.path)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil, ErrNoVersion
@@ -291,7 +291,7 @@ func Init(repoPath string, conf *config.Config) error {
 		return err
 	}
 
-	if err := mfsr.RepoPath(repoPath).WriteVersion(RepoVersion); err != nil {
+	if err := migrations.WriteRepoVersion(repoPath, RepoVersion); err != nil {
 		return err
 	}
 

repo/fsrepo/migrations/fetch.go

@@ -0,0 +1,269 @@
+package migrations
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/exec"
+	"path"
+	"runtime"
+	"strings"
+)
+
+const (
+	envIpfsDistPath = "IPFS_DIST_PATH"
+
+	// Distribution
+	gatewayURL = "https://ipfs.io"
+	ipfsDist   = "/ipns/dist.ipfs.io"
+
+	// Maximum download size
+	fetchSizeLimit = 1024 * 1024 * 512
+)
+
+type limitReadCloser struct {
+	io.Reader
+	io.Closer
+}
+
+var ipfsDistPath string
+
+func init() {
+	SetIpfsDistPath("")
+}
+
+// SetIpfsDistPath sets the ipfs path to the distribution site.  If an empty
+// string is given, then the path is set using the IPFS_DIST_PATH environ
+// veriable, or the default value if that is not defined.
+func SetIpfsDistPath(distPath string) {
+	if distPath != "" {
+		ipfsDistPath = distPath
+		return
+	}
+
+	if dist := os.Getenv(envIpfsDistPath); dist != "" {
+		ipfsDistPath = dist
+	} else {
+		ipfsDistPath = ipfsDist
+	}
+}
+
+// FetchBinary downloads an archive from the distribution site and unpacks it.
+//
+// The base name of the archive file, inside the distribution directory on
+// distribution site, may differ from the distribution name.  If it does, then
+// specify arcName.
+//
+// The base name of the binary inside the archive may differ from the base
+// archive name.  If it does, then specify binName.  For example, the following
+// is needed because the archive "go-ipfs_v0.7.0_linux-amd64.tar.gz" contains a
+// binary named "ipfs"
+//
+//     FetchBinary(ctx, "go-ipfs", "v0.7.0", "go-ipfs", "ipfs", tmpDir)
+//
+// If out is a directory, then the binary is written to that directory with the
+// same name it has inside the archive.  Otherwise, the binary file is written
+// to the file named by out.
+func FetchBinary(ctx context.Context, dist, ver, arcName, binName, out string) (string, error) {
+	// If archive base name not specified, then it is same as dist.
+	if arcName == "" {
+		arcName = dist
+	}
+	// If binary base name is not specified, then it is same as archive base name.
+	if binName == "" {
+		binName = arcName
+	}
+
+	// Name of binary that exists inside archive
+	binName = ExeName(binName)
+
+	// Return error if file exists or stat fails for reason other than not
+	// exists.  If out is a directory, then write extracted binary to that dir.
+	fi, err := os.Stat(out)
+	if !os.IsNotExist(err) {
+		if err != nil {
+			return "", err
+		}
+		if !fi.IsDir() {
+			return "", &os.PathError{
+				Op:   "FetchBinary",
+				Path: out,
+				Err:  os.ErrExist,
+			}
+		}
+		// out exists and is a directory, so compose final name
+		out = path.Join(out, binName)
+	}
+
+	// Create temp directory to store download
+	tmpDir, err := ioutil.TempDir("", arcName)
+	if err != nil {
+		return "", err
+	}
+	defer os.RemoveAll(tmpDir)
+
+	atype := "tar.gz"
+	if runtime.GOOS == "windows" {
+		atype = "zip"
+	}
+
+	arcName = makeArchiveName(arcName, ver, atype)
+	arcIpfsPath := makeIpfsPath(dist, ver, arcName)
+
+	// Create a file to write the archive data to
+	arcPath := path.Join(tmpDir, arcName)
+	arcFile, err := os.Create(arcPath)
+	if err != nil {
+		return "", err
+	}
+	defer arcFile.Close()
+
+	// Open connection to download archive from ipfs path
+	rc, err := fetch(ctx, arcIpfsPath)
+	if err != nil {
+		return "", err
+	}
+	defer rc.Close()
+
+	// Write download data
+	_, err = io.Copy(arcFile, rc)
+	if err != nil {
+		return "", err
+	}
+	arcFile.Close()
+
+	// Unpack the archive and write binary to out
+	err = unpackArchive(arcPath, atype, dist, binName, out)
+	if err != nil {
+		return "", err
+	}
+
+	// Set mode of binary to executable
+	err = os.Chmod(out, 0755)
+	if err != nil {
+		return "", err
+	}
+
+	return out, nil
+}
+
+// fetch attempts to fetch the file at the given ipfs path, first using the
+// local ipfs api if available, then using http.  Returns io.ReadCloser on
+// success, which caller must close.
+func fetch(ctx context.Context, ipfsPath string) (io.ReadCloser, error) {
+	// Try fetching via ipfs daemon
+	rc, err := ipfsFetch(ctx, ipfsPath)
+	if err == nil {
+		// Transferred using local ipfs daemon
+		return rc, nil
+	}
+	// Try fetching via HTTP
+	return httpFetch(ctx, gatewayURL+ipfsPath)
+}
+
+// ipfsFetch attempts to fetch the file at the given ipfs path using the local
+// ipfs api.  Returns io.ReadCloser on success, which caller must close.
+func ipfsFetch(ctx context.Context, ipfsPath string) (io.ReadCloser, error) {
+	sh, _, err := ApiShell("")
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := sh.Request("cat", ipfsPath).Send(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if resp.Error != nil {
+		return nil, resp.Error
+	}
+
+	return newLimitReadCloser(resp.Output, fetchSizeLimit), nil
+}
+
+// httpFetch attempts to fetch the file at the given URL.  Returns
+// io.ReadCloser on success, which caller must close.
+func httpFetch(ctx context.Context, url string) (io.ReadCloser, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("http.NewRequest error: %s", err)
+	}
+
+	req.Header.Set("User-Agent", "go-ipfs")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("http.DefaultClient.Do error: %s", err)
+	}
+
+	if resp.StatusCode >= 400 {
+		defer resp.Body.Close()
+		mes, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return nil, fmt.Errorf("error reading error body: %s", err)
+		}
+		return nil, fmt.Errorf("GET %s error: %s: %s", url, resp.Status, string(mes))
+	}
+
+	return newLimitReadCloser(resp.Body, fetchSizeLimit), nil
+}
+
+func newLimitReadCloser(rc io.ReadCloser, limit int64) io.ReadCloser {
+	return limitReadCloser{
+		Reader: io.LimitReader(rc, limit),
+		Closer: rc,
+	}
+}
+
+// osWithVariant returns the OS name with optional variant.
+// Currently returns either runtime.GOOS, or "linux-musl".
+func osWithVariant() (string, error) {
+	if runtime.GOOS != "linux" {
+		return runtime.GOOS, nil
+	}
+
+	// ldd outputs the system's kind of libc.
+	// - on standard ubuntu: ldd (Ubuntu GLIBC 2.23-0ubuntu5) 2.23
+	// - on alpine: musl libc (x86_64)
+	//
+	// we use the combined stdout+stderr,
+	// because ldd --version prints differently on different OSes.
+	// - on standard ubuntu: stdout
+	// - on alpine: stderr (it probably doesn't know the --version flag)
+	//
+	// we suppress non-zero exit codes (see last point about alpine).
+	out, err := exec.Command("sh", "-c", "ldd --version || true").CombinedOutput()
+	if err != nil {
+		return "", err
+	}
+
+	// now just see if we can find "musl" somewhere in the output
+	scan := bufio.NewScanner(bytes.NewBuffer(out))
+	for scan.Scan() {
+		if strings.Contains(scan.Text(), "musl") {
+			return "linux-musl", nil
+		}
+	}
+
+	return "linux", nil
+}
+
+// makeArchiveName composes the name of a migration binary archive.
+//
+// The archive name is in the format: name_version_osv-GOARCH.atype
+// Example: ipfs-10-to-11_v1.8.0_darwin-amd64.tar.gz
+func makeArchiveName(name, ver, atype string) string {
+	return fmt.Sprintf("%s_%s_%s-%s.%s", name, ver, runtime.GOOS, runtime.GOARCH, atype)
+}
+
+// makeIpfsPath composes the name ipfs path location to download a migration
+// binary from the distribution site.
+//
+// The ipfs path format: distBaseCID/rootdir/version/name/archive
+func makeIpfsPath(dist, ver, arcName string) string {
+	return fmt.Sprintf("%s/%s/%s/%s", ipfsDistPath, dist, ver, arcName)
+}