repo/fsrepo/migrations: verified HTTP migrations (#10324)

docs/changelogs/v0.27.md

@@ -10,6 +10,7 @@
   - [IPNS resolver cache's TTL can now be configured](#ipns-resolver-caches-ttl-can-now-be-configured)
   - [RPC client: deprecated DHT API, added Routing API](#rpc-client-deprecated-dht-api-added-routing-api)
   - [Deprecated DHT commands removed from `/api/v0/dht`](#deprecated-dht-commands-removed-from-apiv0dht)
+  - [Repository migrations are now trustless](#repository-migrations-are-now-trustless)
 - [📝 Changelog](#-changelog)
 - [👨‍👩‍👧‍👦 Contributors](#-contributors)
 
@@ -37,6 +38,10 @@ In the next version, all DHT deprecated methods will be removed from the Go RPC
 
 All the DHT commands that were deprecated for over a year were finally removed from `/api/v0/dht`. Users should switch to modern `/api/v0/routing` which works with [both Amino DHT and Delegated Routers](https://github.com/ipfs/kubo/blob/master/docs/config.md#routing).
 
+#### Repository migrations are now trustless
+
+Kubo now only uses [trustless requests](https://specs.ipfs.tech/http-gateways/trustless-gateway/) (e.g., CAR files) when downloading repository migrations via HTTP. This further strengthens Kubo by not delegating trust to public gateways. The migration binaries are locally verified before being executed. 
+
 ### 📝 Changelog
 
 ### 👨‍👩‍👧‍👦 Contributors

docs/gateway.md

@@ -12,14 +12,18 @@ Kubo's Gateway implementation follows [ipfs/specs: Specification for HTTP Gatewa
 
 By default, Kubo nodes run
 a [path gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#path-gateway) at `http://127.0.0.1:8080/`
-and a [subdomain gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway) at `http://localhost:8080/`
+and a [subdomain gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway) at `http://localhost:8080/`.
+Both support [trustless responses](https://docs.ipfs.tech/reference/http/gateway/#trustless-verifiable-retrieval) as opt-in via `Accept` header.
 
 Additional listening addresses and gateway behaviors can be set in the [config](#configuration) file.
 
 ### Public gateways
 
-Protocol Labs provides a public gateway at `https://ipfs.io` (path) and `https://dweb.link` (subdomain).
-If you've ever seen a link in the form `https://ipfs.io/ipfs/Qm...`, that's being served from *our* gateway.
+Protocol Labs provides a public gateway at
+`https://ipfs.io` ([path](https://specs.ipfs.tech/http-gateways/path-gateway/)),
+`https://dweb.link` ([subdomain](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway)),
+and `https://trustless-gateway.link` ([trustless](https://specs.ipfs.tech/http-gateways/trustless-gateway/) only).
+If you've ever seen a link in the form `https://ipfs.io/ipfs/Qm...`, that's being served from a *public goods* gateway.
 
 There is a list of third-party public gateways provided by the IPFS community at https://ipfs.github.io/public-gateway-checker/
 
@@ -105,7 +109,7 @@ This is a rough equivalent of `ipfs dag export`.
 
 ## Deprecated Subset of RPC API
 
-For legacy reasons, the gateway port exposes a small subset of RPC API under `/api/v0/`.
+For legacy reasons, some gateways may expose a small subset of RPC API under `/api/v0/`.
 While this read-only API exposes a read-only, "safe" subset of the normal API,
 it is deprecated and should not be used for greenfield projects.
 

go.mod

@@ -136,6 +136,7 @@ require (
 	github.com/ipfs/go-bitfield v1.1.0 // indirect
 	github.com/ipfs/go-blockservice v0.5.0 // indirect
 	github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect
+	github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect
 	github.com/ipfs/go-ipfs-delay v0.0.1 // indirect
 	github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
 	github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect

go.sum

@@ -330,11 +330,13 @@ github.com/ipfs/boxo v0.17.1-0.20240206084652-79cb4e2886d7/go.mod h1:pIZgTWdm3k3
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=
+github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
 github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk=
 github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs=
 github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM=
 github.com/ipfs/go-blockservice v0.5.0 h1:B2mwhhhVQl2ntW2EIpaWPwSCxSuqr5fFA93Ms4bYLEY=
 github.com/ipfs/go-blockservice v0.5.0/go.mod h1:W6brZ5k20AehbmERplmERn8o2Ni3ZZubvAxaIUeaT6w=
+github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
 github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
 github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M=
 github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog=
@@ -367,6 +369,7 @@ github.com/ipfs/go-ipfs-blockstore v1.3.0 h1:m2EXaWgwTzAfsmt5UdJ7Is6l4gJcaM/A12X
 github.com/ipfs/go-ipfs-blockstore v1.3.0/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE=
 github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ=
 github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8=
+github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8=
 github.com/ipfs/go-ipfs-cmds v0.10.0 h1:ZB4+RgYaH4UARfJY0uLKl5UXgApqnRjKbuCiJVcErYk=
 github.com/ipfs/go-ipfs-cmds v0.10.0/go.mod h1:sX5d7jkCft9XLPnkgEfXY0z2UBOB5g6fh/obBS0enJE=
 github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=

repo/fsrepo/migrations/fetch_test.go

@@ -5,56 +5,13 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"io"
-	"net/http"
-	"net/http/httptest"
 	"os"
-	"path"
 	"path/filepath"
 	"runtime"
 	"strings"
 	"testing"
 )
 
-func createTestServer() *httptest.Server {
-	reqHandler := func(w http.ResponseWriter, r *http.Request) {
-		defer r.Body.Close()
-		if strings.Contains(r.URL.Path, "not-here") {
-			http.NotFound(w, r)
-		} else if strings.HasSuffix(r.URL.Path, "versions") {
-			fmt.Fprint(w, "v1.0.0\nv1.1.0\nv1.1.2\nv2.0.0-rc1\n2.0.0\nv2.0.1\n")
-		} else if strings.HasSuffix(r.URL.Path, ".tar.gz") {
-			createFakeArchive(r.URL.Path, false, w)
-		} else if strings.HasSuffix(r.URL.Path, "zip") {
-			createFakeArchive(r.URL.Path, true, w)
-		} else {
-			http.NotFound(w, r)
-		}
-	}
-	return httptest.NewServer(http.HandlerFunc(reqHandler))
-}
-
-func createFakeArchive(name string, archZip bool, w io.Writer) {
-	fileName := strings.Split(path.Base(name), "_")[0]
-	root := path.Base(path.Dir(path.Dir(name)))
-
-	// Simulate fetching go-ipfs, which has "ipfs" as the name in the archive.
-	if fileName == "go-ipfs" {
-		fileName = "ipfs"
-	}
-	fileName = ExeName(fileName)
-
-	var err error
-	if archZip {
-		err = writeZip(root, fileName, "FAKE DATA", w)
-	} else {
-		err = writeTarGzip(root, fileName, "FAKE DATA", w)
-	}
-	if err != nil {
-		panic(err)
-	}
-}
-
 func TestGetDistPath(t *testing.T) {
 	os.Unsetenv(envIpfsDistPath)
 	distPath := GetDistPathEnv("")
@@ -91,12 +48,9 @@ func TestHttpFetch(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	ts := createTestServer()
-	defer ts.Close()
-
-	fetcher := NewHttpFetcher("", ts.URL, "", 0)
+	fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0)
 
-	out, err := fetcher.Fetch(ctx, "/versions")
+	out, err := fetcher.Fetch(ctx, "/kubo/versions")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -120,7 +74,7 @@ func TestHttpFetch(t *testing.T) {
 
 	// Check not found
 	_, err = fetcher.Fetch(ctx, "/no_such_file")
-	if err == nil || !strings.Contains(err.Error(), "404") {
+	if err == nil || !strings.Contains(err.Error(), "no link") {
 		t.Fatal("expected error 404")
 	}
 }
@@ -131,10 +85,7 @@ func TestFetchBinary(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	ts := createTestServer()
-	defer ts.Close()
-
-	fetcher := NewHttpFetcher("", ts.URL, "", 0)
+	fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0)
 
 	vers, err := DistVersions(ctx, fetcher, distFSRM, false)
 	if err != nil {
@@ -154,7 +105,7 @@ func TestFetchBinary(t *testing.T) {
 
 	t.Log("downloaded and unpacked", fi.Size(), "byte file:", fi.Name())
 
-	bin, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", tmpDir)
+	bin, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "ipfs", tmpDir)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -167,12 +118,12 @@ func TestFetchBinary(t *testing.T) {
 	t.Log("downloaded and unpacked", fi.Size(), "byte file:", fi.Name())
 
 	// Check error is destination already exists and is not directory
-	_, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", bin)
+	_, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "ipfs", bin)
 	if !os.IsExist(err) {
 		t.Fatal("expected 'exists' error, got", err)
 	}
 
-	_, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", tmpDir)
+	_, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "ipfs", tmpDir)
 	if !os.IsExist(err) {
 		t.Error("expected 'exists' error, got:", err)
 	}
@@ -192,7 +143,7 @@ func TestFetchBinary(t *testing.T) {
 		if err != nil {
 			panic(err)
 		}
-		_, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", tmpDir)
+		_, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "ipfs", tmpDir)
 		if !os.IsPermission(err) {
 			t.Error("expected 'permission' error, got:", err)
 		}
@@ -207,13 +158,13 @@ func TestFetchBinary(t *testing.T) {
 	}
 
 	// Check error if failure to fetch due to bad dist
-	_, err = FetchBinary(ctx, fetcher, "not-here", "v0.3.5", "ipfs", tmpDir)
-	if err == nil || !strings.Contains(err.Error(), "Not Found") {
+	_, err = FetchBinary(ctx, fetcher, "not-here", "v1.0.0", "ipfs", tmpDir)
+	if err == nil || !strings.Contains(err.Error(), "no link") {
 		t.Error("expected 'Not Found' error, got:", err)
 	}
 
 	// Check error if failure to unpack archive
-	_, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "not-such-bin", tmpDir)
+	_, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "not-such-bin", tmpDir)
 	if err == nil || err.Error() != "no binary found in archive" {
 		t.Error("expected 'no binary found in archive' error")
 	}
@@ -223,15 +174,12 @@ func TestMultiFetcher(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	ts := createTestServer()
-	defer ts.Close()
-
 	badFetcher := NewHttpFetcher("", "bad-url", "", 0)
-	fetcher := NewHttpFetcher("", ts.URL, "", 0)
+	fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0)
 
 	mf := NewMultiFetcher(badFetcher, fetcher)
 
-	vers, err := mf.Fetch(ctx, "/versions")
+	vers, err := mf.Fetch(ctx, "/kubo/versions")
 	if err != nil {
 		t.Fatal(err)
 	}