From 595e1ba26857a3f694d5fb122045867b1e946dca Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 19 Feb 2024 14:20:58 +0100 Subject: [PATCH] repo/fsrepo/migrations: verified HTTP migrations (#10324) --- docs/changelogs/v0.27.md | 5 + docs/gateway.md | 12 +- go.mod | 1 + go.sum | 3 + repo/fsrepo/migrations/fetch_test.go | 78 ++------ repo/fsrepo/migrations/httpfetcher.go | 177 ++++++++++++++-- repo/fsrepo/migrations/migrations_test.go | 8 +- repo/fsrepo/migrations/setup_test.go | 233 ++++++++++++++++++++++ repo/fsrepo/migrations/versions_test.go | 8 +- test/sharness/t0003-docker-migrate.sh | 19 +- 10 files changed, 444 insertions(+), 100 deletions(-) create mode 100644 repo/fsrepo/migrations/setup_test.go diff --git a/docs/changelogs/v0.27.md b/docs/changelogs/v0.27.md index 3d4df68eac9..0f9808a15fa 100644 --- a/docs/changelogs/v0.27.md +++ b/docs/changelogs/v0.27.md @@ -10,6 +10,7 @@ - [IPNS resolver cache's TTL can now be configured](#ipns-resolver-caches-ttl-can-now-be-configured) - [RPC client: deprecated DHT API, added Routing API](#rpc-client-deprecated-dht-api-added-routing-api) - [Deprecated DHT commands removed from `/api/v0/dht`](#deprecated-dht-commands-removed-from-apiv0dht) + - [Repository migrations are now trustless](#repository-migrations-are-now-trustless) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -37,6 +38,10 @@ In the next version, all DHT deprecated methods will be removed from the Go RPC All the DHT commands that were deprecated for over a year were finally removed from `/api/v0/dht`. Users should switch to modern `/api/v0/routing` which works with [both Amino DHT and Delegated Routers](https://github.com/ipfs/kubo/blob/master/docs/config.md#routing). +#### Repository migrations are now trustless + +Kubo now only uses [trustless requests](https://specs.ipfs.tech/http-gateways/trustless-gateway/) (e.g., CAR files) when downloading repository migrations via HTTP. This further strengthens Kubo by not delegating trust to public gateways. The migration binaries are locally verified before being executed. + ### ๐Ÿ“ Changelog ### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors diff --git a/docs/gateway.md b/docs/gateway.md index b24d10f0c19..531c8c6f94e 100644 --- a/docs/gateway.md +++ b/docs/gateway.md @@ -12,14 +12,18 @@ Kubo's Gateway implementation follows [ipfs/specs: Specification for HTTP Gatewa By default, Kubo nodes run a [path gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#path-gateway) at `http://127.0.0.1:8080/` -and a [subdomain gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway) at `http://localhost:8080/` +and a [subdomain gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway) at `http://localhost:8080/`. +Both support [trustless responses](https://docs.ipfs.tech/reference/http/gateway/#trustless-verifiable-retrieval) as opt-in via `Accept` header. Additional listening addresses and gateway behaviors can be set in the [config](#configuration) file. ### Public gateways -Protocol Labs provides a public gateway at `https://ipfs.io` (path) and `https://dweb.link` (subdomain). -If you've ever seen a link in the form `https://ipfs.io/ipfs/Qm...`, that's being served from *our* gateway. +Protocol Labs provides a public gateway at +`https://ipfs.io` ([path](https://specs.ipfs.tech/http-gateways/path-gateway/)), +`https://dweb.link` ([subdomain](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway)), +and `https://trustless-gateway.link` ([trustless](https://specs.ipfs.tech/http-gateways/trustless-gateway/) only). +If you've ever seen a link in the form `https://ipfs.io/ipfs/Qm...`, that's being served from a *public goods* gateway. There is a list of third-party public gateways provided by the IPFS community at https://ipfs.github.io/public-gateway-checker/ @@ -105,7 +109,7 @@ This is a rough equivalent of `ipfs dag export`. ## Deprecated Subset of RPC API -For legacy reasons, the gateway port exposes a small subset of RPC API under `/api/v0/`. +For legacy reasons, some gateways may expose a small subset of RPC API under `/api/v0/`. While this read-only API exposes a read-only, "safe" subset of the normal API, it is deprecated and should not be used for greenfield projects. diff --git a/go.mod b/go.mod index 39d2e31a655..8d7cc5ff4ec 100644 --- a/go.mod +++ b/go.mod @@ -136,6 +136,7 @@ require ( github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-blockservice v0.5.0 // indirect github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect + github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect github.com/ipfs/go-ipfs-delay v0.0.1 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect diff --git a/go.sum b/go.sum index 75720e19382..cd6030b32c3 100644 --- a/go.sum +++ b/go.sum @@ -330,11 +330,13 @@ github.com/ipfs/boxo v0.17.1-0.20240206084652-79cb4e2886d7/go.mod h1:pIZgTWdm3k3 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= +github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM= github.com/ipfs/go-blockservice v0.5.0 h1:B2mwhhhVQl2ntW2EIpaWPwSCxSuqr5fFA93Ms4bYLEY= github.com/ipfs/go-blockservice v0.5.0/go.mod h1:W6brZ5k20AehbmERplmERn8o2Ni3ZZubvAxaIUeaT6w= +github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= @@ -367,6 +369,7 @@ github.com/ipfs/go-ipfs-blockstore v1.3.0 h1:m2EXaWgwTzAfsmt5UdJ7Is6l4gJcaM/A12X github.com/ipfs/go-ipfs-blockstore v1.3.0/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8= +github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8= github.com/ipfs/go-ipfs-cmds v0.10.0 h1:ZB4+RgYaH4UARfJY0uLKl5UXgApqnRjKbuCiJVcErYk= github.com/ipfs/go-ipfs-cmds v0.10.0/go.mod h1:sX5d7jkCft9XLPnkgEfXY0z2UBOB5g6fh/obBS0enJE= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= diff --git a/repo/fsrepo/migrations/fetch_test.go b/repo/fsrepo/migrations/fetch_test.go index 27452d386af..6e87c966bbe 100644 --- a/repo/fsrepo/migrations/fetch_test.go +++ b/repo/fsrepo/migrations/fetch_test.go @@ -5,56 +5,13 @@ import ( "bytes" "context" "fmt" - "io" - "net/http" - "net/http/httptest" "os" - "path" "path/filepath" "runtime" "strings" "testing" ) -func createTestServer() *httptest.Server { - reqHandler := func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - if strings.Contains(r.URL.Path, "not-here") { - http.NotFound(w, r) - } else if strings.HasSuffix(r.URL.Path, "versions") { - fmt.Fprint(w, "v1.0.0\nv1.1.0\nv1.1.2\nv2.0.0-rc1\n2.0.0\nv2.0.1\n") - } else if strings.HasSuffix(r.URL.Path, ".tar.gz") { - createFakeArchive(r.URL.Path, false, w) - } else if strings.HasSuffix(r.URL.Path, "zip") { - createFakeArchive(r.URL.Path, true, w) - } else { - http.NotFound(w, r) - } - } - return httptest.NewServer(http.HandlerFunc(reqHandler)) -} - -func createFakeArchive(name string, archZip bool, w io.Writer) { - fileName := strings.Split(path.Base(name), "_")[0] - root := path.Base(path.Dir(path.Dir(name))) - - // Simulate fetching go-ipfs, which has "ipfs" as the name in the archive. - if fileName == "go-ipfs" { - fileName = "ipfs" - } - fileName = ExeName(fileName) - - var err error - if archZip { - err = writeZip(root, fileName, "FAKE DATA", w) - } else { - err = writeTarGzip(root, fileName, "FAKE DATA", w) - } - if err != nil { - panic(err) - } -} - func TestGetDistPath(t *testing.T) { os.Unsetenv(envIpfsDistPath) distPath := GetDistPathEnv("") @@ -91,12 +48,9 @@ func TestHttpFetch(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - ts := createTestServer() - defer ts.Close() - - fetcher := NewHttpFetcher("", ts.URL, "", 0) + fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0) - out, err := fetcher.Fetch(ctx, "/versions") + out, err := fetcher.Fetch(ctx, "/kubo/versions") if err != nil { t.Fatal(err) } @@ -120,7 +74,7 @@ func TestHttpFetch(t *testing.T) { // Check not found _, err = fetcher.Fetch(ctx, "/no_such_file") - if err == nil || !strings.Contains(err.Error(), "404") { + if err == nil || !strings.Contains(err.Error(), "no link") { t.Fatal("expected error 404") } } @@ -131,10 +85,7 @@ func TestFetchBinary(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - ts := createTestServer() - defer ts.Close() - - fetcher := NewHttpFetcher("", ts.URL, "", 0) + fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0) vers, err := DistVersions(ctx, fetcher, distFSRM, false) if err != nil { @@ -154,7 +105,7 @@ func TestFetchBinary(t *testing.T) { t.Log("downloaded and unpacked", fi.Size(), "byte file:", fi.Name()) - bin, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", tmpDir) + bin, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "ipfs", tmpDir) if err != nil { t.Fatal(err) } @@ -167,12 +118,12 @@ func TestFetchBinary(t *testing.T) { t.Log("downloaded and unpacked", fi.Size(), "byte file:", fi.Name()) // Check error is destination already exists and is not directory - _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", bin) + _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "ipfs", bin) if !os.IsExist(err) { t.Fatal("expected 'exists' error, got", err) } - _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", tmpDir) + _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "ipfs", tmpDir) if !os.IsExist(err) { t.Error("expected 'exists' error, got:", err) } @@ -192,7 +143,7 @@ func TestFetchBinary(t *testing.T) { if err != nil { panic(err) } - _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", tmpDir) + _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "ipfs", tmpDir) if !os.IsPermission(err) { t.Error("expected 'permission' error, got:", err) } @@ -207,13 +158,13 @@ func TestFetchBinary(t *testing.T) { } // Check error if failure to fetch due to bad dist - _, err = FetchBinary(ctx, fetcher, "not-here", "v0.3.5", "ipfs", tmpDir) - if err == nil || !strings.Contains(err.Error(), "Not Found") { + _, err = FetchBinary(ctx, fetcher, "not-here", "v1.0.0", "ipfs", tmpDir) + if err == nil || !strings.Contains(err.Error(), "no link") { t.Error("expected 'Not Found' error, got:", err) } // Check error if failure to unpack archive - _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "not-such-bin", tmpDir) + _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "not-such-bin", tmpDir) if err == nil || err.Error() != "no binary found in archive" { t.Error("expected 'no binary found in archive' error") } @@ -223,15 +174,12 @@ func TestMultiFetcher(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - ts := createTestServer() - defer ts.Close() - badFetcher := NewHttpFetcher("", "bad-url", "", 0) - fetcher := NewHttpFetcher("", ts.URL, "", 0) + fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0) mf := NewMultiFetcher(badFetcher, fetcher) - vers, err := mf.Fetch(ctx, "/versions") + vers, err := mf.Fetch(ctx, "/kubo/versions") if err != nil { t.Fatal(err) } diff --git a/repo/fsrepo/migrations/httpfetcher.go b/repo/fsrepo/migrations/httpfetcher.go index 9665a1e98ce..2ae180b1ea2 100644 --- a/repo/fsrepo/migrations/httpfetcher.go +++ b/repo/fsrepo/migrations/httpfetcher.go @@ -2,21 +2,40 @@ package migrations import ( "context" + "errors" "fmt" "io" "net/http" - "path" + gopath "path" "strings" + + "github.com/ipfs/boxo/blockservice" + "github.com/ipfs/boxo/blockstore" + "github.com/ipfs/boxo/exchange/offline" + bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice" + files "github.com/ipfs/boxo/files" + "github.com/ipfs/boxo/ipld/merkledag" + unixfile "github.com/ipfs/boxo/ipld/unixfs/file" + "github.com/ipfs/boxo/ipns" + "github.com/ipfs/boxo/namesys" + "github.com/ipfs/boxo/path" + "github.com/ipfs/boxo/path/resolver" + "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + "github.com/ipfs/go-unixfsnode" + gocarv2 "github.com/ipld/go-car/v2" + dagpb "github.com/ipld/go-codec-dagpb" + madns "github.com/multiformats/go-multiaddr-dns" ) const ( // default is different name than ipfs.io which is being blocked by some ISPs - defaultGatewayURL = "https://dweb.link" + defaultGatewayURL = "https://trustless-gateway.link" // Default maximum download size. defaultFetchLimit = 1024 * 1024 * 512 ) -// HttpFetcher fetches files over HTTP. +// HttpFetcher fetches files over HTTP using verifiable CAR archives. type HttpFetcher struct { //nolint distPath string gateway string @@ -26,7 +45,7 @@ type HttpFetcher struct { //nolint var _ Fetcher = (*HttpFetcher)(nil) -// NewHttpFetcher creates a new HttpFetcher +// NewHttpFetcher creates a new [HttpFetcher]. // // Specifying "" for distPath sets the default IPNS path. // Specifying "" for gateway sets the default. @@ -62,13 +81,89 @@ func NewHttpFetcher(distPath, gateway, userAgent string, fetchLimit int64) *Http // Fetch attempts to fetch the file at the given path, from the distribution // site configured for this HttpFetcher. func (f *HttpFetcher) Fetch(ctx context.Context, filePath string) ([]byte, error) { - gwURL := f.gateway + path.Join(f.distPath, filePath) - fmt.Printf("Fetching with HTTP: %q\n", gwURL) + imPath, err := f.resolvePath(ctx, gopath.Join(f.distPath, filePath)) + if err != nil { + return nil, fmt.Errorf("path could not be resolved: %w", err) + } + + rc, err := f.httpRequest(ctx, imPath, "application/vnd.ipld.car") + if err != nil { + return nil, fmt.Errorf("failed to fetch CAR: %w", err) + } + + return carStreamToFileBytes(ctx, rc, imPath) +} + +func (f *HttpFetcher) Close() error { + return nil +} + +func (f *HttpFetcher) resolvePath(ctx context.Context, pathStr string) (path.ImmutablePath, error) { + p, err := path.NewPath(pathStr) + if err != nil { + return path.ImmutablePath{}, fmt.Errorf("path is invalid: %w", err) + } + + for p.Mutable() { + // Download IPNS record and verify through the gateway, or resolve the + // DNSLink with the default DNS resolver. + name, err := ipns.NameFromString(p.Segments()[1]) + if err == nil { + p, err = f.resolveIPNS(ctx, name) + } else { + p, err = f.resolveDNSLink(ctx, p) + } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, gwURL, nil) + if err != nil { + return path.ImmutablePath{}, err + } + } + + return path.NewImmutablePath(p) +} + +func (f *HttpFetcher) resolveIPNS(ctx context.Context, name ipns.Name) (path.Path, error) { + rc, err := f.httpRequest(ctx, name.AsPath(), "application/vnd.ipfs.ipns-record") + if err != nil { + return path.ImmutablePath{}, err + } + + rc = NewLimitReadCloser(rc, int64(ipns.MaxRecordSize)) + rawRecord, err := io.ReadAll(rc) + if err != nil { + return path.ImmutablePath{}, err + } + + rec, err := ipns.UnmarshalRecord(rawRecord) + if err != nil { + return path.ImmutablePath{}, err + } + + err = ipns.ValidateWithName(rec, name) + if err != nil { + return path.ImmutablePath{}, err + } + + return rec.Value() +} + +func (f *HttpFetcher) resolveDNSLink(ctx context.Context, p path.Path) (path.Path, error) { + dnsResolver := namesys.NewDNSResolver(madns.DefaultResolver.LookupTXT) + res, err := dnsResolver.Resolve(ctx, p) + if err != nil { + return nil, err + } + return res.Path, nil +} + +func (f *HttpFetcher) httpRequest(ctx context.Context, p path.Path, accept string) (io.ReadCloser, error) { + url := f.gateway + p.String() + fmt.Printf("Fetching with HTTP: %q\n", url) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, fmt.Errorf("http.NewRequest error: %w", err) } + req.Header.Set("Accept", accept) if f.userAgent != "" { req.Header.Set("User-Agent", f.userAgent) @@ -85,7 +180,7 @@ func (f *HttpFetcher) Fetch(ctx context.Context, filePath string) ([]byte, error if err != nil { return nil, fmt.Errorf("error reading error body: %w", err) } - return nil, fmt.Errorf("GET %s error: %s: %s", gwURL, resp.Status, string(mes)) + return nil, fmt.Errorf("GET %s error: %s: %s", url, resp.Status, string(mes)) } var rc io.ReadCloser @@ -94,11 +189,69 @@ func (f *HttpFetcher) Fetch(ctx context.Context, filePath string) ([]byte, error } else { rc = resp.Body } - defer rc.Close() - return io.ReadAll(rc) + return rc, nil } -func (f *HttpFetcher) Close() error { - return nil +func carStreamToFileBytes(ctx context.Context, r io.ReadCloser, imPath path.ImmutablePath) ([]byte, error) { + defer r.Close() + + // Create temporary block datastore and dag service. + dataStore := dssync.MutexWrap(datastore.NewMapDatastore()) + blockStore := blockstore.NewBlockstore(dataStore) + blockService := blockservice.New(blockStore, offline.Exchange(blockStore)) + dagService := merkledag.NewDAGService(blockService) + + defer dagService.Blocks.Close() + defer dataStore.Close() + + // Create CAR reader + car, err := gocarv2.NewBlockReader(r) + if err != nil { + fmt.Println(err) + return nil, fmt.Errorf("error creating car reader: %s", err) + } + + // Add all blocks to the blockstore. + for { + block, err := car.Next() + if err != nil && err != io.EOF { + return nil, fmt.Errorf("error reading block from car: %s", err) + } else if block == nil { + break + } + + err = blockStore.Put(ctx, block) + if err != nil { + return nil, fmt.Errorf("error putting block in blockstore: %s", err) + } + } + + fetcherCfg := bsfetcher.NewFetcherConfig(blockService) + fetcherCfg.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser) + fetcher := fetcherCfg.WithReifier(unixfsnode.Reify) + resolver := resolver.NewBasicResolver(fetcher) + + cid, _, err := resolver.ResolveToLastNode(ctx, imPath) + if err != nil { + return nil, fmt.Errorf("failed to resolve: %w", err) + } + + nd, err := dagService.Get(ctx, cid) + if err != nil { + return nil, fmt.Errorf("failed to resolve: %w", err) + } + + // Make UnixFS file out of the node. + uf, err := unixfile.NewUnixfsFile(ctx, dagService, nd) + if err != nil { + return nil, fmt.Errorf("error building unixfs file: %s", err) + } + + // Check if it's a file and return. + if f, ok := uf.(files.File); ok { + return io.ReadAll(f) + } + + return nil, errors.New("unexpected unixfs node type") } diff --git a/repo/fsrepo/migrations/migrations_test.go b/repo/fsrepo/migrations/migrations_test.go index 2fd75b7e9de..96370f864e8 100644 --- a/repo/fsrepo/migrations/migrations_test.go +++ b/repo/fsrepo/migrations/migrations_test.go @@ -110,9 +110,7 @@ func TestFetchMigrations(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - ts := createTestServer() - defer ts.Close() - fetcher := NewHttpFetcher(CurrentIpfsDist, ts.URL, "", 0) + fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0) tmpDir := t.TempDir() @@ -162,9 +160,7 @@ func TestRunMigrations(t *testing.T) { t.Fatal(err) } - ts := createTestServer() - defer ts.Close() - fetcher := NewHttpFetcher(CurrentIpfsDist, ts.URL, "", 0) + fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0) ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/repo/fsrepo/migrations/setup_test.go b/repo/fsrepo/migrations/setup_test.go new file mode 100644 index 00000000000..2e306fda17f --- /dev/null +++ b/repo/fsrepo/migrations/setup_test.go @@ -0,0 +1,233 @@ +package migrations + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http/httptest" + "os" + "path" + "path/filepath" + "strings" + "testing" + + "github.com/ipfs/boxo/blockservice" + "github.com/ipfs/boxo/exchange/offline" + "github.com/ipfs/boxo/gateway" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-unixfsnode/data/builder" + "github.com/ipld/go-car/v2" + carblockstore "github.com/ipld/go-car/v2/blockstore" + "github.com/ipld/go-ipld-prime" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/multiformats/go-multicodec" + "github.com/multiformats/go-multihash" +) + +var ( + testIpfsDist string + testServer *httptest.Server +) + +func TestMain(m *testing.M) { + // Setup test data + testDataDir := makeTestData() + defer os.RemoveAll(testDataDir) + + testCar := makeTestCar(testDataDir) + defer os.RemoveAll(testCar) + + // Setup test gateway + fd := setupTestGateway(testCar) + defer fd.Close() + + // Run tests + os.Exit(m.Run()) +} + +func makeTestData() string { + tempDir, err := os.MkdirTemp("", "kubo-migrations-test-*") + if err != nil { + panic(err) + } + + versions := []string{"v1.0.0", "v1.1.0", "v1.1.2", "v2.0.0-rc1", "2.0.0", "v2.0.1"} + packages := []string{"kubo", "go-ipfs", "fs-repo-migrations", "fs-repo-1-to-2", "fs-repo-2-to-3", "fs-repo-9-to-10", "fs-repo-10-to-11"} + + // Generate fake data + for _, name := range packages { + err = os.MkdirAll(filepath.Join(tempDir, name), 0777) + if err != nil { + panic(err) + } + + err = os.WriteFile(filepath.Join(tempDir, name, "versions"), []byte(strings.Join(versions, "\n")+"\n"), 0666) + if err != nil { + panic(err) + } + + for _, version := range versions { + filename, archName := makeArchivePath(name, name, version, "tar.gz") + createFakeArchive(filepath.Join(tempDir, filename), archName, false) + + filename, archName = makeArchivePath(name, name, version, "zip") + createFakeArchive(filepath.Join(tempDir, filename), archName, true) + } + } + + return tempDir +} + +func createFakeArchive(archName, name string, archZip bool) { + err := os.MkdirAll(filepath.Dir(archName), 0777) + if err != nil { + panic(err) + } + + fileName := strings.Split(path.Base(name), "_")[0] + root := fileName + + // Simulate fetching go-ipfs, which has "ipfs" as the name in the archive. + if fileName == "go-ipfs" || fileName == "kubo" { + fileName = "ipfs" + } + fileName = ExeName(fileName) + + if archZip { + err = writeZipFile(archName, root, fileName, "FAKE DATA") + } else { + err = writeTarGzipFile(archName, root, fileName, "FAKE DATA") + } + if err != nil { + panic(err) + } +} + +// makeTestCar makes a CAR file with the directory [testData]. This code is mostly +// sourced from https://github.com/ipld/go-car/blob/1e2f0bd2c44ee31f48a8f602b25b5671cc0c4687/cmd/car/create.go +func makeTestCar(testData string) string { + // make a cid with the right length that we eventually will patch with the root. + hasher, err := multihash.GetHasher(multihash.SHA2_256) + if err != nil { + panic(err) + } + digest := hasher.Sum([]byte{}) + hash, err := multihash.Encode(digest, multihash.SHA2_256) + if err != nil { + panic(err) + } + proxyRoot := cid.NewCidV1(uint64(multicodec.DagPb), hash) + + // Make CAR file + fd, err := os.CreateTemp("", "kubo-migrations-test-*.car") + if err != nil { + panic(err) + } + defer fd.Close() + filename := fd.Name() + + rw, err := carblockstore.OpenReadWriteFile(fd, []cid.Cid{proxyRoot}, carblockstore.WriteAsCarV1(true)) + if err != nil { + panic(err) + } + defer rw.Close() + + ctx := context.Background() + + ls := cidlink.DefaultLinkSystem() + ls.TrustedStorage = true + ls.StorageReadOpener = func(_ ipld.LinkContext, l ipld.Link) (io.Reader, error) { + cl, ok := l.(cidlink.Link) + if !ok { + return nil, fmt.Errorf("not a cidlink") + } + blk, err := rw.Get(ctx, cl.Cid) + if err != nil { + return nil, err + } + return bytes.NewBuffer(blk.RawData()), nil + } + ls.StorageWriteOpener = func(_ ipld.LinkContext) (io.Writer, ipld.BlockWriteCommitter, error) { + buf := bytes.NewBuffer(nil) + return buf, func(l ipld.Link) error { + cl, ok := l.(cidlink.Link) + if !ok { + return fmt.Errorf("not a cidlink") + } + blk, err := blocks.NewBlockWithCid(buf.Bytes(), cl.Cid) + if err != nil { + return err + } + return rw.Put(ctx, blk) + }, nil + } + + l, _, err := builder.BuildUnixFSRecursive(testData, &ls) + if err != nil { + panic(err) + } + + rcl, ok := l.(cidlink.Link) + if !ok { + panic(fmt.Errorf("could not interpret %s", l)) + } + + if err := rw.Finalize(); err != nil { + panic(err) + } + // re-open/finalize with the final root. + err = car.ReplaceRootsInFile(filename, []cid.Cid{rcl.Cid}) + if err != nil { + panic(err) + } + + return filename +} + +func setupTestGateway(testCar string) io.Closer { + blockService, roots, fd, err := newBlockServiceFromCAR(testCar) + if err != nil { + panic(err) + } + + if len(roots) != 1 { + panic("expected car with 1 root") + } + + backend, err := gateway.NewBlocksBackend(blockService) + if err != nil { + panic(err) + } + conf := gateway.Config{ + NoDNSLink: false, + DeserializedResponses: false, + } + + testIpfsDist = "/ipfs/" + roots[0].String() + testServer = httptest.NewServer(gateway.NewHandler(conf, backend)) + + return fd +} + +func newBlockServiceFromCAR(filepath string) (blockservice.BlockService, []cid.Cid, io.Closer, error) { + r, err := os.Open(filepath) + if err != nil { + return nil, nil, nil, err + } + + bs, err := carblockstore.NewReadOnly(r, nil) + if err != nil { + _ = r.Close() + return nil, nil, nil, err + } + + roots, err := bs.Roots() + if err != nil { + return nil, nil, nil, err + } + + blockService := blockservice.New(bs, offline.Exchange(bs)) + return blockService, roots, r, nil +} diff --git a/repo/fsrepo/migrations/versions_test.go b/repo/fsrepo/migrations/versions_test.go index 18de72b779c..dd62f9bde8d 100644 --- a/repo/fsrepo/migrations/versions_test.go +++ b/repo/fsrepo/migrations/versions_test.go @@ -13,9 +13,7 @@ func TestDistVersions(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - ts := createTestServer() - defer ts.Close() - fetcher := NewHttpFetcher("", ts.URL, "", 0) + fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0) vers, err := DistVersions(ctx, fetcher, testDist, true) if err != nil { @@ -32,9 +30,7 @@ func TestLatestDistVersion(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - ts := createTestServer() - defer ts.Close() - fetcher := NewHttpFetcher("", ts.URL, "", 0) + fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0) latest, err := LatestDistVersion(ctx, fetcher, testDist, false) if err != nil { diff --git a/test/sharness/t0003-docker-migrate.sh b/test/sharness/t0003-docker-migrate.sh index ac3c7aee2bc..c2c7ce9697c 100755 --- a/test/sharness/t0003-docker-migrate.sh +++ b/test/sharness/t0003-docker-migrate.sh @@ -36,15 +36,20 @@ test_expect_success "configure migration sources" ' ipfs config --json Migration.DownloadSources "[\"http://127.0.0.1:17233\"]" ' -test_expect_success "make repo be version 4" ' - echo 4 > "$IPFS_PATH/version" -' - test_expect_success "setup http response" ' + mkdir migration && + echo "v1.1.1" > migration/versions && + mkdir -p migration/fs-repo-6-to-7 && + echo "v1.1.1" > migration/fs-repo-6-to-7/versions && + CID=$(ipfs add -r -Q migration) && echo "HTTP/1.1 200 OK" > vers_resp && - echo "Content-Length: 7" >> vers_resp && + echo "Content-Type: application/vnd.ipld.car" >> vers_resp && echo "" >> vers_resp && - echo "v1.1.1" >> vers_resp + ipfs dag export $CID >> vers_resp +' + +test_expect_success "make repo be version 4" ' + echo 4 > "$IPFS_PATH/version" ' test_expect_success "startup fake dists server" ' @@ -53,7 +58,7 @@ test_expect_success "startup fake dists server" ' ' test_expect_success "docker image runs" ' - DOC_ID=$(docker run -d -v "$IPFS_PATH":/data/ipfs --net=host "$IMAGE_TAG") + DOC_ID=$(docker run -d -v "$IPFS_PATH":/data/ipfs -e IPFS_DIST_PATH=/ipfs/$CID --net=host "$IMAGE_TAG") ' test_expect_success "docker container tries to pull migrations from netcat" '