feat: migration from 15 to 16 (#190)

* chore: init fs-repo-15-to-16 from 14-to-16

mostly copy & paste

* feat: append /webrtc-direct if /quic-v1 present

context:
ipfs/kubo#10463

* chore: bump latest version

* test: sharness last

this way we see if latest migration e2e passes, before we hit any issues
with legacy tooling tests, which is takes second place in priority
of things

.github/workflows/test.yml

@@ -1,7 +1,7 @@
 name: Tests
 
 env:
-  GO: 1.18
+  GO: 1.21
 
 on:
   push:
@@ -14,12 +14,12 @@ jobs:
     name: Tests
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 2
 
     - name: Set up Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: ${{ env.GO }}
 

Makefile

@@ -4,7 +4,7 @@ MIG_DIRS = $(shell ls -d fs-repo-*-to-*)
 IGNORED_DIRS := $(shell cat ignored-migrations)
 ACTIVE_DIRS := $(filter-out $(IGNORED_DIRS),$(MIG_DIRS))
 
-.PHONY: all build clean cmd sharness test test_go test_13_to_14 test_14_to_15
+.PHONY: all build clean cmd sharness test test_go test_14_to_15 test_15_to_16
 
 all: build
 
@@ -26,7 +26,7 @@ fs-repo-migrations/fs-repo-migrations:
 sharness:
 	make -C sharness
 
-test: test_go sharness test_13_to_14 test_14_to_15
+test: test_go test_14_to_15 test_15_to_16 sharness
 
 clean: $(subst fs-repo,clean.fs-repo,$(ACTIVE_DIRS))
 	@make -C sharness clean
@@ -52,3 +52,6 @@ test_13_to_14:
 
 test_14_to_15:
 	@cd fs-repo-14-to-15/not-sharness && ./test.sh
+
+test_15_to_16:
+	@cd fs-repo-15-to-16/test-e2e && ./test.sh

README.md

@@ -50,7 +50,8 @@ Here is the table showing which repo version corresponds to which Kubo version:
 |                12 | 0.12.0 - 0.17.0  |
 |                13 | 0.18.0 - 0.20.0  |
 |                14 | 0.21.0 - 0.22.0  |
-|                15 | 0.23.0 - current |
+|                15 | 0.23.0 - 0.29.0  |
+|                16 | 0.30.0 - current |
 
 ### How to Run Migrations
 

fs-repo-15-to-16/Makefile

@@ -0,0 +1,10 @@
+.PHONY: build clean
+
+build:
+	go build -mod=vendor
+
+clean:
+	go clean
+
+test:
+	@cd test-e2e && ./test.sh

fs-repo-15-to-16/atomicfile/atomicfile.go

@@ -0,0 +1,59 @@
+// Package atomicfile provides the ability to write a file with an eventual
+// rename on Close (using os.Rename). This allows for a file to always be in a
+// consistent state and never represent an in-progress write.
+//
+// NOTE: `os.Rename` may not be atomic on your operating system.
+package atomicfile
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+// File behaves like os.File, but does an atomic rename operation at Close.
+type File struct {
+	*os.File
+	path string
+}
+
+// New creates a new temporary file that will replace the file at the given
+// path when Closed.
+func New(path string, mode os.FileMode) (*File, error) {
+	f, err := ioutil.TempFile(filepath.Dir(path), filepath.Base(path))
+	if err != nil {
+		return nil, err
+	}
+	if err := os.Chmod(f.Name(), mode); err != nil {
+		f.Close()
+		os.Remove(f.Name())
+		return nil, err
+	}
+	return &File{File: f, path: path}, nil
+}
+
+// Close the file replacing the configured file.
+func (f *File) Close() error {
+	if err := f.File.Close(); err != nil {
+		os.Remove(f.File.Name())
+		return err
+	}
+	if err := os.Rename(f.Name(), f.path); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Abort closes the file and removes it instead of replacing the configured
+// file. This is useful if after starting to write to the file you decide you
+// don't want it anymore.
+func (f *File) Abort() error {
+	if err := f.File.Close(); err != nil {
+		os.Remove(f.Name())
+		return err
+	}
+	if err := os.Remove(f.Name()); err != nil {
+		return err
+	}
+	return nil
+}

fs-repo-15-to-16/go.mod

@@ -0,0 +1,5 @@
+module github.com/ipfs/fs-repo-migrations/fs-repo-15-to-16
+
+go 1.22
+
+require github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea

fs-repo-15-to-16/go.sum

@@ -0,0 +1,2 @@
+github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea h1:lgfk2PMrJI3bh8FflcBTXyNi3rPLqa75J7KcoUfRJmc=
+github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea/go.mod h1:fADeaHKxwS+SKhc52rsL0P1MUcnyK31a9AcaG0KcfY8=

fs-repo-15-to-16/main.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	mg15 "github.com/ipfs/fs-repo-migrations/fs-repo-15-to-16/migration"
+	migrate "github.com/ipfs/fs-repo-migrations/tools/go-migrate"
+)
+
+func main() {
+	m := mg15.Migration{}
+	migrate.Main(m)
+}

fs-repo-15-to-16/migration/migration.go

@@ -0,0 +1,231 @@
+// package mg15 contains the code to perform 15-16 repository migration in Kubo.
+// This handles the following:
+// - Add /webrtc-direct listener if preexisting /udp/ /quic-v1 exists
+package mg15
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+
+	migrate "github.com/ipfs/fs-repo-migrations/tools/go-migrate"
+	mfsr "github.com/ipfs/fs-repo-migrations/tools/mfsr"
+	lock "github.com/ipfs/fs-repo-migrations/tools/repolock"
+	log "github.com/ipfs/fs-repo-migrations/tools/stump"
+
+	"github.com/ipfs/fs-repo-migrations/fs-repo-15-to-16/atomicfile"
+)
+
+const backupSuffix = ".15-to-16.bak"
+
+// Migration implements the migration described above.
+type Migration struct{}
+
+// Versions returns the current version string for this migration.
+func (m Migration) Versions() string {
+	return "15-to-16"
+}
+
+// Reversible returns true, as we keep old config around
+func (m Migration) Reversible() bool {
+	return true
+}
+
+// Apply update the config.
+func (m Migration) Apply(opts migrate.Options) error {
+	log.Verbose = opts.Verbose
+	log.Log("applying %s repo migration", m.Versions())
+
+	log.VLog("locking repo at %q", opts.Path)
+	lk, err := lock.Lock2(opts.Path)
+	if err != nil {
+		return err
+	}
+	defer lk.Close()
+
+	repo := mfsr.RepoPath(opts.Path)
+
+	log.VLog("  - verifying version is '15'")
+	if err := repo.CheckVersion("15"); err != nil {
+		return err
+	}
+
+	log.Log("> Upgrading config to new format")
+
+	path := filepath.Join(opts.Path, "config")
+	in, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+
+	// make backup
+	backup, err := atomicfile.New(path+backupSuffix, 0600)
+	if err != nil {
+		return err
+	}
+	if _, err := backup.ReadFrom(in); err != nil {
+		panicOnError(backup.Abort())
+		return err
+	}
+	if _, err := in.Seek(0, io.SeekStart); err != nil {
+		panicOnError(backup.Abort())
+		return err
+	}
+
+	// Create a temp file to write the output to on success
+	out, err := atomicfile.New(path, 0600)
+	if err != nil {
+		panicOnError(backup.Abort())
+		panicOnError(in.Close())
+		return err
+	}
+
+	if err := convert(in, out); err != nil {
+		panicOnError(out.Abort())
+		panicOnError(backup.Abort())
+		panicOnError(in.Close())
+		return err
+	}
+
+	if err := in.Close(); err != nil {
+		panicOnError(out.Abort())
+		panicOnError(backup.Abort())
+	}
+
+	if err := repo.WriteVersion("16"); err != nil {
+		log.Error("failed to update version file to 16")
+		// There was an error so abort writing the output and clean up temp file
+		panicOnError(out.Abort())
+		panicOnError(backup.Abort())
+		return err
+	} else {
+		// Write the output and clean up temp file
+		panicOnError(out.Close())
+		panicOnError(backup.Close())
+	}
+
+	log.Log("updated version file")
+
+	log.Log("Migration 15 to 16 succeeded")
+	return nil
+}
+
+// panicOnError is reserved for checks we can't solve transactionally if an error occurs
+func panicOnError(e error) {
+	if e != nil {
+		panic(fmt.Errorf("error can't be dealt with transactionally: %w", e))
+	}
+}
+
+func (m Migration) Revert(opts migrate.Options) error {
+	log.Verbose = opts.Verbose
+	log.Log("reverting migration")
+	lk, err := lock.Lock2(opts.Path)
+	if err != nil {
+		return err
+	}
+	defer lk.Close()
+
+	repo := mfsr.RepoPath(opts.Path)
+	if err := repo.CheckVersion("16"); err != nil {
+		return err
+	}
+
+	cfg := filepath.Join(opts.Path, "config")
+	if err := os.Rename(cfg+backupSuffix, cfg); err != nil {
+		return err
+	}
+
+	if err := repo.WriteVersion("15"); err != nil {
+		return err
+	}
+	if opts.Verbose {
+		log.Log("lowered version number to 15")
+	}
+
+	return nil
+}
+
+var quicRegex = regexp.MustCompilePOSIX("/quic-v1$")
+
+// convert converts the config from one version to another
+func convert(in io.Reader, out io.Writer) error {
+	confMap := make(map[string]any)
+	if err := json.NewDecoder(in).Decode(&confMap); err != nil {
+		return err
+	}
+
+	// Append /webrtc-direct listener if /udp/../quic-v1 is present in any of .Addresses fields
+	if err := func() error {
+		a, ok := confMap["Addresses"]
+		if !ok {
+			return nil
+		}
+		addresses, ok := a.(map[string]any)
+		if !ok {
+			fmt.Printf("invalid type for .Addresses got %T expected json map; skipping .Addresses\n", a)
+			return nil
+		}
+
+		for _, addressToRemove := range [...]string{"Swarm", "Announce", "AppendAnnounce", "NoAnnounce"} {
+			s, ok := addresses[addressToRemove]
+			if !ok {
+				continue
+			}
+
+			swarm, ok := s.([]interface{})
+			if !ok {
+				fmt.Printf("invalid type for .Addresses.%s got %T expected json array; skipping .Addresses.%s\n", addressToRemove, s, addressToRemove)
+				continue
+			}
+
+			var newSwarm []interface{}
+			uniq := map[string]struct{}{}
+			for _, v := range swarm {
+				if addr, ok := v.(string); ok {
+
+					// if /quic-v1, add /webrtc-direct under the same port
+					if quicRegex.MatchString(addr) {
+						newAddr := quicRegex.ReplaceAllString(addr, "/webrtc-direct")
+
+						if _, ok := uniq[newAddr]; ok {
+							continue
+						}
+						uniq[newAddr] = struct{}{}
+
+						newSwarm = append(newSwarm, newAddr)
+					}
+
+					// keep original addr
+					if _, ok := uniq[addr]; ok {
+						continue
+					}
+					uniq[addr] = struct{}{}
+
+					newSwarm = append(newSwarm, addr)
+					continue
+				}
+				newSwarm = append(newSwarm, v)
+			}
+			addresses[addressToRemove] = newSwarm
+		}
+		return nil
+	}(); err != nil {
+		return err
+	}
+
+	// Save new config
+	fixed, err := json.MarshalIndent(confMap, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	if _, err := out.Write(fixed); err != nil {
+		return err
+	}
+	_, err = out.Write([]byte("\n"))
+	return err
+}

fs-repo-15-to-16/test-e2e/.gitignore

@@ -0,0 +1 @@
+repotest