Rework shared/idmap

.golangci.yml

@@ -72,9 +72,6 @@ linters-settings:
       # https://github.com/mgechev/revive/blob/2a1701aadbedfcc175cb92836a51407bec382652/RULES_DESCRIPTIONS.md#comment-spacings
       - name: comment-spacings
 
-      # https://github.com/mgechev/revive/blob/2a1701aadbedfcc175cb92836a51407bec382652/RULES_DESCRIPTIONS.md#confusing-results
-      - name: confusing-results
-
       # https://github.com/mgechev/revive/blob/2a1701aadbedfcc175cb92836a51407bec382652/RULES_DESCRIPTIONS.md#use-any
       - name: use-any
 

Makefile

@@ -306,9 +306,6 @@ ifeq ($(shell command -v shellcheck),)
 	echo "Please install shellcheck"
 	exit 1
 else
-ifneq "$(shell shellcheck --version | grep version: | cut -d ' ' -f2)" "0.8.0"
-	@echo "WARN: shellcheck version is not 0.8.0"
-endif
 endif
 ifeq ($(shell command -v flake8),)
 	echo "Please install flake8"

cmd/fuidshift/main_shift.go

@@ -62,8 +62,16 @@ func (c *cmdShift) Run(cmd *cobra.Command, args []string) error {
 
 	directory := args[0]
 
+	var skipper func(dir string, absPath string, fi os.FileInfo, newuid int64, newgid int64) error
+	if c.flagTestMode {
+		skipper = func(dir string, absPath string, fi os.FileInfo, newuid int64, newgid int64) error {
+			fmt.Printf("I would shift %q to %d %d\n", absPath, newuid, newgid)
+			return fmt.Errorf("dry run")
+		}
+	}
+
 	// Parse the maps
-	idmapSet := idmap.IdmapSet{}
+	idmapSet := &idmap.Set{}
 	for _, arg := range args[1:] {
 		var err error
 		idmapSet, err = idmapSet.Append(arg)
@@ -74,7 +82,7 @@ func (c *cmdShift) Run(cmd *cobra.Command, args []string) error {
 
 	// Reverse shifting
 	if c.flagReverse {
-		err := idmapSet.UidshiftFromContainer(directory, c.flagTestMode)
+		err := idmapSet.UnshiftPath(directory, skipper)
 		if err != nil {
 			return err
 		}
@@ -83,7 +91,7 @@ func (c *cmdShift) Run(cmd *cobra.Command, args []string) error {
 	}
 
 	// Normal shifting
-	err := idmapSet.UidshiftIntoContainer(directory, c.flagTestMode)
+	err := idmapSet.ShiftPath(directory, skipper)
 	if err != nil {
 		return err
 	}

cmd/incus-user/server.go

@@ -5,13 +5,15 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 
 	"github.com/lxc/incus/client"
 	"github.com/lxc/incus/internal/linux"
 	"github.com/lxc/incus/internal/revert"
 	internalUtil "github.com/lxc/incus/internal/util"
 	"github.com/lxc/incus/shared/api"
+	"github.com/lxc/incus/shared/idmap"
 	"github.com/lxc/incus/shared/subprocess"
 	localtls "github.com/lxc/incus/shared/tls"
 	"github.com/lxc/incus/shared/util"
@@ -254,11 +256,8 @@ func serverSetupUser(uid uint32) error {
 	}
 
 	// Setup default profile.
-	err = client.UseProject(projectName).UpdateProfile("default", api.ProfilePut{
+	req := api.ProfilePut{
 		Description: "Default Incus profile",
-		Config: map[string]string{
-			"raw.idmap": fmt.Sprintf("uid %s %s\ngid %s %s", pw[2], pw[2], pw[3], pw[3]),
-		},
 		Devices: map[string]map[string]string{
 			"root": {
 				"type": "disk",
@@ -271,7 +270,43 @@ func serverSetupUser(uid uint32) error {
 				"network": networkName,
 			},
 		},
-	}, "")
+	}
+
+	// Add uid/gid map if possible.
+	pwUID, err := strconv.ParseInt(pw[2], 10, 64)
+	if err != nil {
+		return err
+	}
+
+	pwGID, err := strconv.ParseInt(pw[3], 10, 64)
+	if err != nil {
+		return err
+	}
+
+	idmapset, err := idmap.NewSetFromSystem("", "root")
+	if err != nil && err != idmap.ErrSubidUnsupported {
+		return fmt.Errorf("Failed to load system idmap: %w", err)
+	}
+
+	idmapAllowed := true
+	if idmapset != nil {
+		entries := []idmap.Entry{
+			{IsUID: true, HostID: pwUID, MapRange: 1},
+			{IsGID: true, HostID: pwGID, MapRange: 1},
+		}
+
+		if !idmapset.Includes(&idmap.Set{Entries: entries}) {
+			idmapAllowed = false
+		}
+	}
+
+	if idmapAllowed {
+		req.Config = map[string]string{
+			"raw.idmap": fmt.Sprintf("uid %d %d\ngid %d %d", pwUID, pwUID, pwGID, pwGID),
+		}
+	}
+
+	err = client.UseProject(projectName).UpdateProfile("default", req, "")
 	if err != nil {
 		return fmt.Errorf("Unable to update the default profile: %w", err)
 	}

cmd/incusd/backup.go

@@ -123,10 +123,10 @@ func backupCreate(s *state.State, args db.InstanceBackup, sourceInst instance.In
 	revert.Add(func() { _ = os.Remove(target) })
 
 	// Get IDMap to unshift container as the tarball is created.
-	var idmap *idmap.IdmapSet
+	var idmapSet *idmap.Set
 	if sourceInst.Type() == instancetype.Container {
 		c := sourceInst.(instance.Container)
-		idmap, err = c.DiskIdmap()
+		idmapSet, err = c.DiskIdmap()
 		if err != nil {
 			return fmt.Errorf("Error getting container IDMAP: %w", err)
 		}
@@ -135,7 +135,7 @@ func backupCreate(s *state.State, args db.InstanceBackup, sourceInst instance.In
 	// Create the tarball.
 	tarPipeReader, tarPipeWriter := io.Pipe()
 	defer func() { _ = tarPipeWriter.Close() }() // Ensure that go routine below always ends.
-	tarWriter := instancewriter.NewInstanceTarWriter(tarPipeWriter, idmap)
+	tarWriter := instancewriter.NewInstanceTarWriter(tarPipeWriter, idmapSet)
 
 	// Setup tar writer go routine, with optional compression.
 	tarWriterRes := make(chan error)

cmd/incusd/daemon.go

@@ -954,12 +954,12 @@ func (d *Daemon) init() error {
 	 * so we don't need to bother with atomic.StoreInt32() when touching
 	 * VFS3Fscaps.
 	 */
-	d.os.VFS3Fscaps = idmap.SupportsVFS3Fscaps("")
+	d.os.VFS3Fscaps = idmap.SupportsVFS3FSCaps("")
 	if d.os.VFS3Fscaps {
-		idmap.VFS3Fscaps = idmap.VFS3FscapsSupported
+		idmap.VFS3FSCaps = idmap.VFS3FSCapsSupported
 		logger.Infof(" - unprivileged file capabilities: yes")
 	} else {
-		idmap.VFS3Fscaps = idmap.VFS3FscapsUnsupported
+		idmap.VFS3FSCaps = idmap.VFS3FSCapsUnsupported
 		logger.Infof(" - unprivileged file capabilities: no")
 	}
 

cmd/incusd/dev_incus.go

@@ -295,7 +295,7 @@ func hoistReq(f func(*Daemon, instance.Instance, http.ResponseWriter, *http.Requ
 
 		idmapset, err := c.CurrentIdmap()
 		if err == nil && idmapset != nil {
-			uid, _ := idmapset.ShiftIntoNs(0, 0)
+			uid, _ := idmapset.ShiftIntoNS(0, 0)
 			rootUID = uint32(uid)
 		}
 

cmd/incusd/instance_exec.go

@@ -203,7 +203,7 @@ func (s *execWs) Do(op *operations.Operation) error {
 			}
 
 			if idmapset != nil {
-				rootUID, rootGID = idmapset.ShiftIntoNs(0, 0)
+				rootUID, rootGID = idmapset.ShiftIntoNS(0, 0)
 			}
 
 			devptsFd, _ = c.DevptsFd()

cmd/incusd/instance_test.go

@@ -360,18 +360,18 @@ func (suite *containerTestSuite) TestContainer_findIdmap_isolated() {
 	map2, err := c2.(instance.Container).NextIdmap()
 	suite.Req.Nil(err)
 
-	host := suite.d.os.IdmapSet.Idmap[0]
+	host := suite.d.os.IdmapSet.Entries[0]
 
 	for i := 0; i < 2; i++ {
-		suite.Req.Equal(host.Hostid+65536, map1.Idmap[i].Hostid, "hostids don't match %d", i)
-		suite.Req.Equal(int64(0), map1.Idmap[i].Nsid, "nsid nonzero")
-		suite.Req.Equal(int64(65536), map1.Idmap[i].Maprange, "incorrect maprange")
+		suite.Req.Equal(host.HostID+65536, map1.Entries[i].HostID, "hostids don't match %d", i)
+		suite.Req.Equal(int64(0), map1.Entries[i].NSID, "nsid nonzero")
+		suite.Req.Equal(int64(65536), map1.Entries[i].MapRange, "incorrect maprange")
 	}
 
 	for i := 0; i < 2; i++ {
-		suite.Req.Equal(host.Hostid+65536*2, map2.Idmap[i].Hostid, "hostids don't match")
-		suite.Req.Equal(int64(0), map2.Idmap[i].Nsid, "nsid nonzero")
-		suite.Req.Equal(int64(65536), map2.Idmap[i].Maprange, "incorrect maprange")
+		suite.Req.Equal(host.HostID+65536*2, map2.Entries[i].HostID, "hostids don't match")
+		suite.Req.Equal(int64(0), map2.Entries[i].NSID, "nsid nonzero")
+		suite.Req.Equal(int64(65536), map2.Entries[i].MapRange, "incorrect maprange")
 	}
 }
 
@@ -403,18 +403,18 @@ func (suite *containerTestSuite) TestContainer_findIdmap_mixed() {
 	map2, err := c2.(instance.Container).NextIdmap()
 	suite.Req.Nil(err)
 
-	host := suite.d.os.IdmapSet.Idmap[0]
+	host := suite.d.os.IdmapSet.Entries[0]
 
 	for i := 0; i < 2; i++ {
-		suite.Req.Equal(host.Hostid, map1.Idmap[i].Hostid, "hostids don't match %d", i)
-		suite.Req.Equal(int64(0), map1.Idmap[i].Nsid, "nsid nonzero")
-		suite.Req.Equal(host.Maprange, map1.Idmap[i].Maprange, "incorrect maprange")
+		suite.Req.Equal(host.HostID, map1.Entries[i].HostID, "hostids don't match %d", i)
+		suite.Req.Equal(int64(0), map1.Entries[i].NSID, "nsid nonzero")
+		suite.Req.Equal(host.MapRange, map1.Entries[i].MapRange, "incorrect maprange")
 	}
 
 	for i := 0; i < 2; i++ {
-		suite.Req.Equal(host.Hostid+65536, map2.Idmap[i].Hostid, "hostids don't match")
-		suite.Req.Equal(int64(0), map2.Idmap[i].Nsid, "nsid nonzero")
-		suite.Req.Equal(int64(65536), map2.Idmap[i].Maprange, "incorrect maprange")
+		suite.Req.Equal(host.HostID+65536, map2.Entries[i].HostID, "hostids don't match")
+		suite.Req.Equal(int64(0), map2.Entries[i].NSID, "nsid nonzero")
+		suite.Req.Equal(int64(65536), map2.Entries[i].MapRange, "incorrect maprange")
 	}
 }
 
@@ -434,29 +434,29 @@ func (suite *containerTestSuite) TestContainer_findIdmap_raw() {
 	map1, err := c1.(instance.Container).NextIdmap()
 	suite.Req.Nil(err)
 
-	host := suite.d.os.IdmapSet.Idmap[0]
+	host := suite.d.os.IdmapSet.Entries[0]
 
 	for _, i := range []int{0, 3} {
-		suite.Req.Equal(host.Hostid, map1.Idmap[i].Hostid, "hostids don't match")
-		suite.Req.Equal(int64(0), map1.Idmap[i].Nsid, "nsid nonzero")
-		suite.Req.Equal(int64(1000), map1.Idmap[i].Maprange, "incorrect maprange")
+		suite.Req.Equal(host.HostID, map1.Entries[i].HostID, "hostids don't match")
+		suite.Req.Equal(int64(0), map1.Entries[i].NSID, "nsid nonzero")
+		suite.Req.Equal(int64(1000), map1.Entries[i].MapRange, "incorrect maprange")
 	}
 
 	for _, i := range []int{1, 4} {
-		suite.Req.Equal(int64(1000), map1.Idmap[i].Hostid, "hostids don't match")
-		suite.Req.Equal(int64(1000), map1.Idmap[i].Nsid, "invalid nsid")
-		suite.Req.Equal(int64(1), map1.Idmap[i].Maprange, "incorrect maprange")
+		suite.Req.Equal(int64(1000), map1.Entries[i].HostID, "hostids don't match")
+		suite.Req.Equal(int64(1000), map1.Entries[i].NSID, "invalid nsid")
+		suite.Req.Equal(int64(1), map1.Entries[i].MapRange, "incorrect maprange")
 	}
 
 	for _, i := range []int{2, 5} {
-		suite.Req.Equal(host.Hostid+1001, map1.Idmap[i].Hostid, "hostids don't match")
-		suite.Req.Equal(int64(1001), map1.Idmap[i].Nsid, "invalid nsid")
-		suite.Req.Equal(host.Maprange-1000-1, map1.Idmap[i].Maprange, "incorrect maprange")
+		suite.Req.Equal(host.HostID+1001, map1.Entries[i].HostID, "hostids don't match")
+		suite.Req.Equal(int64(1001), map1.Entries[i].NSID, "invalid nsid")
+		suite.Req.Equal(host.MapRange-1000-1, map1.Entries[i].MapRange, "incorrect maprange")
 	}
 }
 
 func (suite *containerTestSuite) TestContainer_findIdmap_maxed() {
-	maps := []*idmap.IdmapSet{}
+	maps := []*idmap.Set{}
 
 	for i := 0; i < 7; i++ {
 		c, op, _, err := instance.CreateInternal(suite.d.State(), db.InstanceArgs{
@@ -490,8 +490,8 @@ func (suite *containerTestSuite) TestContainer_findIdmap_maxed() {
 				continue
 			}
 
-			for _, e := range m2.Idmap {
-				suite.Req.False(m1.HostidsIntersect(e), "%d and %d's idmaps intersect %v %v", i, j, m1, m2)
+			for _, e := range m2.Entries {
+				suite.Req.False(m1.HostIDsIntersect(e), "%d and %d's idmaps intersect %v %v", i, j, m1, m2)
 			}
 		}
 	}

cmd/incusd/main_activateifneeded.go

@@ -91,11 +91,8 @@ func (c *cmdActivateifneeded) Run(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	// Load the idmap for unprivileged instances
-	d.os.IdmapSet, err = idmap.DefaultIdmapSet("", "")
-	if err != nil {
-		return err
-	}
+	// Set a non-nil IdmapSet to be able to load unprivileged instances
+	d.os.IdmapSet = &idmap.Set{}
 
 	// Look for auto-started or previously started instances
 	path = d.os.GlobalDatabasePath()

cmd/incusd/main_test.go

@@ -33,9 +33,9 @@ func mockStartDaemon() (*Daemon, error) {
 		return nil, err
 	}
 
-	d.os.IdmapSet = &idmap.IdmapSet{Idmap: []idmap.IdmapEntry{
-		{Isuid: true, Hostid: 100000, Nsid: 0, Maprange: 500000},
-		{Isgid: true, Hostid: 100000, Nsid: 0, Maprange: 500000},
+	d.os.IdmapSet = &idmap.Set{Entries: []idmap.Entry{
+		{IsUID: true, HostID: 100000, NSID: 0, MapRange: 500000},
+		{IsGID: true, HostID: 100000, NSID: 0, MapRange: 500000},
 	}}
 
 	return d, nil

cmd/incusd/migrate.go

@@ -198,7 +198,7 @@ type migrationSinkArgs struct {
 	// Instance specific fields
 	Instance              instance.Instance
 	InstanceOnly          bool
-	Idmap                 *idmap.IdmapSet
+	Idmap                 *idmap.Set
 	Live                  bool
 	Refresh               bool
 	ClusterMoveSourceName string

internal/instancewriter/instance_tar_writer.go

@@ -19,12 +19,12 @@ import (
 // InstanceTarWriter provides a TarWriter implementation that handles ID shifting and hardlink tracking.
 type InstanceTarWriter struct {
 	tarWriter *tar.Writer
-	idmapSet  *idmap.IdmapSet
+	idmapSet  *idmap.Set
 	linkMap   map[uint64]string
 }
 
 // NewInstanceTarWriter returns a ContainerTarWriter for the provided target Writer and id map.
-func NewInstanceTarWriter(writer io.Writer, idmapSet *idmap.IdmapSet) *InstanceTarWriter {
+func NewInstanceTarWriter(writer io.Writer, idmapSet *idmap.Set) *InstanceTarWriter {
 	ctw := new(InstanceTarWriter)
 	ctw.tarWriter = tar.NewWriter(writer)
 	ctw.idmapSet = idmapSet
@@ -92,7 +92,7 @@ func (ctw *InstanceTarWriter) WriteFile(name string, srcPath string, fi os.FileI
 
 	// Unshift the id under rootfs/ for unpriv containers.
 	if strings.HasPrefix(hdr.Name, "rootfs") && ctw.idmapSet != nil {
-		hUID, hGID := ctw.idmapSet.ShiftFromNs(int64(hdr.Uid), int64(hdr.Gid))
+		hUID, hGID := ctw.idmapSet.ShiftFromNS(int64(hdr.Uid), int64(hdr.Gid))
 		hdr.Uid = int(hUID)
 		hdr.Gid = int(hGID)
 		if hdr.Uid == -1 || hdr.Gid == -1 {