Introduce a runtime directory

cmd/incusd/daemon.go

@@ -1521,15 +1521,15 @@ func (d *Daemon) init() error {
 
 		// Setup seccomp handler
 		if d.os.SeccompListener {
-			seccompServer, err := seccomp.NewSeccompServer(d.State(), internalUtil.VarPath("seccomp.socket"), func(pid int32, state *state.State) (seccomp.Instance, error) {
+			seccompServer, err := seccomp.NewSeccompServer(d.State(), internalUtil.RunPath("seccomp.socket"), func(pid int32, state *state.State) (seccomp.Instance, error) {
 				return findContainerForPid(pid, state)
 			})
 			if err != nil {
 				return err
 			}
 
 			d.seccomp = seccompServer
-			logger.Info("Started seccomp handler", logger.Ctx{"path": internalUtil.VarPath("seccomp.socket")})
+			logger.Info("Started seccomp handler", logger.Ctx{"path": internalUtil.RunPath("seccomp.socket")})
 		}
 
 		// Read the trusted certificates

cmd/incusd/instance_logs.go

@@ -94,7 +94,6 @@ var instanceExecOutputsCmd = APIEndpoint{
 //	            type: string
 //	          example: |-
 //	            [
-//	              "/1.0/instances/foo/logs/lxc.conf",
 //	              "/1.0/instances/foo/logs/lxc.log"
 //	            ]
 //	  "403":
@@ -670,9 +669,7 @@ func validLogFileName(fname string) bool {
 	 * to deal with any escaping or whatever.
 	 */
 	return fname == "lxc.log" ||
-		fname == "lxc.conf" ||
 		fname == "qemu.log" ||
-		fname == "qemu.conf" ||
 		strings.HasPrefix(fname, "migration_") ||
 		strings.HasPrefix(fname, "snapshot_")
 }

cmd/incusd/patches.go

@@ -17,6 +17,7 @@ import (
 	"github.com/lxc/incus/internal/server/db"
 	dbCluster "github.com/lxc/incus/internal/server/db/cluster"
 	"github.com/lxc/incus/internal/server/db/query"
+	"github.com/lxc/incus/internal/server/instance"
 	"github.com/lxc/incus/internal/server/instance/instancetype"
 	"github.com/lxc/incus/internal/server/network"
 	"github.com/lxc/incus/internal/server/node"
@@ -81,6 +82,7 @@ var patches = []patch{
 	{name: "snapshots_rename", stage: patchPreDaemonStorage, run: patchSnapshotsRename},
 	{name: "storage_zfs_unset_invalid_block_settings", stage: patchPostDaemonStorage, run: patchStorageZfsUnsetInvalidBlockSettings},
 	{name: "storage_zfs_unset_invalid_block_settings_v2", stage: patchPostDaemonStorage, run: patchStorageZfsUnsetInvalidBlockSettingsV2},
+	{name: "runtime_directory", stage: patchPostDaemonStorage, run: patchRuntimeDirectory},
 }
 
 type patch struct {
@@ -1186,4 +1188,60 @@ func patchStorageZfsUnsetInvalidBlockSettingsV2(_ string, d *Daemon) error {
 	return nil
 }
 
+func patchRuntimeDirectory(name string, d *Daemon) error {
+	s := d.State()
+
+	// Get the list of local instances.
+	instances, err := instance.LoadNodeAll(s, instancetype.Any)
+	if err != nil {
+		return fmt.Errorf("Failed loading local instances: %w", err)
+	}
+
+	for _, inst := range instances {
+		if !util.PathExists(inst.LogPath()) {
+			continue
+		}
+
+		err = os.MkdirAll(inst.RunPath(), 0700)
+		if err != nil && !os.IsExist(err) {
+			return fmt.Errorf("Failed to create runtime directory for %q in project %q: %w", inst.Name(), inst.Project(), err)
+		}
+
+		files, err := os.ReadDir(inst.LogPath())
+		if err != nil {
+			return fmt.Errorf("Failed to list log files for %q in project %q: %w", inst.Name(), inst.Project(), err)
+		}
+
+		for _, fi := range files {
+			name := fi.Name()
+
+			// Keep actual log files where they are.
+			if strings.HasSuffix(name, ".log") || strings.HasSuffix(name, ".log.old") || strings.HasSuffix(name, ".gz") {
+				continue
+			}
+
+			info, err := fi.Info()
+			if err != nil {
+				return fmt.Errorf("Failed getting file info on %q for instance %q in project %q: %w", name, inst.Name(), inst.Project(), err)
+			}
+
+			if info.Mode().IsRegular() {
+				// Relocate the file.
+				_, err := subprocess.RunCommand("mv", filepath.Join(inst.LogPath(), name), inst.RunPath())
+				if err != nil {
+					return fmt.Errorf("Failed to relocate runtime file %q for instance %q in project %q: %w", name, inst.Name(), inst.Project(), err)
+				}
+			} else {
+				// For pipes and sockets, we need to use a symlink to avoid breaking the listener.
+				err := os.Symlink(filepath.Join(inst.LogPath(), name), filepath.Join(inst.RunPath(), name))
+				if err != nil {
+					return fmt.Errorf("Failed to symlink runtime file %q for instance %q in project %q: %w", name, inst.Name(), inst.Project(), err)
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
 // Patches end here

doc/reference/instance_options.md

@@ -255,7 +255,7 @@ Therefore, you should avoid setting any of these keys.
 
 For VM instances, Incus configures QEMU through a configuration file that is passed to QEMU with the `-readconfig` command-line option.
 This configuration file is generated for each instance before boot.
-It can be found at `/var/log/incus/<instance_name>/qemu.conf`.
+It can be found at `/run/incus/<instance_name>/qemu.conf`.
 
 The default configuration works fine for Incus' most common use case: modern UEFI guests with VirtIO devices.
 In some situations, however, you might need to override the generated configuration.

doc/rest-api.yaml

@@ -9055,7 +9055,6 @@ paths:
                                 description: List of endpoints
                                 example: |-
                                     [
-                                      "/1.0/instances/foo/logs/lxc.conf",
                                       "/1.0/instances/foo/logs/lxc.log"
                                     ]
                                 items:

internal/server/apparmor/instance.go

@@ -23,6 +23,7 @@ type instance interface {
 	ExpandedConfig() map[string]string
 	Type() instancetype.Type
 	LogPath() string
+	RunPath() string
 	Path() string
 	DevicesPath() string
 }
@@ -211,6 +212,7 @@ func instanceProfile(sysOS *sys.OS, inst instance, extraBinaries []string) (stri
 			"extra_binaries": extraBinaries,
 			"libraryPath":    strings.Split(os.Getenv("LD_LIBRARY_PATH"), ":"),
 			"logPath":        inst.LogPath(),
+			"runPath":        inst.RunPath(),
 			"name":           InstanceProfileName(inst),
 			"path":           path,
 			"raw":            rawContent,

internal/server/apparmor/instance_qemu.go

@@ -63,6 +63,7 @@ profile "{{ .name }}" flags=(attach_disconnected,mediate_deleted) {
 
   # Instance specific paths
   {{ .logPath }}/** rwk,
+  {{ .runPath }}/** rwk,
   {{ .path }}/** rwk,
   {{ .devicesPath }}/** rwk,
 

internal/server/instance/drivers/driver_common.go

@@ -386,6 +386,12 @@ func (d *common) LogPath() string {
 	return internalUtil.LogPath(name)
 }
 
+// RunPath returns the instance's runtime path.
+func (d *common) RunPath() string {
+	name := project.Instance(d.project.Name, d.name)
+	return internalUtil.RunPath(name)
+}
+
 // Path returns the instance's path.
 func (d *common) Path() string {
 	return storagePools.InstancePath(d.dbType, d.project.Name, d.name, d.isSnapshot)

internal/server/instance/drivers/driver_lxc.go

@@ -1017,7 +1017,7 @@ func (d *lxc) initLXC(config bool) (*liblxc.Container, error) {
 		// System requirement errors are handled during policy generation instead of here
 		ok, err := seccomp.InstanceNeedsIntercept(d.state, d)
 		if err == nil && ok {
-			err = lxcSetConfigItem(cc, "lxc.seccomp.notify.proxy", fmt.Sprintf("unix:%s", internalUtil.VarPath("seccomp.socket")))
+			err = lxcSetConfigItem(cc, "lxc.seccomp.notify.proxy", fmt.Sprintf("unix:%s", internalUtil.RunPath("seccomp.socket")))
 			if err != nil {
 				return nil, err
 			}
@@ -2052,6 +2052,11 @@ func (d *lxc) startCommon() (string, []func() error, error) {
 		return "", nil, err
 	}
 
+	err = os.MkdirAll(d.RunPath(), 0700)
+	if err != nil {
+		return "", nil, err
+	}
+
 	err = os.MkdirAll(d.DevicesPath(), 0711)
 	if err != nil {
 		return "", nil, err
@@ -2285,7 +2290,7 @@ func (d *lxc) startCommon() (string, []func() error, error) {
 	}
 
 	// Generate the LXC config
-	configPath := filepath.Join(d.LogPath(), "lxc.conf")
+	configPath := filepath.Join(d.RunPath(), "lxc.conf")
 	err = cc.SaveConfigFile(configPath)
 	if err != nil {
 		_ = os.Remove(configPath)
@@ -3962,6 +3967,16 @@ func (d *lxc) Rename(newName string, applyTemplateTrigger bool) error {
 		}
 	}
 
+	// Rename the runtime path.
+	newFullName = project.Instance(d.Project().Name, d.Name())
+	_ = os.RemoveAll(internalUtil.RunPath(newFullName))
+	if util.PathExists(d.RunPath()) {
+		err := os.Rename(d.RunPath(), internalUtil.RunPath(newFullName))
+		if err != nil {
+			d.logger.Error("Failed renaming instance", ctxMap)
+			return fmt.Errorf("Failed renaming instance: %w", err)
+		}
+	}
 	revert := revert.New()
 	defer revert.Fail()
 
@@ -6709,13 +6724,13 @@ func (d *lxc) FileSFTPConn() (net.Conn, error) {
 	defer spawnUnlock()
 
 	// Create any missing directories in case the instance has never been started before.
-	err = os.MkdirAll(d.LogPath(), 0700)
+	err = os.MkdirAll(d.RunPath(), 0700)
 	if err != nil {
 		return nil, err
 	}
 
 	// Trickery to handle paths > 108 chars.
-	dirFile, err := os.Open(d.LogPath())
+	dirFile, err := os.Open(d.RunPath())
 	if err != nil {
 		return nil, err
 	}
@@ -6728,7 +6743,7 @@ func (d *lxc) FileSFTPConn() (net.Conn, error) {
 	}
 
 	// Attempt to connect on existing socket.
-	forkfilePath := filepath.Join(d.LogPath(), "forkfile.sock")
+	forkfilePath := filepath.Join(d.RunPath(), "forkfile.sock")
 	forkfileConn, err := net.DialUnix("unix", nil, forkfileAddr)
 	if err == nil {
 		// Found an existing server.
@@ -6871,7 +6886,7 @@ func (d *lxc) FileSFTPConn() (net.Conn, error) {
 		}
 
 		// Write PID file.
-		pidFile := filepath.Join(d.LogPath(), "forkfile.pid")
+		pidFile := filepath.Join(d.RunPath(), "forkfile.pid")
 		err = os.WriteFile(pidFile, []byte(fmt.Sprintf("%d\n", forkfile.Process.Pid)), 0600)
 		if err != nil {
 			chReady <- fmt.Errorf("Failed to write forkfile PID: %w", err)
@@ -6951,7 +6966,7 @@ func (d *lxc) stopForkfile(force bool) {
 		unlock()
 	}()
 
-	content, err := os.ReadFile(filepath.Join(d.LogPath(), "forkfile.pid"))
+	content, err := os.ReadFile(filepath.Join(d.RunPath(), "forkfile.pid"))
 	if err != nil {
 		return
 	}
@@ -6985,7 +7000,7 @@ func (d *lxc) Console(protocol string) (*os.File, chan error, error) {
 		"forkconsole",
 		project.Instance(d.Project().Name, d.Name()),
 		d.state.OS.LxcPath,
-		filepath.Join(d.LogPath(), "lxc.conf"),
+		filepath.Join(d.RunPath(), "lxc.conf"),
 		"tty=0",
 		"escape=-1"}
 
@@ -7063,7 +7078,7 @@ func (d *lxc) ConsoleLog(opts liblxc.ConsoleLogOptions) (string, error) {
 // Exec executes a command inside the instance.
 func (d *lxc) Exec(req api.InstanceExecPost, stdin *os.File, stdout *os.File, stderr *os.File) (instance.Cmd, error) {
 	// Generate the LXC config if missing.
-	configPath := filepath.Join(d.LogPath(), "lxc.conf")
+	configPath := filepath.Join(d.RunPath(), "lxc.conf")
 	if !util.PathExists(configPath) {
 		cc, err := d.initLXC(true)
 		if err != nil {
@@ -7100,7 +7115,7 @@ func (d *lxc) Exec(req api.InstanceExecPost, stdin *os.File, stdout *os.File, st
 		"forkexec",
 		cname,
 		d.state.OS.LxcPath,
-		filepath.Join(d.LogPath(), "lxc.conf"),
+		filepath.Join(d.RunPath(), "lxc.conf"),
 		req.Cwd,
 		fmt.Sprintf("%d", req.User),
 		fmt.Sprintf("%d", req.Group),
@@ -7592,7 +7607,7 @@ func (d *lxc) insertMountGo(source, target, fstype string, flags int, mntnsPID i
 
 func (d *lxc) insertMountLXC(source, target, fstype string, flags int) error {
 	cname := project.Instance(d.Project().Name, d.Name())
-	configPath := filepath.Join(d.LogPath(), "lxc.conf")
+	configPath := filepath.Join(d.RunPath(), "lxc.conf")
 	if fstype == "" {
 		fstype = "none"
 	}
@@ -7688,7 +7703,7 @@ func (d *lxc) removeMount(mount string) error {
 	}
 
 	if d.state.OS.LXCFeatures["mount_injection_file"] {
-		configPath := filepath.Join(d.LogPath(), "lxc.conf")
+		configPath := filepath.Join(d.RunPath(), "lxc.conf")
 		cname := project.Instance(d.Project().Name, d.Name())
 
 		if !strings.HasPrefix(mount, "/") {

internal/server/instance/drivers/driver_qemu.go

@@ -582,8 +582,8 @@ func (d *qemu) configDriveMountPathClear() error {
 
 // configVirtiofsdPaths returns the path for the socket and PID file to use with config drive virtiofsd process.
 func (d *qemu) configVirtiofsdPaths() (string, string) {
-	sockPath := filepath.Join(d.LogPath(), "virtio-fs.config.sock")
-	pidPath := filepath.Join(d.LogPath(), "virtiofsd.pid")
+	sockPath := filepath.Join(d.RunPath(), "virtio-fs.config.sock")
+	pidPath := filepath.Join(d.RunPath(), "virtiofsd.pid")
 
 	return sockPath, pidPath
 }
@@ -1226,6 +1226,12 @@ func (d *qemu) start(stateful bool, op *operationlock.InstanceOperation) error {
 		return err
 	}
 
+	err = os.MkdirAll(d.RunPath(), 0700)
+	if err != nil {
+		op.Done(err)
+		return err
+	}
+
 	err = os.MkdirAll(d.DevicesPath(), 0711)
 	if err != nil {
 		op.Done(err)
@@ -2546,19 +2552,19 @@ func (d *qemu) deviceDetachNIC(deviceName string) error {
 }
 
 func (d *qemu) monitorPath() string {
-	return filepath.Join(d.LogPath(), "qemu.monitor")
+	return filepath.Join(d.RunPath(), "qemu.monitor")
 }
 
 func (d *qemu) nvramPath() string {
 	return filepath.Join(d.Path(), "qemu.nvram")
 }
 
 func (d *qemu) consolePath() string {
-	return filepath.Join(d.LogPath(), "qemu.console")
+	return filepath.Join(d.RunPath(), "qemu.console")
 }
 
 func (d *qemu) spicePath() string {
-	return filepath.Join(d.LogPath(), "qemu.spice")
+	return filepath.Join(d.RunPath(), "qemu.spice")
 }
 
 func (d *qemu) spiceCmdlineConfig() string {
@@ -3435,7 +3441,7 @@ func (d *qemu) generateQemuConfigFile(cpuInfo *cpuTopology, mountInfo *storagePo
 	cfg = qemuRawCfgOverride(cfg, d.expandedConfig)
 	// Write the config file to disk.
 	sb := qemuStringifyCfg(cfg...)
-	configPath := filepath.Join(d.LogPath(), "qemu.conf")
+	configPath := filepath.Join(d.RunPath(), "qemu.conf")
 	return configPath, monHooks, os.WriteFile(configPath, []byte(sb.String()), 0640)
 }
 
@@ -4589,7 +4595,7 @@ func (d *qemu) addVmgenDeviceConfig(cfg *[]cfgSection, guid string) error {
 
 // pidFilePath returns the path where the qemu process should write its PID.
 func (d *qemu) pidFilePath() string {
-	return filepath.Join(d.LogPath(), "qemu.pid")
+	return filepath.Join(d.RunPath(), "qemu.pid")
 }
 
 // pid gets the PID of the running qemu process. Returns 0 if PID file or process not found, and -1 if err non-nil.
@@ -5100,6 +5106,17 @@ func (d *qemu) Rename(newName string, applyTemplateTrigger bool) error {
 		}
 	}
 
+	// Rename the runtime path.
+	newFullName = project.Instance(d.Project().Name, d.Name())
+	_ = os.RemoveAll(internalUtil.RunPath(newFullName))
+	if util.PathExists(d.RunPath()) {
+		err := os.Rename(d.RunPath(), internalUtil.RunPath(newFullName))
+		if err != nil {
+			d.logger.Error("Failed renaming instance", ctxMap)
+			return err
+		}
+	}
+
 	revert := revert.New()
 	defer revert.Fail()
 

internal/server/instance/instance_interface.go

@@ -150,6 +150,7 @@ type Instance interface {
 	LogFilePath() string
 	ConsoleBufferLogPath() string
 	LogPath() string
+	RunPath() string
 	DevicesPath() string
 
 	// Storage.