From 26ea414cf2fcc70101d929cb42b180a77825de31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Wed, 27 Mar 2024 17:17:49 -0400 Subject: [PATCH 1/8] incusd/instance/qemu: Set auto-converge on all migrations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- internal/server/instance/drivers/driver_qemu.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/internal/server/instance/drivers/driver_qemu.go b/internal/server/instance/drivers/driver_qemu.go index 27b68bd1ea6..4af3faaf9ab 100644 --- a/internal/server/instance/drivers/driver_qemu.go +++ b/internal/server/instance/drivers/driver_qemu.go @@ -6687,6 +6687,17 @@ func (d *qemu) migrateSendLive(pool storagePools.Pool, clusterMoveSourceName str defer revert.Fail() // Run the revert fail before the earlier defers. d.logger.Debug("Setup temporary migration storage snapshot") + } else { + // Still set some options for shared storage. + capabilities := map[string]bool{ + // Automatically throttle down the guest to speed up convergence of RAM migration. + "auto-converge": true, + } + + err = monitor.MigrateSetCapabilities(capabilities) + if err != nil { + return fmt.Errorf("Failed setting migration capabilities: %w", err) + } } // Perform storage transfer while instance is still running. From e134d99072870259ea7727e1eb8b2b70cafdfed4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Wed, 27 Mar 2024 17:27:59 -0400 Subject: [PATCH 2/8] incusd/device/disk: Allow hotplug of disks on live-migratable VMs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- internal/server/device/disk.go | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/internal/server/device/disk.go b/internal/server/device/disk.go index 6d99644fc0b..e7cfcd9d130 100644 --- a/internal/server/device/disk.go +++ b/internal/server/device/disk.go @@ -121,18 +121,7 @@ func (d *disk) sourceIsCeph() bool { // CanHotPlug returns whether the device can be managed whilst the instance is running. func (d *disk) CanHotPlug() bool { - // Containers support hot-plugging all disk types. - if d.inst.Type() == instancetype.Container { - return true - } - - // Only VirtioFS works with path hotplug. - // As migration.stateful turns off VirtioFS, this also turns off hotplugging of paths. - if util.IsTrue(d.inst.ExpandedConfig()["migration.stateful"]) { - return false - } - - // Block disks can be hot-plugged into VMs. + // All disks can be hot-plugged. return true } From 19e428ced76907c21fd2eb19f8e34a848b09fa7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Wed, 27 Mar 2024 17:33:18 -0400 Subject: [PATCH 3/8] incusd/device/disk: Remove bad comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- internal/server/device/disk.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/server/device/disk.go b/internal/server/device/disk.go index e7cfcd9d130..3d9c1258200 100644 --- a/internal/server/device/disk.go +++ b/internal/server/device/disk.go @@ -125,7 +125,6 @@ func (d *disk) CanHotPlug() bool { return true } -// validateConfig checks the supplied config for correctness. // isRequired indicates whether the supplied device config requires this device to start OK. func (d *disk) isRequired(devConfig deviceConfig.Device) bool { // Defaults to required. From 356c6842d99b31a4da6083287ae579fe3b73bc64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Wed, 27 Mar 2024 17:49:42 -0400 Subject: [PATCH 4/8] incusd/device/disk: Check for VM live-migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- internal/server/device/disk.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/internal/server/device/disk.go b/internal/server/device/disk.go index 3d9c1258200..980345775d3 100644 --- a/internal/server/device/disk.go +++ b/internal/server/device/disk.go @@ -456,6 +456,25 @@ func (d *disk) validateConfig(instConf instance.ConfigReader) error { } } + // Restrict disks allowed when live-migratable. + if instConf.Type() == instancetype.VM && util.IsTrue(instConf.ExpandedConfig()["migration.stateful"]) { + if d.config["path"] != "" && d.config["path"] != "/" { + return fmt.Errorf("Shared filesystem are incompatible with migration.stateful=true") + } + + if d.config["pool"] == "" { + return fmt.Errorf("Only Incus-managed disks are allowed with migration.stateful=true") + } + + if d.config["io.bus"] == "nvme" { + return fmt.Errorf("NVME disks aren't supported with migration.stateful=true") + } + + if d.config["path"] != "/" && d.pool != nil && !d.pool.Driver().Info().Remote { + return fmt.Errorf("Only additional disks coming from a shared storage pool are supported with migration.stateful=true") + } + } + return nil } From 72a06a4f33a0a693533f4a5799b4e1c1029501ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Wed, 27 Mar 2024 17:56:09 -0400 Subject: [PATCH 5/8] incusd/instance: Add ID to ConfigReader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- internal/server/instance/instance_interface.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/server/instance/instance_interface.go b/internal/server/instance/instance_interface.go index 6046e13b498..e675a7df972 100644 --- a/internal/server/instance/instance_interface.go +++ b/internal/server/instance/instance_interface.go @@ -62,6 +62,8 @@ type ConfigReader interface { Project() api.Project Type() instancetype.Type Architecture() int + ID() int + ExpandedConfig() map[string]string ExpandedDevices() deviceConfig.Devices LocalConfig() map[string]string @@ -127,7 +129,6 @@ type Instance interface { OnHook(hookName string, args map[string]string) error // Properties. - ID() int Location() string Name() string CloudInitID() string From 6499313e75f4539123be2a9b8cead15022ea6140 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Wed, 27 Mar 2024 17:56:29 -0400 Subject: [PATCH 6/8] incusd/device/disk: Allow external disk live-migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- internal/server/device/disk.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/server/device/disk.go b/internal/server/device/disk.go index 980345775d3..f96e9b8e307 100644 --- a/internal/server/device/disk.go +++ b/internal/server/device/disk.go @@ -387,7 +387,7 @@ func (d *disk) validateConfig(instConf instance.ConfigReader) error { return fmt.Errorf("Failed checking if custom volume is exclusively attached to another instance: %w", err) } - if remoteInstance != nil { + if remoteInstance != nil && remoteInstance.ID != instConf.ID() { return fmt.Errorf("Custom volume is already attached to an instance on a different node") } From 5c8e099216a96b4b5a08e70f932731091032f0a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Wed, 27 Mar 2024 18:13:50 -0400 Subject: [PATCH 7/8] incusd/storage/lvmcluster: Handle custom volumes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- internal/server/storage/drivers/driver_lvm_volumes.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/server/storage/drivers/driver_lvm_volumes.go b/internal/server/storage/drivers/driver_lvm_volumes.go index a98fecb0f55..53d583342e8 100644 --- a/internal/server/storage/drivers/driver_lvm_volumes.go +++ b/internal/server/storage/drivers/driver_lvm_volumes.go @@ -168,7 +168,7 @@ func (d *lvm) CreateVolumeFromMigration(vol Volume, conn io.ReadWriteCloser, vol } // Mark the volume for shared locking during live migration. - if vol.volType == VolumeTypeVM { + if vol.volType == VolumeTypeVM || vol.IsCustomBlock() { volDevPath := d.lvmDevPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.Name()) _, err := subprocess.RunCommand("lvchange", "--activate", "sy", "--ignoreactivationskip", volDevPath) if err != nil { @@ -870,7 +870,7 @@ func (d *lvm) RenameVolume(vol Volume, newVolName string, op *operations.Operati func (d *lvm) MigrateVolume(vol Volume, conn io.ReadWriteCloser, volSrcArgs *migration.VolumeSourceArgs, op *operations.Operation) error { if d.clustered && volSrcArgs.ClusterMove { // Mark the volume for shared locking during live migration. - if vol.volType == VolumeTypeVM { + if vol.volType == VolumeTypeVM || vol.IsCustomBlock() { // Block volume. volDevPath := d.lvmDevPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.Name()) _, err := subprocess.RunCommand("lvchange", "--activate", "sy", "--ignoreactivationskip", volDevPath) From 67777f96337938374fbc208d9fa9beed5e227346 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Wed, 27 Mar 2024 21:34:29 -0400 Subject: [PATCH 8/8] incusd/instance/qemu: Support live-migration of instances with extra disks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #642 Signed-off-by: Stéphane Graber --- .../server/instance/drivers/driver_qemu.go | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/internal/server/instance/drivers/driver_qemu.go b/internal/server/instance/drivers/driver_qemu.go index 4af3faaf9ab..757ea8bcc22 100644 --- a/internal/server/instance/drivers/driver_qemu.go +++ b/internal/server/instance/drivers/driver_qemu.go @@ -6711,6 +6711,38 @@ func (d *qemu) migrateSendLive(pool storagePools.Pool, clusterMoveSourceName str return err } + // Derive the effective storage project name from the instance config's project. + storageProjectName, err := project.StorageVolumeProject(d.state.DB.Cluster, d.project.Name, db.StoragePoolVolumeTypeCustom) + if err != nil { + return err + } + + // Notify the shared disks that they're going to be accessed from another system. + for _, dev := range d.expandedDevices.Sorted() { + if dev.Config["type"] != "disk" || dev.Config["path"] == "/" { + continue + } + + // Load the pool for the disk. + diskPool, err := storagePools.LoadByName(d.state, dev.Config["pool"]) + if err != nil { + return fmt.Errorf("Failed loading storage pool: %w", err) + } + + // Setup the volume entry. + extraSourceArgs := &localMigration.VolumeSourceArgs{ + ClusterMove: true, + } + + vol := diskPool.GetVolume(storageDrivers.VolumeTypeCustom, storageDrivers.ContentTypeBlock, project.StorageVolume(storageProjectName, dev.Config["source"]), nil) + + // Call MigrateVolume on the source. + err = diskPool.Driver().MigrateVolume(vol, nil, extraSourceArgs, nil) + if err != nil { + return fmt.Errorf("Failed to prepare device %q for migration: %w", dev.Name, err) + } + } + // Non-shared storage snapshot transfer. if !sharedStorage { listener, err := net.Listen("unix", "") @@ -7191,6 +7223,38 @@ func (d *qemu) MigrateReceive(args instance.MigrateReceiveArgs) error { return fmt.Errorf("Failed creating instance on target: %w", err) } + // Derive the effective storage project name from the instance config's project. + storageProjectName, err := project.StorageVolumeProject(d.state.DB.Cluster, d.project.Name, db.StoragePoolVolumeTypeCustom) + if err != nil { + return err + } + + // Notify the shared disks that they're going to be accessed from another system. + for _, dev := range d.expandedDevices.Sorted() { + if dev.Config["type"] != "disk" || dev.Config["path"] == "/" { + continue + } + + // Load the pool for the disk. + diskPool, err := storagePools.LoadByName(d.state, dev.Config["pool"]) + if err != nil { + return fmt.Errorf("Failed loading storage pool: %w", err) + } + + // Setup the volume entry. + extraTargetArgs := localMigration.VolumeTargetArgs{ + ClusterMoveSourceName: args.ClusterMoveSourceName, + } + + vol := diskPool.GetVolume(storageDrivers.VolumeTypeCustom, storageDrivers.ContentTypeBlock, project.StorageVolume(storageProjectName, dev.Config["source"]), nil) + + // Call MigrateVolume on the source. + err = diskPool.Driver().CreateVolumeFromMigration(vol, nil, extraTargetArgs, nil, nil) + if err != nil { + return fmt.Errorf("Failed to prepare device %q for migration: %w", dev.Name, err) + } + } + // Only delete all instance volumes on error if the pool volume creation has succeeded to // avoid deleting an existing conflicting volume. isRemoteClusterMove := args.ClusterMoveSourceName != "" && poolInfo.Remote