MCO-356: daemon/firstboot: Do a secondary in-place update if rpm-ostr…

…ee is too old

xref coreos/rpm-ostree@89f5802

Change the logic for firstboot OS updates to detect if rpm-ostree is too
old to natively fetch a container image on its own.  If so,
we use `ex deploy-from-self` via podman.

Introduce a new `/etc/machine-config-daemon-force-once` file that
will cause us to skip validation, and hence we should re-reconcile
and attempt to apply the OS upgrade again, this time natively
via rpm-ostree.

This is needed for scaleup of old nodes, as well as temporarily
for 4.11 upgrades.

go.mod

@@ -285,7 +285,7 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.66.6 // indirect
 	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	honnef.co/go/tools v0.3.3 // indirect
 	k8s.io/apiserver v0.25.1 // indirect

pkg/daemon/daemon.go

@@ -108,6 +108,9 @@ type Daemon struct {
 	// booting is true when all initial synchronization to the target
 	// machineconfig is done
 	booting bool
+	// needSecondaryReboot is used for https://issues.redhat.com/browse/MCO-356
+	// when we have an old rpm-ostree and need to reboot into the new one.
+	needSecondaryReboot bool
 
 	currentConfigPath string
 
@@ -884,6 +887,31 @@ func (dn *Daemon) RunFirstbootCompleteMachineconfig() error {
 	if err != nil {
 		return fmt.Errorf("failed to parse MachineConfig: %w", err)
 	}
+	newEnough, err := RpmOstreeIsNewEnoughForLayering()
+	if err != nil {
+		return err
+	}
+
+	// If the host isn't new enough to understand the new container model natively, run as a privileged container.
+	// See https://github.com/coreos/rpm-ostree/pull/3961 and https://issues.redhat.com/browse/MCO-356
+	// This currently will incur a double reboot; see https://github.com/coreos/rpm-ostree/issues/4018
+	if !newEnough {
+		dn.logSystem("rpm-ostree is not new enough for new-format image; forcing an update via container and queuing immediate reboot")
+		err := runCmdSync("systemd-run", "--unit", "machine-config-daemon-update-rpmostree-via-container", "--collect", "--wait", "--", "podman", "run", "--authfile", "/var/lib/kubelet/config.json", "--privileged", "--pid=host", "--net=host", "--rm", "-v", "/:/run/host", mc.Spec.OSImageURL, "rpm-ostree", "ex", "deploy-from-self", "/run/host")
+		if err != nil {
+			return err
+		}
+		rebootCmd := rebootCommand("extra reboot for in-place update")
+		if err := rebootCmd.Run(); err != nil {
+			dn.logSystem("failed to run reboot: %v", err)
+			return err
+		}
+		// wait to be killed via SIGTERM
+		time.Sleep(defaultRebootTimeout)
+		return fmt.Errorf("failed to reboot for secondary in-place update")
+	}
+
+	glog.Info("rpm-ostree has container feature")
 
 	// Start with an empty config, then add our *booted* osImageURL to
 	// it, reflecting the current machine state.
@@ -1407,9 +1435,11 @@ func (dn *Daemon) checkStateOnFirstRun() error {
 		return err
 	}
 	var pendingConfigName, bootID string
+	var pendingSpecifiesForce bool
 	if pendingState != nil {
 		pendingConfigName = pendingState.Message
 		bootID = pendingState.BootID
+		pendingSpecifiesForce = pendingState.Force == "true"
 	}
 	// XXX: drop this
 	// we need this compatibility layer for now
@@ -1456,7 +1486,7 @@ func (dn *Daemon) checkStateOnFirstRun() error {
 		targetOSImageURL := state.currentConfig.Spec.OSImageURL
 		osMatch := dn.checkOS(targetOSImageURL)
 		if !osMatch {
-			glog.Infof("Bootstrap pivot required to: %s", targetOSImageURL)
+			dn.logSystem("Bootstrap pivot required to: %s", targetOSImageURL)
 
 			// Check to see if we have a layered/new format image
 			isLayeredImage, err := dn.NodeUpdaterClient.IsBootableImage(targetOSImageURL)
@@ -1487,7 +1517,7 @@ func (dn *Daemon) checkStateOnFirstRun() error {
 			}
 			return dn.reboot(fmt.Sprintf("Node will reboot into config %v", state.currentConfig.GetName()))
 		}
-		glog.Info("No bootstrap pivot required; unlinking bootstrap node annotations")
+		dn.logSystem("No bootstrap pivot required; unlinking bootstrap node annotations")
 
 		// Rename the bootstrap node annotations; the
 		// currentConfig's osImageURL should now be *truth*.
@@ -1532,8 +1562,13 @@ func (dn *Daemon) checkStateOnFirstRun() error {
 		expectedConfig = state.currentConfig
 	}
 
-	if forceFileExists() {
-		glog.Infof("Skipping on-disk validation; %s present", constants.MachineConfigDaemonForceFile)
+	force := pendingSpecifiesForce || forceFileExists()
+	if force {
+		if pendingSpecifiesForce {
+			dn.logSystem("Skipping on-disk validation; pending config specifies forcing")
+		} else {
+			dn.logSystem("Skipping on-disk validation; %s present", constants.MachineConfigDaemonForceFile)
+		}
 		return dn.triggerUpdateWithMachineConfig(state.currentConfig, state.desiredConfig)
 	}
 
@@ -1543,7 +1578,7 @@ func (dn *Daemon) checkStateOnFirstRun() error {
 		return wErr
 	}
 
-	glog.Info("Validated on-disk state")
+	dn.logSystem("Validated on-disk state")
 
 	// We've validated state. Now, ensure that node is in desired state
 	var inDesiredConfig bool

pkg/daemon/rpm-ostree.go

@@ -14,6 +14,7 @@ import (
 	"github.com/golang/glog"
 	"github.com/opencontainers/go-digest"
 	pivotutils "github.com/openshift/machine-config-operator/pkg/daemon/pivot/utils"
+	"gopkg.in/yaml.v2"
 )
 
 const (
@@ -346,6 +347,47 @@ func (r *RpmOstreeClient) IsBootableImage(imgURL string) (bool, error) {
 	return isBootableImage == "true", nil
 }
 
+// RpmOstreeIsNewEnoughForLayering returns true if the version of rpm-ostree on the
+// host system is new enough for layering.
+// VersionData represents the static information about rpm-ostree.
+type VersionData struct {
+	Version  string   `yaml:"Version"`
+	Features []string `yaml:"Features"`
+	Git      string   `yaml:"Git"`
+}
+
+type RpmOstreeVersionData struct {
+	Root VersionData `yaml:"rpm-ostree"`
+}
+
+// RpmOstreeVersion returns the running rpm-ostree version number
+func rpmOstreeVersion() (*VersionData, error) {
+	buf, err := runGetOut("rpm-ostree", "--version")
+	if err != nil {
+		return nil, err
+	}
+
+	var q RpmOstreeVersionData
+	if err := yaml.Unmarshal(buf, &q); err != nil {
+		return nil, fmt.Errorf("failed to parse `rpm-ostree --version` output: %w", err)
+	}
+
+	return &q.Root, nil
+}
+
+func RpmOstreeIsNewEnoughForLayering() (bool, error) {
+	verdata, err := rpmOstreeVersion()
+	if err != nil {
+		return false, err
+	}
+	for _, v := range verdata.Features {
+		if v == "container" {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
 // RebaseLayered rebases system or errors if already rebased
 func (r *RpmOstreeClient) RebaseLayered(imgURL string) (err error) {
 	glog.Infof("Executing rebase to %s", imgURL)

pkg/daemon/rpm-ostree_test.go

@@ -0,0 +1,27 @@
+package daemon
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"gopkg.in/yaml.v2"
+)
+
+func TestParseVersion(t *testing.T) {
+	verdata := `rpm-ostree:
+  Version: '2022.10'
+  Git: 6b302116c969397fd71899e3b9bb3b8c100d1af9
+  Features:
+   - rust
+   - compose
+   - rhsm
+`
+	var q RpmOstreeVersionData
+	if err := yaml.UnmarshalStrict([]byte(verdata), &q); err != nil {
+		panic(err)
+	}
+
+	assert.Equal(t, "2022.10", q.Root.Version)
+	assert.Contains(t, q.Root.Features, "rust")
+	assert.NotContains(t, q.Root.Features, "container")
+}

pkg/daemon/update.go

@@ -1908,8 +1908,31 @@ func (dn *Daemon) updateOS(config *mcfgv1.MachineConfig, osImageContentDir strin
 func (dn *Daemon) updateLayeredOS(config *mcfgv1.MachineConfig) error {
 	newURL := config.Spec.OSImageURL
 	glog.Infof("Updating OS to layered image %s", newURL)
-	if err := dn.NodeUpdaterClient.RebaseLayered(newURL); err != nil {
-		return fmt.Errorf("failed to update OS to %s : %w", newURL, err)
+
+	newEnough, err := RpmOstreeIsNewEnoughForLayering()
+	if err != nil {
+		return err
+	}
+	// If the host isn't new enough to understand the new container model natively, run as a privileged container.
+	// See https://github.com/coreos/rpm-ostree/pull/3961 and https://issues.redhat.com/browse/MCO-356
+	// This currently will incur a double reboot; see https://github.com/coreos/rpm-ostree/issues/4018
+	if !newEnough {
+		dn.logSystem("rpm-ostree is not new enough for layering; forcing an update via container")
+		err := runCmdSync("systemd-run", "--unit", "machine-config-daemon-update-rpmostree-via-container", "--collect", "--wait", "--", "podman", "run", "--authfile", "/var/lib/kubelet/config.json", "--privileged", "--pid=host", "--net=host", "--rm", "-v", "/:/run/host", newURL, "rpm-ostree", "ex", "deploy-from-self", "/run/host")
+		if err != nil {
+			return err
+		}
+		dn.needSecondaryReboot = true
+		// We'll need to do a second reconciliation pass i.e. an extra reboot today, but that won't
+		// be necessary after we ship the newer rpm-ostree into older releases.
+		// See also https://github.com/coreos/rpm-ostree/issues/4018
+		if err := os.WriteFile(constants.MachineConfigDaemonPersistentForceOnceFile, []byte(""), 0o644); err != nil {
+			return err
+		}
+	} else {
+		if err := dn.NodeUpdaterClient.RebaseLayered(newURL); err != nil {
+			return fmt.Errorf("failed to update OS to %s : %w", newURL, err)
+		}
 	}
 
 	return nil
@@ -1957,6 +1980,7 @@ type journalMsg struct {
 	Message   string `json:"MESSAGE,omitempty"`
 	BootID    string `json:"BOOT_ID,omitempty"`
 	Pending   string `json:"PENDING,omitempty"`
+	Force     string `json:"MCO_FORCE,omitempty"`
 	OldLogger string `json:"OPENSHIFT_MACHINE_CONFIG_DAEMON_LEGACY_LOG_HACK,omitempty"` // unused today
 }
 
@@ -2020,7 +2044,8 @@ func (dn *Daemon) storePendingState(pending *mcfgv1.MachineConfig, isPending int
 	pendingState.WriteString(fmt.Sprintf(`MESSAGE_ID=%s
 MESSAGE=%s
 BOOT_ID=%s
-PENDING=%d`, pendingStateMessageID, pending.GetName(), dn.bootID, isPending))
+MCO_FORCE=%v
+PENDING=%d`, pendingStateMessageID, pending.GetName(), dn.bootID, dn.needSecondaryReboot, isPending))
 
 	logger.Stdin = &pendingState
 	return logger.CombinedOutput()