Skip to content

Commit

Permalink
Implement elemental-register upgrade
Browse files Browse the repository at this point in the history
Signed-off-by: Andrea Mazzotti <[email protected]>
  • Loading branch information
anmazzotti committed Oct 30, 2024
1 parent f300a43 commit 2ca6dd8
Show file tree
Hide file tree
Showing 31 changed files with 2,095 additions and 31 deletions.
2 changes: 2 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -252,9 +252,11 @@ generate-mocks: $(MOCKGEN)
$(MOCKGEN) -copyright_file=scripts/boilerplate.go.txt -destination=pkg/register/mocks/client.go -package=mocks github.com/rancher/elemental-operator/pkg/register Client
$(MOCKGEN) -copyright_file=scripts/boilerplate.go.txt -destination=pkg/register/mocks/state.go -package=mocks github.com/rancher/elemental-operator/pkg/register StateHandler
$(MOCKGEN) -copyright_file=scripts/boilerplate.go.txt -destination=pkg/install/mocks/install.go -package=mocks github.com/rancher/elemental-operator/pkg/install Installer
$(MOCKGEN) -copyright_file=scripts/boilerplate.go.txt -destination=pkg/upgrade/mocks/upgrade.go -package=mocks github.com/rancher/elemental-operator/pkg/upgrade Upgrader
$(MOCKGEN) -copyright_file=scripts/boilerplate.go.txt -destination=pkg/elementalcli/mocks/elementalcli.go -package=mocks github.com/rancher/elemental-operator/pkg/elementalcli Runner
$(MOCKGEN) -copyright_file=scripts/boilerplate.go.txt -destination=pkg/network/mocks/network.go -package=mocks github.com/rancher/elemental-operator/pkg/network Configurator
$(MOCKGEN) -copyright_file=scripts/boilerplate.go.txt -destination=pkg/util/mocks/command_runner.go -package=mocks github.com/rancher/elemental-operator/pkg/util CommandRunner
$(MOCKGEN) -copyright_file=scripts/boilerplate.go.txt -destination=pkg/util/mocks/nsenter.go -package=mocks github.com/rancher/elemental-operator/pkg/util NsEnter

.PHONY: generate-go
generate-go: generate-mocks $(CONTROLLER_GEN) ## Runs Go related generate targets for the operator
Expand Down
4 changes: 4 additions & 0 deletions api/v1beta1/common_consts.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,10 @@ const (
// ElementalManagedOSVersionChannelLabel is used to filter a set of ManagedOSVersions given the channel they originate from.
ElementalManagedOSVersionChannelLabel = "elemental.cattle.io/channel"

// ElementalUpgradeCorrelationIDLabel is used to correlate a system snapshot to an individual upgrade plan generated by a ManagedOSImage.
// This label is consistently applied to ManagedOSImages, upgrade.cattle.io/v1/Plans, and system snapshots on the downstream machine.
ElementalUpgradeCorrelationIDLabel = "elemental.cattle.io/upgrade-correlation-id"

// ElementalManagedOSVersionChannelLastSyncAnnotation reports when a ManagedOSVersion was last synced from a channel.
ElementalManagedOSVersionChannelLastSyncAnnotation = "elemental.cattle.io/channel-last-sync"

Expand Down
1 change: 1 addition & 0 deletions cmd/register/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ func main() {
cmd.AddCommand(
newVersionCommand(),
newDumpDataCommand(),
newUpgradeCommand(),
)
if err := cmd.Execute(); err != nil {
log.Fatalf("FATAL: %s", err)
Expand Down
153 changes: 153 additions & 0 deletions cmd/register/upgrade.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
/*
Copyright © 2022 - 2024 SUSE LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"errors"
"fmt"
"reflect"

"strings"

"github.com/mitchellh/mapstructure"
"github.com/twpayne/go-vfs"

"github.com/rancher/elemental-operator/pkg/elementalcli"
"github.com/rancher/elemental-operator/pkg/log"
"github.com/rancher/elemental-operator/pkg/upgrade"
"github.com/rancher/elemental-operator/pkg/util"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)

var (
ErrRebooting = errors.New("Machine needs reboot after upgrade")
ErrAlreadyShuttingDown = errors.New("System is already shutting down")
ErrMissingCorrelationID = errors.New("Missing upgrade correlation ID")
)

var decodeHook = viper.DecodeHook(
mapstructure.ComposeDecodeHookFunc(
KeyValuePairHook(),
),
)

func KeyValuePairHook() mapstructure.DecodeHookFuncType {
return func(from reflect.Type, to reflect.Type, data interface{}) (interface{}, error) {
if from.Kind() != reflect.String {
return data, nil
}

if to != reflect.TypeOf(elementalcli.KeyValuePair{}) {
return data, nil
}

return elementalcli.KeyValuePairFromData(data)
}
}

func newUpgradeCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "upgrade",
Short: "Upgrades the machine",
RunE: func(_ *cobra.Command, _ []string) error {
upgradeConfig := elementalcli.UpgradeConfig{Bootloader: true}
if err := viper.Unmarshal(&upgradeConfig, decodeHook); err != nil {
return fmt.Errorf("unmarshalling config: %w", err)
}
environment := upgrade.Environment{
Config: upgradeConfig,
}
if err := viper.Unmarshal(&environment); err != nil {
return fmt.Errorf("unmarshalling context: %w", err)
}

if upgradeConfig.Debug {
log.EnableDebugLogging()
}

nsEnter := util.NewNsEnter()
upgrader := upgrade.NewUpgrader(vfs.OSFS)
return upgradeElemental(nsEnter, upgrader, environment)
},
}

viper.AutomaticEnv()
replacer := strings.NewReplacer("-", "_")
viper.SetEnvKeyReplacer(replacer)
viper.SetEnvPrefix("ELEMENTAL_REGISTER_UPGRADE")

cmd.Flags().String("host-dir", "/host", "The machine root directory where to apply the upgrade")
_ = viper.BindPFlag("host-dir", cmd.Flags().Lookup("host-dir"))

cmd.Flags().String("cloud-config", "/run/data/cloud-config", "The path of a cloud-config file to install on the machine during upgrade")
_ = viper.BindPFlag("cloud-config", cmd.Flags().Lookup("cloud-config"))

cmd.Flags().String("system", "dir:/", "The system image uri or filesystem location to upgrade to")
_ = viper.BindPFlag("system", cmd.Flags().Lookup("system"))

cmd.Flags().String("correlation-id", "", "A correlationID to label the upgrade snapshot with")
_ = viper.BindPFlag("correlation-id", cmd.Flags().Lookup("correlation-id"))

cmd.Flags().Bool("recovery", false, "Upgrades the recovery partition together with the system")
_ = viper.BindPFlag("recovery", cmd.Flags().Lookup("recovery"))

cmd.Flags().Bool("recovery-only", false, "Upgrades the recovery partition only")
_ = viper.BindPFlag("recovery-only", cmd.Flags().Lookup("recovery-only"))

cmd.Flags().Bool("debug", true, "Prints debug logs when performing upgrade")
_ = viper.BindPFlag("debug", cmd.Flags().Lookup("debug"))

cmd.Flags().StringToString("snapshot-labels", map[string]string{}, "Labels to apply to the upgrade snapshot")
_ = viper.BindPFlag("snapshot-labels", cmd.Flags().Lookup("snapshot-labels"))

return cmd
}

func upgradeElemental(nsEnter util.NsEnter, upgrader upgrade.Upgrader, environment upgrade.Environment) error {
// For sanity, this needs to be verified or the upgrade process may end up in an infinite loop
if len(environment.Config.CorrelationID) == 0 {
return ErrMissingCorrelationID
}

// If the system is shutting down, return an error so we can try again on next reboot.
alreadyShuttingDown, err := nsEnter.IsSystemShuttingDown(environment.HostDir)
if err != nil {
return fmt.Errorf("determining if system is running: %w", err)
}
if alreadyShuttingDown {
return ErrAlreadyShuttingDown
}

// If system is not shutting down we can proceed.
needsReboot, err := upgrader.UpgradeElemental(environment)
// If the upgrade could not be applied or verified,
// then this command will fail but the machine will not reboot.
if err != nil {
return fmt.Errorf("upgrading machine: %w", err)
}
// If the machine needs a reboot after an upgrade has been applied,
// so that consumers can try again after reboot to validate the upgrade has been applied successfully.
if needsReboot {
log.Infof("Rebooting machine after %s upgrade", environment.Config.CorrelationID)
nsEnter.Reboot(environment.HostDir)
return ErrRebooting
}
// Upgrade has been applied successfully, nothing to do.
log.Infof("Upgrade %s applied successfully", environment.Config.CorrelationID)
return nil
}
84 changes: 84 additions & 0 deletions cmd/register/upgrade_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
/*
Copyright © 2022 - 2024 SUSE LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"go.uber.org/mock/gomock"

"github.com/rancher/elemental-operator/pkg/elementalcli"
"github.com/rancher/elemental-operator/pkg/upgrade"
upgrademocks "github.com/rancher/elemental-operator/pkg/upgrade/mocks"
utilmocks "github.com/rancher/elemental-operator/pkg/util/mocks"
)

var _ = Describe("elemental-register upgrade", Label("elemental-register", "upgrade"), func() {
var upgrader *upgrademocks.MockUpgrader
var nsenter *utilmocks.MockNsEnter
BeforeEach(func() {
mockCtrl := gomock.NewController(GinkgoT())
upgrader = upgrademocks.NewMockUpgrader(mockCtrl)
nsenter = utilmocks.NewMockNsEnter(mockCtrl)
})
It("should error if correlationID missing", func() {
environment := upgrade.Environment{}
err := upgradeElemental(nsenter, upgrader, environment)
Expect(err).To(Equal(ErrMissingCorrelationID))
})
It("should not upgrade if system shutting down", func() {
environment := upgrade.Environment{
HostDir: "/foo",
Config: elementalcli.UpgradeConfig{
CorrelationID: "bar",
},
}
nsenter.EXPECT().IsSystemShuttingDown(environment.HostDir).Return(true, nil)
err := upgradeElemental(nsenter, upgrader, environment)
Expect(err).To(Equal(ErrAlreadyShuttingDown))
})
It("should upgrade and reboot", func() {
environment := upgrade.Environment{
HostDir: "/foo",
Config: elementalcli.UpgradeConfig{
CorrelationID: "bar",
},
}
gomock.InOrder(
nsenter.EXPECT().IsSystemShuttingDown(environment.HostDir).Return(false, nil),
upgrader.EXPECT().UpgradeElemental(environment).Return(true, nil),
nsenter.EXPECT().Reboot(environment.HostDir),
)

err := upgradeElemental(nsenter, upgrader, environment)
Expect(err).To(Equal(ErrRebooting))
})
It("should upgrade and not reboot", func() {
environment := upgrade.Environment{
HostDir: "/foo",
Config: elementalcli.UpgradeConfig{
CorrelationID: "bar",
},
}
gomock.InOrder(
nsenter.EXPECT().IsSystemShuttingDown(environment.HostDir).Return(false, nil),
upgrader.EXPECT().UpgradeElemental(environment).Return(false, nil),
)

Expect(upgradeElemental(nsenter, upgrader, environment)).Should(Succeed())
})
})
65 changes: 64 additions & 1 deletion controllers/managedosimage_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,12 @@ func (r *ManagedOSImageReconciler) newFleetBundleResources(ctx context.Context,
// we just do a safe name conversion here.
uniqueName = toDNSLabel(uniqueName)

upgradeContainerSpecCopy := *upgradeContainerSpec.DeepCopy()
correlationID, err := applyCorrelationLabels(managedOSImage, version, &upgradeContainerSpecCopy)
if err != nil {
return nil, fmt.Errorf("applying upgrade environment variables: %w", err)
}

objs := []runtime.Object{
&rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{
Expand Down Expand Up @@ -326,13 +332,15 @@ func (r *ManagedOSImageReconciler) newFleetBundleResources(ctx context.Context,
ObjectMeta: metav1.ObjectMeta{
Name: uniqueName,
Namespace: rancherSystemNamespace,
Labels: map[string]string{elementalv1.ElementalUpgradeCorrelationIDLabel: correlationID},
},
Spec: upgradev1.PlanSpec{
Concurrency: concurrency,
Version: version,
Tolerations: []corev1.Toleration{{
Operator: corev1.TolerationOpExists,
}},
Exclusive: true,
ServiceAccountName: uniqueName,
NodeSelector: selector,
Cordon: cordon,
Expand All @@ -342,7 +350,7 @@ func (r *ManagedOSImageReconciler) newFleetBundleResources(ctx context.Context,
Name: uniqueName,
Path: "/run/data",
}},
Upgrade: upgradeContainerSpec,
Upgrade: &upgradeContainerSpecCopy,
},
},
}
Expand Down Expand Up @@ -582,10 +590,65 @@ func metadataEnv(m map[string]runtime.RawExtension) []corev1.EnvVar {
return envs
}

func applyCorrelationLabels(managedOSImage *elementalv1.ManagedOSImage, imageVersion string, upgradeContainerSpec *upgradev1.ContainerSpec) (string, error) {
// Include additional snapshot labels
upgradeContainerSpec.Env = append(upgradeContainerSpec.Env, corev1.EnvVar{
Name: "ELEMENTAL_REGISTER_UPGRADE_SNAPSHOT_LABELS",
Value: formatSnapshotLabels(*managedOSImage, imageVersion, *upgradeContainerSpec),
})

// Calculate the managedOSImage.Spec after all changes
correlationID, err := managedOSImageHash(managedOSImage.Spec)
if err != nil {
return "", fmt.Errorf("calculating ManagedOSImage hash: %w", err)
}
// Tag this ManagedOSImage with the correlation ID label
if managedOSImage.Labels == nil {
managedOSImage.Labels = map[string]string{}
}
managedOSImage.Labels[elementalv1.ElementalUpgradeCorrelationIDLabel] = correlationID
// Use the hash as correlation ID that will be applied as snapshot label on the machine.
upgradeContainerSpec.Env = append(upgradeContainerSpec.Env, corev1.EnvVar{
Name: "ELEMENTAL_REGISTER_UPGRADE_CORRELATION_ID",
Value: correlationID,
})

return correlationID, nil
}

// This converts any string to RFC 1123 DNS label standard by replacing invalid characters with "-"
func toDNSLabel(input string) string {
output := dnsLabelRegex.ReplaceAllString(input, "-")
output = strings.TrimPrefix(output, "-")
output = strings.TrimSuffix(output, "-")
return output
}

func managedOSImageHash(spec elementalv1.ManagedOSImageSpec) (string, error) {
// Do not calculate a new hash if target changes.
spec.Targets = []fleetv1.BundleTarget{}

specData, err := json.Marshal(spec)
if err != nil {
return "", fmt.Errorf("unmarshalling ManagedOSImage.Spec: %w", err)
}
hash := sha256.New224() //sha256 produces 64 chars (max label value is 63). Use sha224 instead.
if _, err := hash.Write(specData); err != nil {
return "", fmt.Errorf("writing hash: %w", err)
}

result := hash.Sum(nil)
return fmt.Sprintf("%x", result), nil
}

// formatSnapshotLabels formats the *_SNAPSHOT_LABELS environment variable: "managedOSImage=foo,image=bar:v1.2.3"
// It is required for this function to always generate the same value in a predictable way, in order to keep the computed Plan hash the same at every loop.
func formatSnapshotLabels(managedOSImage elementalv1.ManagedOSImage, imageVersion string, upgradeContainerSpec upgradev1.ContainerSpec) string {
imageWithVersion := fmt.Sprintf("%s:%s", upgradeContainerSpec.Image, imageVersion)

formattedLabels := fmt.Sprintf("managedOSImage=%s,image=%s", managedOSImage.Name, imageWithVersion)
if len(managedOSImage.Spec.ManagedOSVersionName) > 0 {
formattedLabels = fmt.Sprintf("%s,managedOSVersion=%s", formattedLabels, managedOSImage.Spec.ManagedOSVersionName)
}
return formattedLabels
}
Loading

0 comments on commit 2ca6dd8

Please sign in to comment.