Skip to content

Commit

Permalink
add managed upgrade operator configuration settings and connected MUO…
Browse files Browse the repository at this point in the history 
… if allowed and a pullsecret exists
  • Loading branch information
@hawkowl
hawkowl committed Mar 23, 2022
1 parent b98114f commit 3713571
Show file tree
Hide file tree
Showing 5 changed files with 441 additions and 71 deletions.
10 changes: 10 additions & 0 deletions pkg/operator/controllers/muo/config/config.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package config

// Copyright (c) Microsoft Corporation.
// Licensed under the Apache License 2.0.

type MUODeploymentConfig struct {
Pullspec string
EnableConnected bool
OCMBaseURL string
}
77 changes: 65 additions & 12 deletions pkg/operator/controllers/muo/deploy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,36 +8,55 @@ import (
"errors"
"strings"

"github.com/ghodss/yaml"
"github.com/ugorji/go/codec"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
kruntime "k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"

"github.com/Azure/ARO-RP/pkg/api"
arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1"
"github.com/Azure/ARO-RP/pkg/operator/controllers/muo/config"
"github.com/Azure/ARO-RP/pkg/util/dynamichelper"
"github.com/Azure/ARO-RP/pkg/util/ready"
)

type muoConfig struct {
api.MissingFields
ConfigManager struct {
api.MissingFields
Source string `json:"source,omitempty"`
OcmBaseUrl string `json:"ocmBaseUrl,omitempty"`
LocalConfigName string `json:"localConfigName,omitempty"`
} `json:"configManager,omitempty"`
}

type Deployer interface {
CreateOrUpdate(context.Context, *arov1alpha1.Cluster) error
CreateOrUpdate(context.Context, *arov1alpha1.Cluster, *config.MUODeploymentConfig) error
Remove(context.Context) error
IsReady(ctx context.Context) (bool, error)
Resources(string) ([]kruntime.Object, error)
Resources(*config.MUODeploymentConfig) ([]kruntime.Object, error)
}

type deployer struct {
kubernetescli kubernetes.Interface
dh dynamichelper.Interface

jsonHandle *codec.JsonHandle
}

func newDeployer(kubernetescli kubernetes.Interface, dh dynamichelper.Interface) Deployer {
return &deployer{
kubernetescli: kubernetescli,
dh: dh,

jsonHandle: new(codec.JsonHandle),
}
}

func (o *deployer) Resources(pullspec string) ([]kruntime.Object, error) {
func (o *deployer) Resources(config *config.MUODeploymentConfig) ([]kruntime.Object, error) {
results := []kruntime.Object{}
for _, assetName := range AssetNames() {
b, err := Asset(assetName)
Expand All @@ -53,7 +72,46 @@ func (o *deployer) Resources(pullspec string) ([]kruntime.Object, error) {
// set the image for the deployments
if d, ok := obj.(*appsv1.Deployment); ok {
for i := range d.Spec.Template.Spec.Containers {
d.Spec.Template.Spec.Containers[i].Image = pullspec
d.Spec.Template.Spec.Containers[i].Image = config.Pullspec
}
}

if cm, ok := obj.(*corev1.ConfigMap); ok {
if cm.Name == "managed-upgrade-operator-config" && cm.Namespace == "openshift-managed-upgrade-operator" {
// read the config.yaml from the MUO ConfigMap which stores defaults
configDataJSON, err := yaml.YAMLToJSON([]byte(cm.Data["config.yaml"]))
if err != nil {
return nil, err
}

var configData muoConfig
err = codec.NewDecoderBytes(configDataJSON, o.jsonHandle).Decode(&configData)
if err != nil {
return nil, err
}

if config.EnableConnected {
configData.ConfigManager.Source = "OCM"
configData.ConfigManager.OcmBaseUrl = config.OCMBaseURL
configData.ConfigManager.LocalConfigName = ""
} else {
configData.ConfigManager.Source = "LOCAL"
configData.ConfigManager.LocalConfigName = "managed-upgrade-config"
configData.ConfigManager.OcmBaseUrl = ""
}

// Write the yaml back into the ConfigMap
var b []byte
err = codec.NewEncoderBytes(&b, o.jsonHandle).Encode(configData)
if err != nil {
return nil, err
}

cmYaml, err := yaml.JSONToYAML(b)
if err != nil {
return nil, err
}
cm.Data["config.yaml"] = string(cmYaml)
}
}

Expand All @@ -63,13 +121,8 @@ func (o *deployer) Resources(pullspec string) ([]kruntime.Object, error) {
return results, nil
}

func (o *deployer) CreateOrUpdate(ctx context.Context, cluster *arov1alpha1.Cluster) error {
imagePullspec, ext := cluster.Spec.OperatorFlags[controllerPullSpec]
if !ext {
return errors.New("missing pullspec")
}

resources, err := o.Resources(imagePullspec)
func (o *deployer) CreateOrUpdate(ctx context.Context, cluster *arov1alpha1.Cluster, config *config.MUODeploymentConfig) error {
resources, err := o.Resources(config)
if err != nil {
return err
}
Expand All @@ -88,7 +141,7 @@ func (o *deployer) CreateOrUpdate(ctx context.Context, cluster *arov1alpha1.Clus
}

func (o *deployer) Remove(ctx context.Context) error {
resources, err := o.Resources("")
resources, err := o.Resources(&config.MUODeploymentConfig{})
if err != nil {
return err
}
Expand Down
152 changes: 138 additions & 14 deletions pkg/operator/controllers/muo/deploy_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,20 @@ package muo
import (
"context"
"errors"
"strings"
"testing"

"github.com/go-test/deep"
"github.com/golang/mock/gomock"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kruntime "k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/fake"

arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1"
"github.com/Azure/ARO-RP/pkg/operator/controllers/muo/config"
mock_dynamichelper "github.com/Azure/ARO-RP/pkg/util/mocks/dynamichelper"
)

Expand All @@ -29,18 +32,13 @@ func TestDeployCreateOrUpdateCorrectKinds(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{
Name: arov1alpha1.SingletonClusterName,
},
Spec: arov1alpha1.ClusterSpec{
OperatorFlags: arov1alpha1.OperatorFlags{
controllerPullSpec: setPullSpec,
},
},
}

k8scli := fake.NewSimpleClientset()
dh := mock_dynamichelper.NewMockInterface(controller)

// When the DynamicHelper is called, count the number of objects it creates
// and capture any deployments so that we can check the pull secret
// and capture any deployments so that we can check the pullspec
var deployments []*appsv1.Deployment
deployedObjects := make(map[string]int)
check := func(ctx context.Context, objs ...kruntime.Object) error {
Expand All @@ -50,22 +48,25 @@ func TestDeployCreateOrUpdateCorrectKinds(t *testing.T) {
if err != nil {
return err
}
if d, ok := i.(*appsv1.Deployment); ok {
deployments = append(deployments, d)
}
deployedObjects[kind] = deployedObjects[kind] + 1
}
return nil
}
dh.EXPECT().Ensure(gomock.Any(), gomock.Any()).Do(check).Return(nil)

deployer := newDeployer(k8scli, dh)
err := deployer.CreateOrUpdate(context.Background(), cluster)
err := deployer.CreateOrUpdate(context.Background(), cluster, &config.MUODeploymentConfig{Pullspec: setPullSpec})
if err != nil {
t.Error(err)
}

// We expect these numbers of resources to be created
expectedKinds := map[string]int{
"ClusterRole": 1,
"ConfigMap": 1,
"ConfigMap": 2,
"ClusterRoleBinding": 1,
"CustomResourceDefinition": 1,
"Deployment": 1,
Expand Down Expand Up @@ -98,11 +99,6 @@ func TestDeployCreateOrUpdateSetsOwnerReferences(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{
Name: arov1alpha1.SingletonClusterName,
},
Spec: arov1alpha1.ClusterSpec{
OperatorFlags: arov1alpha1.OperatorFlags{
controllerPullSpec: setPullSpec,
},
},
}

k8scli := fake.NewSimpleClientset()
Expand Down Expand Up @@ -134,7 +130,7 @@ func TestDeployCreateOrUpdateSetsOwnerReferences(t *testing.T) {
dh.EXPECT().Ensure(gomock.Any(), gomock.Any()).Do(check).Return(nil)

deployer := newDeployer(k8scli, dh)
err := deployer.CreateOrUpdate(context.Background(), cluster)
err := deployer.CreateOrUpdate(context.Background(), cluster, &config.MUODeploymentConfig{Pullspec: setPullSpec})
if err != nil {
t.Error(err)
}
Expand Down Expand Up @@ -224,3 +220,131 @@ func TestDeployIsReadyMissing(t *testing.T) {
}

}

func TestDeployConfig(t *testing.T) {
controller := gomock.NewController(t)
defer controller.Finish()

cluster := &arov1alpha1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: arov1alpha1.SingletonClusterName,
},
}

tests := []struct {
name string
deploymentConfig *config.MUODeploymentConfig
expected []string
}{
{
name: "local",
deploymentConfig: &config.MUODeploymentConfig{EnableConnected: false},
expected: []string{
"configManager:",
" localConfigName: managed-upgrade-config",
" source: LOCAL",
" watchInterval: 1",
"healthCheck:",
" ignoredCriticals:",
" - PrometheusRuleFailures",
" - CannotRetrieveUpdates",
" - FluentdNodeDown",
" ignoredNamespaces:",
" - openshift-logging",
" - openshift-redhat-marketplace",
" - openshift-operators",
" - openshift-user-workload-monitoring",
" - openshift-pipelines",
"maintenance:",
" controlPlaneTime: 90",
" ignoredAlerts:",
" controlPlaneCriticals:",
" - ClusterOperatorDown",
" - ClusterOperatorDegraded",
"nodeDrain:",
" expectedNodeDrainTime: 8",
" timeOut: 45",
"scale:",
" timeOut: 30",
"upgradeWindow:",
" delayTrigger: 30",
" timeOut: 120",
"",
},
},
{
name: "connected",
deploymentConfig: &config.MUODeploymentConfig{EnableConnected: true, OCMBaseURL: "https://example.com"},
expected: []string{
"configManager:",
" ocmBaseUrl: https://example.com",
" source: OCM",
" watchInterval: 1",
"healthCheck:",
" ignoredCriticals:",
" - PrometheusRuleFailures",
" - CannotRetrieveUpdates",
" - FluentdNodeDown",
" ignoredNamespaces:",
" - openshift-logging",
" - openshift-redhat-marketplace",
" - openshift-operators",
" - openshift-user-workload-monitoring",
" - openshift-pipelines",
"maintenance:",
" controlPlaneTime: 90",
" ignoredAlerts:",
" controlPlaneCriticals:",
" - ClusterOperatorDown",
" - ClusterOperatorDegraded",
"nodeDrain:",
" expectedNodeDrainTime: 8",
" timeOut: 45",
"scale:",
" timeOut: 30",
"upgradeWindow:",
" delayTrigger: 30",
" timeOut: 120",
"",
},
},
}
for _, tt := range tests {
k8scli := fake.NewSimpleClientset()
dh := mock_dynamichelper.NewMockInterface(controller)

// When the DynamicHelper is called, capture configmaps to inspect them
var configs []*corev1.ConfigMap
check := func(ctx context.Context, objs ...kruntime.Object) error {
for _, i := range objs {
if cm, ok := i.(*corev1.ConfigMap); ok {
configs = append(configs, cm)
}

}
return nil
}
dh.EXPECT().Ensure(gomock.Any(), gomock.Any()).Do(check).Return(nil)

deployer := newDeployer(k8scli, dh)
err := deployer.CreateOrUpdate(context.Background(), cluster, tt.deploymentConfig)
if err != nil {
t.Error(err)
}

foundConfig := false
for _, cms := range configs {
if cms.Name == "managed-upgrade-operator-config" && cms.Namespace == "openshift-managed-upgrade-operator" {
foundConfig = true
errs := deep.Equal(tt.expected, strings.Split(cms.Data["config.yaml"], "\n"))
for _, e := range errs {
t.Error(e)
}
}
}

if !foundConfig {
t.Error("MUO config was not found")
}
}
}
Loading

0 comments on commit 3713571

Please sign in to comment.