Merge branch 'master' of https://github.com/Azure/ARO-RP into proxy-r…

…efactor-and-tes

.gitignore

@@ -13,6 +13,7 @@ gomock_reflect_*
 /e2e.test
 /env*
 !/env.example
+!/env-int.example
 /id_rsa
 /proxy
 /pyenv*

.pipelines/onebranch/pipeline.buildrp.official.yml

@@ -47,7 +47,20 @@ extends:
         suppressionSet: default
 
     stages:
+    - stage: Build_ARO
+      jobs:
+      - job: Build_ARO
+        pool:
+          type: linux
+
+        variables: # More settings at https://aka.ms/obpipelines/yaml/jobs
+          ob_outputDirectory: $(Build.SourcesDirectory)/out   # this directory is uploaded to pipeline artifacts, reddog and cloudvault. More info at https://aka.ms/obpipelines/artifacts
+
+        steps:
+        - template: .pipelines/onebranch/templates/template-buildrp-buildaro.yml@self
+
     - stage: Build_Docker_Image
+      dependsOn: Build_ARO
       jobs:
       - job: Build_Docker_Image
         pool:
@@ -56,19 +69,7 @@ extends:
 
         variables:
           ob_git_checkout: true
+          release_tag: $[stageDependencies.Build_ARO.Build_ARO.outputs['buildaro.releasetag']]
 
         steps:
         - template: .pipelines/onebranch/templates/template-buildrp-builddocker.yml@self
-
-
-    - stage: Build_ARO
-      jobs:
-      - job: Build_ARO
-        pool:
-          type: linux
-
-        variables: # More settings at https://aka.ms/obpipelines/yaml/jobs
-          ob_outputDirectory: $(Build.SourcesDirectory)/out   # this directory is uploaded to pipeline artifacts, reddog and cloudvault. More info at https://aka.ms/obpipelines/artifacts
-
-        steps:
-        - template: .pipelines/onebranch/templates/template-buildrp-buildaro.yml@self

.pipelines/onebranch/pipeline.buildrp.pullrequest.yml

@@ -47,7 +47,20 @@ extends:
         suppressionSet: default
 
     stages:
+    - stage: Build_ARO
+      jobs:
+      - job: Build_ARO
+        pool:
+          type: linux
+
+        variables: # More settings at https://aka.ms/obpipelines/yaml/jobs
+          ob_outputDirectory: $(Build.SourcesDirectory)/out   # this directory is uploaded to pipeline artifacts, reddog and cloudvault. More info at https://aka.ms/obpipelines/artifacts
+
+        steps:
+        - template: .pipelines/onebranch/templates/template-buildrp-buildaro.yml@self
+
     - stage: Build_Docker_Image
+      dependsOn: Build_ARO
       jobs:
       - job: Build_Docker_Image
         pool:
@@ -56,19 +69,7 @@ extends:
 
         variables:
           ob_git_checkout: true
+          release_tag: $[stageDependencies.Build_ARO.Build_ARO.outputs['buildaro.releasetag']]
 
         steps:
         - template: .pipelines/onebranch/templates/template-buildrp-builddocker.yml@self
-
-
-    - stage: Build_ARO
-      jobs:
-      - job: Build_ARO
-        pool:
-          type: linux
-
-        variables: # More settings at https://aka.ms/obpipelines/yaml/jobs
-          ob_outputDirectory: $(Build.SourcesDirectory)/out   # this directory is uploaded to pipeline artifacts, reddog and cloudvault. More info at https://aka.ms/obpipelines/artifacts
-
-        steps:
-        - template: .pipelines/onebranch/templates/template-buildrp-buildaro.yml@self

.pipelines/onebranch/templates/template-buildrp-buildaro.yml

@@ -5,13 +5,24 @@ steps:
     targetType: inline
     script: |
       export GOPATH=$(Agent.TempDirectory)
+      export TAG=$(git describe --exact-match 2>/dev/null)
+      export COMMIT=$(git rev-parse --short=7 HEAD)$([[ $(git status --porcelain) = "" ]] || echo -dirty)
+      if [ -z "$TAG" ];
+      then
+        export VERSION=${COMMIT}
+      else
+        export VERSION=${TAG}
+      fi
+      echo "Version: ${VERSION}"
+      echo "##vso[task.setvariable variable=releasetag;isOutput=true]${VERSION}"
       mkdir -p $(Agent.TempDirectory)/src/github.com/Azure/
       cp -rd $(Build.SourcesDirectory) $(Agent.TempDirectory)/src/github.com/Azure/ARO-RP
       cd $(Agent.TempDirectory)/src/github.com/Azure/ARO-RP
       make aro
       mkdir -p $(ob_outputDirectory) 
       cp aro $(ob_outputDirectory)/aro
     workingDirectory: $(Build.SourcesDirectory)
+  name: buildaro
 - task: Bash@3
   displayName: 🕵️ Validate FIPS
   inputs:

.pipelines/onebranch/templates/template-buildrp-builddocker.yml

@@ -9,4 +9,4 @@ steps:
     saveImageToPath: aro-rp.tar
     buildkit: 1
     enable_network: true
-    build_tag: $(Build.SourceBranchName)
+    build_tag: $(release_tag)

Dockerfile.muo

@@ -0,0 +1,29 @@
+FROM registry.access.redhat.com/ubi8/go-toolset:1.16.12 AS builder
+ARG MUOVERSION
+ENV DOWNLOAD_URL=https://github.com/openshift/managed-upgrade-operator/archive/${MUOVERSION}.tar.gz
+ENV GOOS=linux \
+    GOPATH=/go/ \
+    GOARCH=amd64 \
+    CGO_ENABLED=0
+WORKDIR ${GOPATH}/src/github.com/openshift/managed-upgrade-operator
+USER root
+RUN yum update -yq
+RUN curl -Lq $DOWNLOAD_URL | tar -xz --strip-components=1
+RUN go build -gcflags="all=-trimpath=/go/" -asmflags="all=-trimpath=/go/" -tags mandate_fips -o build/_output/bin/managed-upgrade-operator ./cmd/manager
+
+#### Runtime container
+FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
+
+ENV USER_UID=1001 \
+    USER_NAME=managed-upgrade-operator
+
+RUN microdnf update && microdnf clean all
+COPY --from=builder /go/src/github.com/openshift/managed-upgrade-operator/build/_output/bin/* \
+                    /go/src/github.com/openshift/managed-upgrade-operator/build/bin/* \
+                    /usr/local/bin/
+RUN /usr/local/bin/user_setup
+
+ENTRYPOINT ["/usr/local/bin/entrypoint"]
+USER ${USER_UID}
+LABEL io.openshift.managed.name="managed-upgrade-operator" \
+      io.openshift.managed.description="Operator to manage upgrades for Openshift version 4 clusters"

Makefile

@@ -2,7 +2,7 @@ SHELL = /bin/bash
 TAG ?= $(shell git describe --exact-match 2>/dev/null)
 COMMIT = $(shell git rev-parse --short=7 HEAD)$(shell [[ $$(git status --porcelain) = "" ]] || echo -dirty)
 ARO_IMAGE_BASE = ${RP_IMAGE_ACR}.azurecr.io/aro
-E2E_FLAGS ?= -test.timeout 180m -test.v -ginkgo.v
+E2E_FLAGS ?= -test.timeout 180m -test.v -ginkgo.v -ginkgo.noColor
 
 # fluentbit version must also be updated in RP code, see pkg/util/version/const.go
 FLUENTBIT_VERSION = 1.7.8-1

cmd/aro/operator.go

@@ -23,6 +23,7 @@ import (
 	pkgoperator "github.com/Azure/ARO-RP/pkg/operator"
 	aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/alertwebhook"
+	"github.com/Azure/ARO-RP/pkg/operator/controllers/autosizednodes"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/banner"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/checker"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/clusteroperatoraro"
@@ -210,6 +211,11 @@ func operator(ctx context.Context, log *logrus.Entry) error {
 		if err = (muo.NewReconciler(arocli, kubernetescli, dh)).SetupWithManager(mgr); err != nil {
 			return fmt.Errorf("unable to create controller %s: %v", muo.ControllerName, err)
 		}
+		if err = (autosizednodes.NewReconciler(
+			log.WithField("controller", autosizednodes.ControllerName),
+			mgr)).SetupWithManager(mgr); err != nil {
+			return fmt.Errorf("unable to create controller %s: %v", autosizednodes.ControllerName, err)
+		}
 	}
 
 	if err = (checker.NewReconciler(

docs/deploy-development-rp.md

@@ -150,7 +150,7 @@
   curl -X PATCH -k "https://localhost:8443/subscriptions/$AZURE_SUBSCRIPTION_ID/resourceGroups/$RESOURCEGROUP/providers/Microsoft.RedHatOpenShift/openShiftClusters/$CLUSTER?api-version=admin" --header "Content-Type: application/json" -d "{}"
   ```
 
-* Get Cluster detials of a dev cluster
+* Get Cluster details of a dev cluster
   ```bash
   curl -X GET -k "https://localhost:8443/subscriptions/$AZURE_SUBSCRIPTION_ID/resourceGroups/$RESOURCEGROUP/providers/Microsoft.RedHatOpenShift/openShiftClusters/$CLUSTER?api-version=admin" --header "Content-Type: application/json" -d "{}"
   ```

docs/deploy-full-rp-service-in-dev.md

@@ -21,24 +21,11 @@
     make dev-config.yaml
     ```
 
-1. Update and resource your environment file
-    > It should look something like below once completed
+1. Create a full environment file, which overrides some default `./env` options when sourced
     ```bash
-    export LOCATION=eastus
-    export ARO_IMAGE=arointsvc.azurecr.io/aro:latest
-
-    . secrets/env
-
-    export RESOURCEGROUP=$USER-aro-$LOCATION
-    export DATABASE_ACCOUNT_NAME=$USER-aro-$LOCATION
-    export DATABASE_NAME=ARO
-    export KEYVAULT_PREFIX=$USER-aro-$LOCATION
-    export ARO_IMAGE=${USER}aro.azurecr.io/aro:$(git rev-parse --short=7 HEAD)$([[ $(git status --porcelain) = "" ]] || echo -dirty)
-    export FLUENTBIT_IMAGE=${USER}aro.azurecr.io/fluentbit:latest
-    ```
-
-    ```bash
-    . ./env
+    cp env-int.example env-int
+    vi env-int
+    . ./env-int
     ```
 
 1. Run `make deploy`
@@ -104,6 +91,7 @@
         export SRC_AUTH_QUAY=$(echo $USER_PULL_SECRET | jq -r '.auths."quay.io".auth')
         export SRC_AUTH_REDHAT=$(echo $USER_PULL_SECRET | jq -r '.auths."registry.redhat.io".auth')
         export DST_AUTH=$(echo -n '00000000-0000-0000-0000-000000000000:'$(az acr login -n ${DST_ACR_NAME} --expose-token | jq -r .accessToken) | base64 -w0)
+        ```
 
     1. Login to the Azure Container Registry
         ```bash

docs/prepare-your-dev-environment.md

@@ -18,11 +18,19 @@ This document goes through the development dependencies one requires in order to
 
 1. Install the relevant packages required for your OS defined below.
 
-### Fedora Packages
+1. Install [Podman](https://podman.io/getting-started/installation) and [podman-docker](https://developers.redhat.com/blog/2019/02/21/podman-and-buildah-for-docker-users#) if you haven't already, used for building container images.
+
+1. Run for `az acr login` compatability
+```bash
+sudo touch /etc/containers/nodocker
+```
 
+### Fedora Packages
 1. Install the `gpgme-devel`, `libassuan-devel`, and `openssl` packages.
 > `sudo dnf install -y gpgme-devel libassuan-devel openssl`
 
+1. Install [Docker 17.05+](https://docs.docker.com/engine/install/fedora/) or later, used as an alternative to podman.
+
 ### Debian Packages
 1. Install the `libgpgme-dev` package.
 
@@ -71,5 +79,14 @@ sudo pip3 install virtualenv
 1. Clone the repository to your local machine:
     ```bash
     go get -u github.com/Azure/ARO-RP/...
+    ```
+    Alternatively you can also use:
+    ```bash
+    git clone https://github.com/Azure/ARO-RP.git $GOPATH/src/github.com/Azure/ARO-RP
+    ```
+
+1. Go to project:
+    ```bash
     cd ${GOPATH:-$HOME/go}/src/github.com/Azure/ARO-RP
     ```
+

env-int.example

@@ -0,0 +1,9 @@
+. ./env
+
+# Overrides for a full int-like local environment
+export RESOURCEGROUP=$USER-aro-$LOCATION
+export DATABASE_ACCOUNT_NAME=$USER-aro-$LOCATION
+export DATABASE_NAME=ARO
+export KEYVAULT_PREFIX=$USER-aro-$LOCATION
+export ARO_IMAGE=${USER}aro.azurecr.io/aro:$(git rev-parse --short=7 HEAD)$([[ $(git status --porcelain) = "" ]] || echo -dirty)
+export FLUENTBIT_IMAGE=${USER}aro.azurecr.io/fluentbit:latest

go.mod

@@ -57,7 +57,7 @@ require (
 	github.com/gorilla/securecookie v1.1.1
 	github.com/gorilla/sessions v1.2.1
 	github.com/h2non/filetype v1.1.3 // indirect
-	github.com/jewzaam/go-cosmosdb v0.0.0-20211018134417-8d1494e7862f
+	github.com/jewzaam/go-cosmosdb v0.0.0-20220315232836-282b67c5b234
 	github.com/jstemmer/go-junit-report v0.9.1
 	github.com/klauspost/compress v1.14.2 // indirect
 	github.com/libvirt/libvirt-go v7.4.0+incompatible // indirect

go.sum

@@ -1529,8 +1529,8 @@ github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4Fw
 github.com/jen20/awspolicyequivalence v1.1.0/go.mod h1:PV1fS2xyHhCLp83vbgSMFr2drM4GzG61wkz+k4pOG3E=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
-github.com/jewzaam/go-cosmosdb v0.0.0-20211018134417-8d1494e7862f h1:G0wto7wY9OnIuwSPiLfyefcUN4FYEdtvwQMsCQdlDas=
-github.com/jewzaam/go-cosmosdb v0.0.0-20211018134417-8d1494e7862f/go.mod h1:Sxto8tJPhBRIa7jDsuTx4TZAcTYu0NH8E39iWqqMeRs=
+github.com/jewzaam/go-cosmosdb v0.0.0-20220315232836-282b67c5b234 h1:R0Hokq55Hv3SpbXyfZRh8vIrKFAPZ+SQqyTsRc7J/4E=
+github.com/jewzaam/go-cosmosdb v0.0.0-20220315232836-282b67c5b234/go.mod h1:kZxm8EB19+pd3nT92t0aQblXb7K4sHG8cp59cleqdNc=
 github.com/jewzaam/installer-aro v0.9.0-master.0.20220208140934-766bcf74e25c h1:yg14zwP4cFO98qqFlbkNqcouP/J0+zdCUf11qa/J/PU=
 github.com/jewzaam/installer-aro v0.9.0-master.0.20220208140934-766bcf74e25c/go.mod h1:qjvlt/LNPsGbLg2DtyY0VzxfKo+WVwcxJhektwKU+5E=
 github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74=

pkg/api/defaults.go

@@ -76,5 +76,6 @@ func DefaultOperatorFlags() OperatorFlags {
 		"aro.routefix.enabled":                     flagTrue,
 		"aro.storageaccounts.enabled":              flagTrue,
 		"aro.workaround.enabled":                   flagTrue,
+		"aro.autosizednodes.enable":                flagFalse,
 	}
 }

pkg/cluster/adminupdate_test.go

@@ -41,6 +41,7 @@ func TestAdminUpdateSteps(t *testing.T) {
 			},
 			shouldRunSteps: []string{
 				"[Action initializeKubernetesClients-fm]",
+				"[Action initializeOperatorDeployer-fm]",
 				"[Action ensureBillingRecord-fm]",
 				"[Action ensureDefaults-fm]",
 				"[Action fixupClusterSPObjectID-fm]",
@@ -61,6 +62,7 @@ func TestAdminUpdateSteps(t *testing.T) {
 			},
 			shouldRunSteps: []string{
 				"[Action initializeKubernetesClients-fm]",
+				"[Action initializeOperatorDeployer-fm]",
 				"[Action ensureBillingRecord-fm]",
 				"[Action ensureDefaults-fm]",
 				"[Action fixupClusterSPObjectID-fm]",
@@ -99,6 +101,7 @@ func TestAdminUpdateSteps(t *testing.T) {
 			},
 			shouldRunSteps: []string{
 				"[Action initializeKubernetesClients-fm]",
+				"[Action initializeOperatorDeployer-fm]",
 				"[Action ensureBillingRecord-fm]",
 				"[Action ensureDefaults-fm]",
 				"[Action fixupClusterSPObjectID-fm]",

pkg/cluster/arooperator.go

@@ -5,27 +5,34 @@ package cluster
 
 import (
 	"context"
-
-	"github.com/Azure/ARO-RP/pkg/operator/deploy"
 )
 
+func (m *manager) isIngressProfileAvailable() bool {
+	// We try to aqcuire the IngressProfiles data at frontend best effort enrichment time only.
+	// When we start deallocated VMs and wait for the API do become available again, we don't pick
+	// the information up, even though it would be available.
+	return len(m.doc.OpenShiftCluster.Properties.IngressProfiles) != 0
+}
+
 func (m *manager) ensureAROOperator(ctx context.Context) error {
-	dep, err := deploy.New(m.log, m.env, m.doc.OpenShiftCluster, m.arocli, m.extensionscli, m.kubernetescli)
-	if err != nil {
-		m.log.Errorf("cannot ensureAROOperator.New: %s", err.Error())
-		return err
+	//ensure the IngressProfile information is available from the cluster which is not the case when the cluster vms were freshly restarted.
+	if !m.isIngressProfileAvailable() {
+		m.log.Error("skip ensureAROOperator")
+		return nil
 	}
-	err = dep.CreateOrUpdate(ctx)
+
+	err := m.aroOperatorDeployer.CreateOrUpdate(ctx)
 	if err != nil {
 		m.log.Errorf("cannot ensureAROOperator.CreateOrUpdate: %s", err.Error())
 	}
 	return err
 }
 
 func (m *manager) aroDeploymentReady(ctx context.Context) (bool, error) {
-	dep, err := deploy.New(m.log, m.env, m.doc.OpenShiftCluster, m.arocli, m.extensionscli, m.kubernetescli)
-	if err != nil {
-		return false, err
+	if !m.isIngressProfileAvailable() {
+		m.log.Error("skip aroDeploymentReady")
+		// skip and don't retry
+		return true, nil
 	}
-	return dep.IsReady(ctx)
+	return m.aroOperatorDeployer.IsReady(ctx)
 }