From 60526d58d667c40f9a56ee9012a5c62cee3d24ad Mon Sep 17 00:00:00 2001 From: RishabhSaini Date: Wed, 18 Sep 2024 09:15:05 -0400 Subject: [PATCH 1/3] buildah-build: Add proxy configuration while building the containerfile and pushing to the image registry --- pkg/controller/build/assets/buildah-build.sh | 7 +++++++ pkg/controller/build/build_controller.go | 7 +++++++ pkg/controller/build/image_build_request.go | 21 ++++++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/pkg/controller/build/assets/buildah-build.sh b/pkg/controller/build/assets/buildah-build.sh index 29c1a7fc01..7322e9c0b1 100644 --- a/pkg/controller/build/assets/buildah-build.sh +++ b/pkg/controller/build/assets/buildah-build.sh @@ -10,6 +10,10 @@ ETC_PKI_RPM_GPG_MOUNTPOINT="${ETC_PKI_RPM_GPG_MOUNTPOINT:-}" ETC_YUM_REPOS_D_MOUNTPOINT="${ETC_YUM_REPOS_D_MOUNTPOINT:-}" MAX_RETRIES="${MAX_RETRIES:-3}" +export HTTP_PROXY="${HTTP_PROXY:-}" +export HTTPS_PROXY="${HTTPS_PROXY:-}" +export NO_PROXY="${NO_PROXY:-}" + # Retry a command up to a specific number of times until it exits successfully. # Adapted from https://gist.github.com/sj26/88e1c6584397bb7c13bd11108a579746 function retry { @@ -43,6 +47,9 @@ build_args=( --authfile="$BASE_IMAGE_PULL_CREDS" --tag "$TAG" --file="$build_context/Dockerfile" + --build-arg HTTP_PROXY="$HTTP_PROXY" + --build-arg HTTPS_PROXY="$HTTPS_PROXY" + --build-arg NO_PROXY="$NO_PROXY" ) mount_opts="z,rw" diff --git a/pkg/controller/build/build_controller.go b/pkg/controller/build/build_controller.go index 7fa1de410c..71375f1c55 100644 --- a/pkg/controller/build/build_controller.go +++ b/pkg/controller/build/build_controller.go @@ -922,6 +922,13 @@ func (ctrl *Controller) getImagesConfig() (*ctrlcommon.Images, error) { func (ctrl *Controller) prepareForBuild(mosb *mcfgv1alpha1.MachineOSBuild, mosc *mcfgv1alpha1.MachineOSConfig) (ImageBuildRequest, error) { ibr := newImageBuildRequestFromBuildInputs(mosb, mosc) + cc, err := ctrl.ccLister.Get(ctrlcommon.ControllerConfigName) + if err != nil { + return ibr, fmt.Errorf("could not get controller config: %w", err) + } + + ibr.Proxy = cc.Spec.Proxy + imagesConfig, err := ctrl.getImagesConfig() if err != nil { return ibr, fmt.Errorf("could not get images.json config: %w", err) diff --git a/pkg/controller/build/image_build_request.go b/pkg/controller/build/image_build_request.go index 92452aa52e..98f17465a3 100644 --- a/pkg/controller/build/image_build_request.go +++ b/pkg/controller/build/image_build_request.go @@ -7,6 +7,7 @@ import ( "strings" "text/template" + configv1 "github.com/openshift/api/config/v1" mcfgv1 "github.com/openshift/api/machineconfiguration/v1" mcfgv1alpha1 "github.com/openshift/api/machineconfiguration/v1alpha1" ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" @@ -51,6 +52,8 @@ type ImageBuildRequest struct { HasEtcYumReposDConfigs bool // Has /etc/pki/rpm-gpg configs HasEtcPkiRpmGpgKeys bool + // Proxy Configurations + Proxy *configv1.ProxyStatus } // Constructs a simple ImageBuildRequest. @@ -171,6 +174,12 @@ func (i ImageBuildRequest) toBuildPod() *corev1.Pod { // the official Buildah image. // nolint:dupl // I don't want to deduplicate this yet since there are still some unknowns. func (i ImageBuildRequest) toBuildahPod() *corev1.Pod { + var httpProxy, httpsProxy, noProxy string + if i.Proxy != nil { + httpProxy = i.Proxy.HTTPProxy + httpsProxy = i.Proxy.HTTPSProxy + noProxy = i.Proxy.NoProxy + } env := []corev1.EnvVar{ // How many times the build / push steps should be retried. In the future, // this should be wired up to the MachineOSConfig or other higher-level @@ -214,6 +223,18 @@ func (i ImageBuildRequest) toBuildahPod() *corev1.Pod { Name: "BUILDAH_ISOLATION", Value: "chroot", }, + { + Name: "HTTP_PROXY", + Value: httpProxy, + }, + { + Name: "HTTPS_PROXY", + Value: httpsProxy, + }, + { + Name: "NO_PROXY", + Value: noProxy, + }, } var uid int64 = 1000 From 6d130a81a92dbcd6c15f5a7912d62d36d30a5df7 Mon Sep 17 00:00:00 2001 From: RishabhSaini Date: Fri, 20 Sep 2024 16:57:33 -0400 Subject: [PATCH 2/3] Containerfile: Add proxy configs trustedCA to the Trust Anchors of the system wide trust store buildah-build: Add the mounted additional-trust-bundle to the build context of buildah build_controller: Stored Additional Trust Bundle as a ConfigMap image_build_request: Add the Additional Trust Bundle Config Map as a volume to the Buildah pod and mount it When using a cluster wide Proxy Configuration, the trustedCA needs to be added to the system wide trust store in /etc/pki/ca-trust/source, hence enabling a use case where self-signed certificates are used to download a package from a YUM repo. It will be successfully validated by DNF using the proxy. --- .../Containerfile.on-cluster-build-template | 3 ++ pkg/controller/build/assets/buildah-build.sh | 13 ++++++- pkg/controller/build/build_controller.go | 9 +++++ pkg/controller/build/image_build_request.go | 35 +++++++++++++++++++ 4 files changed, 59 insertions(+), 1 deletion(-) diff --git a/pkg/controller/build/assets/Containerfile.on-cluster-build-template b/pkg/controller/build/assets/Containerfile.on-cluster-build-template index 49471ab139..3d26712e22 100644 --- a/pkg/controller/build/assets/Containerfile.on-cluster-build-template +++ b/pkg/controller/build/assets/Containerfile.on-cluster-build-template @@ -10,6 +10,9 @@ COPY ./machineconfig/machineconfig.json.gz /tmp/machineconfig.json.gz RUN mkdir -p /etc/machine-config-daemon && \ cat /tmp/machineconfig.json.gz | base64 -d | gunzip - > /etc/machine-config-daemon/currentconfig +COPY ./openshift-config-user-ca-bundle.crt /etc/pki/ca-trust/source/anchors/openshift-config-user-ca-bundle.crt +RUN update-ca-trust extract + {{if .MachineOSConfig.Spec.BuildInputs.BaseOSExtensionsImagePullspec}} # Pull our extensions image. Not sure yet what / how this should be wired up # though. Ideally, I'd like to use some Buildah tricks to have the extensions diff --git a/pkg/controller/build/assets/buildah-build.sh b/pkg/controller/build/assets/buildah-build.sh index 7322e9c0b1..e7de16b4cb 100644 --- a/pkg/controller/build/assets/buildah-build.sh +++ b/pkg/controller/build/assets/buildah-build.sh @@ -37,9 +37,20 @@ build_context="$HOME/context" # Create a directory to hold our build context. mkdir -p "$build_context/machineconfig" -# Copy the Dockerfile and Machineconfigs from configmaps into our build context. +# Copy the Dockerfile, Machineconfigs and AdditionalTrustBundle from configmaps into our build context. cp /tmp/dockerfile/Dockerfile "$build_context" cp /tmp/machineconfig/machineconfig.json.gz "$build_context/machineconfig/" +cp /etc/pki/ca-trust/source/anchors/openshift-config-user-ca-bundle.crt "$build_context" + +DEST=/etc/pki/ca-trust/extracted + +# Prevent p11-kit from reading user configuration files. +export P11_KIT_NO_USER_CONFIG=1 + +# OpenSSL PEM bundle that includes trust flags +/usr/bin/p11-kit extract --format=openssl-bundle --filter=certificates --overwrite --comment $DEST/openssl/ca-bundle.trust.crt +/usr/bin/p11-kit extract --format=pem-bundle --filter=ca-anchors --overwrite --comment --purpose server-auth $DEST/pem/tls-ca-bundle.pem + build_args=( --log-level=DEBUG diff --git a/pkg/controller/build/build_controller.go b/pkg/controller/build/build_controller.go index 71375f1c55..42193b1e8d 100644 --- a/pkg/controller/build/build_controller.go +++ b/pkg/controller/build/build_controller.go @@ -928,6 +928,7 @@ func (ctrl *Controller) prepareForBuild(mosb *mcfgv1alpha1.MachineOSBuild, mosc } ibr.Proxy = cc.Spec.Proxy + ibr.AdditionalTrustBundle = cc.Spec.AdditionalTrustBundle imagesConfig, err := ctrl.getImagesConfig() if err != nil { @@ -1006,6 +1007,14 @@ func (ctrl *Controller) prepareForBuild(mosb *mcfgv1alpha1.MachineOSBuild, mosc klog.Infof("Stored Dockerfile for build %s in ConfigMap %s for build", ibr.getBuildName(), dockerfileConfigMap.Name) + additionalTrustBundleConfigMap := ibr.additionalTrustBundleToConfigMap() + _, err = ctrl.kubeclient.CoreV1().ConfigMaps(ctrlcommon.MCONamespace).Create(context.TODO(), additionalTrustBundleConfigMap, metav1.CreateOptions{}) + if err != nil { + return ImageBuildRequest{}, fmt.Errorf("could not load rendered Additional Trust Bundle %s into configmap: %w", additionalTrustBundleConfigMap.Name, err) + } + + klog.Infof("Stored Additional Trust Bundle for build %s in ConfigMap %s for build", ibr.getBuildName(), additionalTrustBundleConfigMap.Name) + return ibr, nil } diff --git a/pkg/controller/build/image_build_request.go b/pkg/controller/build/image_build_request.go index 98f17465a3..956683a210 100644 --- a/pkg/controller/build/image_build_request.go +++ b/pkg/controller/build/image_build_request.go @@ -54,6 +54,8 @@ type ImageBuildRequest struct { HasEtcPkiRpmGpgKeys bool // Proxy Configurations Proxy *configv1.ProxyStatus + // Additional trust bundles for proxy (user defined) + AdditionalTrustBundle []byte } // Constructs a simple ImageBuildRequest. @@ -110,6 +112,19 @@ func (i ImageBuildRequest) dockerfileToConfigMap() (*corev1.ConfigMap, error) { return configmap, nil } +// Gets the Additional Trust Bundle and injects it into a ConfigMap for consumption by the image builder. +func (i ImageBuildRequest) additionalTrustBundleToConfigMap() *corev1.ConfigMap { + configmap := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: i.getObjectMeta(i.getAdditionalTrustBundleConfigMapName()), + BinaryData: map[string][]byte{ + "openshift-config-user-ca-bundle.crt": i.AdditionalTrustBundle, + }, + } + + return configmap +} + // Stuffs a given MachineConfig into a ConfigMap, gzipping and base64-encoding it. func (i ImageBuildRequest) toConfigMap(mc *mcfgv1.MachineConfig) (*corev1.ConfigMap, error) { out, err := json.Marshal(mc) @@ -256,6 +271,10 @@ func (i ImageBuildRequest) toBuildahPod() *corev1.Pod { Name: "dockerfile", MountPath: "/tmp/dockerfile", }, + { + Name: "additional-trust-bundle", + MountPath: "/etc/pki/ca-trust/source/anchors", + }, { Name: "base-image-pull-creds", MountPath: "/tmp/base-image-pull-creds", @@ -294,6 +313,17 @@ func (i ImageBuildRequest) toBuildahPod() *corev1.Pod { }, }, }, + { + // Provides the user defined Additional Trust Bundle + Name: "additional-trust-bundle", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: i.getAdditionalTrustBundleConfigMapName(), + }, + }, + }, + }, { // Provides the credentials needed to pull the base OS image. Name: "base-image-pull-creds", @@ -521,6 +551,11 @@ func (i ImageBuildRequest) getObjectMeta(name string) metav1.ObjectMeta { } } +// Computes the AdditionalTrustBundle ConfigMap name based upon the MachineConfigPool name. +func (i ImageBuildRequest) getAdditionalTrustBundleConfigMapName() string { + return fmt.Sprintf("additionaltrustbundle-%s", i.MachineOSBuild.Spec.DesiredConfig.Name) +} + // Computes the Dockerfile ConfigMap name based upon the MachineConfigPool name. func (i ImageBuildRequest) getDockerfileConfigMapName() string { return fmt.Sprintf("dockerfile-%s", i.MachineOSBuild.Spec.DesiredConfig.Name) From 621f441c4957efc9bb694f2224527b4634ad6bb1 Mon Sep 17 00:00:00 2001 From: RishabhSaini Date: Thu, 26 Sep 2024 15:57:12 -0400 Subject: [PATCH 3/3] use the ConfigMap CA injector for trusted CA bundles --- .../Containerfile.on-cluster-build-template | 3 +- pkg/controller/build/assets/buildah-build.sh | 12 +------ pkg/controller/build/build_controller.go | 9 ------ pkg/controller/build/image_build_request.go | 31 ++++--------------- pkg/controller/common/constants.go | 2 ++ pkg/operator/sync.go | 24 ++++++++++++++ 6 files changed, 34 insertions(+), 47 deletions(-) diff --git a/pkg/controller/build/assets/Containerfile.on-cluster-build-template b/pkg/controller/build/assets/Containerfile.on-cluster-build-template index 3d26712e22..65f7d6b35c 100644 --- a/pkg/controller/build/assets/Containerfile.on-cluster-build-template +++ b/pkg/controller/build/assets/Containerfile.on-cluster-build-template @@ -10,8 +10,7 @@ COPY ./machineconfig/machineconfig.json.gz /tmp/machineconfig.json.gz RUN mkdir -p /etc/machine-config-daemon && \ cat /tmp/machineconfig.json.gz | base64 -d | gunzip - > /etc/machine-config-daemon/currentconfig -COPY ./openshift-config-user-ca-bundle.crt /etc/pki/ca-trust/source/anchors/openshift-config-user-ca-bundle.crt -RUN update-ca-trust extract +COPY ./tls-ca-bundle.pem /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem {{if .MachineOSConfig.Spec.BuildInputs.BaseOSExtensionsImagePullspec}} # Pull our extensions image. Not sure yet what / how this should be wired up diff --git a/pkg/controller/build/assets/buildah-build.sh b/pkg/controller/build/assets/buildah-build.sh index e7de16b4cb..613e880a3c 100644 --- a/pkg/controller/build/assets/buildah-build.sh +++ b/pkg/controller/build/assets/buildah-build.sh @@ -40,17 +40,7 @@ mkdir -p "$build_context/machineconfig" # Copy the Dockerfile, Machineconfigs and AdditionalTrustBundle from configmaps into our build context. cp /tmp/dockerfile/Dockerfile "$build_context" cp /tmp/machineconfig/machineconfig.json.gz "$build_context/machineconfig/" -cp /etc/pki/ca-trust/source/anchors/openshift-config-user-ca-bundle.crt "$build_context" - -DEST=/etc/pki/ca-trust/extracted - -# Prevent p11-kit from reading user configuration files. -export P11_KIT_NO_USER_CONFIG=1 - -# OpenSSL PEM bundle that includes trust flags -/usr/bin/p11-kit extract --format=openssl-bundle --filter=certificates --overwrite --comment $DEST/openssl/ca-bundle.trust.crt -/usr/bin/p11-kit extract --format=pem-bundle --filter=ca-anchors --overwrite --comment --purpose server-auth $DEST/pem/tls-ca-bundle.pem - +cp /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem "$build_context" build_args=( --log-level=DEBUG diff --git a/pkg/controller/build/build_controller.go b/pkg/controller/build/build_controller.go index 42193b1e8d..71375f1c55 100644 --- a/pkg/controller/build/build_controller.go +++ b/pkg/controller/build/build_controller.go @@ -928,7 +928,6 @@ func (ctrl *Controller) prepareForBuild(mosb *mcfgv1alpha1.MachineOSBuild, mosc } ibr.Proxy = cc.Spec.Proxy - ibr.AdditionalTrustBundle = cc.Spec.AdditionalTrustBundle imagesConfig, err := ctrl.getImagesConfig() if err != nil { @@ -1007,14 +1006,6 @@ func (ctrl *Controller) prepareForBuild(mosb *mcfgv1alpha1.MachineOSBuild, mosc klog.Infof("Stored Dockerfile for build %s in ConfigMap %s for build", ibr.getBuildName(), dockerfileConfigMap.Name) - additionalTrustBundleConfigMap := ibr.additionalTrustBundleToConfigMap() - _, err = ctrl.kubeclient.CoreV1().ConfigMaps(ctrlcommon.MCONamespace).Create(context.TODO(), additionalTrustBundleConfigMap, metav1.CreateOptions{}) - if err != nil { - return ImageBuildRequest{}, fmt.Errorf("could not load rendered Additional Trust Bundle %s into configmap: %w", additionalTrustBundleConfigMap.Name, err) - } - - klog.Infof("Stored Additional Trust Bundle for build %s in ConfigMap %s for build", ibr.getBuildName(), additionalTrustBundleConfigMap.Name) - return ibr, nil } diff --git a/pkg/controller/build/image_build_request.go b/pkg/controller/build/image_build_request.go index 956683a210..07ec344f92 100644 --- a/pkg/controller/build/image_build_request.go +++ b/pkg/controller/build/image_build_request.go @@ -54,8 +54,6 @@ type ImageBuildRequest struct { HasEtcPkiRpmGpgKeys bool // Proxy Configurations Proxy *configv1.ProxyStatus - // Additional trust bundles for proxy (user defined) - AdditionalTrustBundle []byte } // Constructs a simple ImageBuildRequest. @@ -112,19 +110,6 @@ func (i ImageBuildRequest) dockerfileToConfigMap() (*corev1.ConfigMap, error) { return configmap, nil } -// Gets the Additional Trust Bundle and injects it into a ConfigMap for consumption by the image builder. -func (i ImageBuildRequest) additionalTrustBundleToConfigMap() *corev1.ConfigMap { - configmap := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: i.getObjectMeta(i.getAdditionalTrustBundleConfigMapName()), - BinaryData: map[string][]byte{ - "openshift-config-user-ca-bundle.crt": i.AdditionalTrustBundle, - }, - } - - return configmap -} - // Stuffs a given MachineConfig into a ConfigMap, gzipping and base64-encoding it. func (i ImageBuildRequest) toConfigMap(mc *mcfgv1.MachineConfig) (*corev1.ConfigMap, error) { out, err := json.Marshal(mc) @@ -272,8 +257,8 @@ func (i ImageBuildRequest) toBuildahPod() *corev1.Pod { MountPath: "/tmp/dockerfile", }, { - Name: "additional-trust-bundle", - MountPath: "/etc/pki/ca-trust/source/anchors", + Name: "openshift-config-managed-trusted-ca-bundle", + MountPath: "/etc/pki/ca-trust/extracted/pem", }, { Name: "base-image-pull-creds", @@ -314,13 +299,14 @@ func (i ImageBuildRequest) toBuildahPod() *corev1.Pod { }, }, { - // Provides the user defined Additional Trust Bundle - Name: "additional-trust-bundle", + // Provides the trusted-ca-bundle (merge of system and proxy trust bundle extracted) + Name: "openshift-config-managed-trusted-ca-bundle", VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: i.getAdditionalTrustBundleConfigMapName(), + Name: ctrlcommon.TrustedCABundleConfigMapName, }, + Items: []corev1.KeyToPath{{Key: "ca-bundle.crt", Path: "tls-ca-bundle.pem"}}, }, }, }, @@ -551,11 +537,6 @@ func (i ImageBuildRequest) getObjectMeta(name string) metav1.ObjectMeta { } } -// Computes the AdditionalTrustBundle ConfigMap name based upon the MachineConfigPool name. -func (i ImageBuildRequest) getAdditionalTrustBundleConfigMapName() string { - return fmt.Sprintf("additionaltrustbundle-%s", i.MachineOSBuild.Spec.DesiredConfig.Name) -} - // Computes the Dockerfile ConfigMap name based upon the MachineConfigPool name. func (i ImageBuildRequest) getDockerfileConfigMapName() string { return fmt.Sprintf("dockerfile-%s", i.MachineOSBuild.Spec.DesiredConfig.Name) diff --git a/pkg/controller/common/constants.go b/pkg/controller/common/constants.go index 3a55246441..da14be2f51 100644 --- a/pkg/controller/common/constants.go +++ b/pkg/controller/common/constants.go @@ -95,4 +95,6 @@ const ( MachineConfigOperatorImagesConfigMapName string = "machine-config-operator-images" // The name of the machine-config-osimageurl ConfigMap. MachineConfigOSImageURLConfigMapName string = "machine-config-osimageurl" + // The name of the openshift-config-managed-trusted-ca-bundle ConfigMap. + TrustedCABundleConfigMapName string = "openshift-config-managed-trusted-ca-bundle" ) diff --git a/pkg/operator/sync.go b/pkg/operator/sync.go index 920c5c67cc..de8afc076e 100644 --- a/pkg/operator/sync.go +++ b/pkg/operator/sync.go @@ -574,6 +574,30 @@ func (optr *Operator) syncRenderConfig(_ *renderConfig, _ *configv1.ClusterOpera return fmt.Errorf("configmap %s/%s doesn't have a valid PEM bundle", "openshift-config", proxy.Spec.TrustedCA.Name) } trustBundle = append(trustBundle, proxyTrustBundle...) + + // Ensure the extracted and merged(system+proxy) trusted CA bundles for the system exists as a ConfigMap + _, err = optr.kubeClient.CoreV1().ConfigMaps(ctrlcommon.MCONamespace).Get(context.TODO(), ctrlcommon.TrustedCABundleConfigMapName, metav1.GetOptions{}) + if err != nil && apierrors.IsNotFound(err) { + // Relies on ConfigMap CA Injector from the Cluster Network Operator to populate the ca-bundle.crt + // field with the merged system and proxy trust bundles + openshiftConfigManagedTrustedCABundle := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ctrlcommon.MCONamespace, + Name: ctrlcommon.TrustedCABundleConfigMapName, + Labels: map[string]string{ + "config.openshift.io/inject-trusted-cabundle": "true", + }, + }, + } + _, err = optr.kubeClient.CoreV1().ConfigMaps(ctrlcommon.MCONamespace).Create(context.TODO(), openshiftConfigManagedTrustedCABundle, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("could not create %s configmap: %w", openshiftConfigManagedTrustedCABundle.Name, err) + } + + } + if err != nil { + return fmt.Errorf("could not get %s configmap: %w", ctrlcommon.TrustedCABundleConfigMapName, err) + } } } spec.AdditionalTrustBundle = trustBundle