Merge pull request #648 from opendevstack/feature/ghcr-images

Remove image wrappers

CHANGELOG.md

@@ -17,6 +17,7 @@ listed in the changelog.
 ### Changed
 
 - Node.js 18 is now the default for `ods-build-npm` task ([#585](https://github.com/opendevstack/ods-pipeline/issues/585))
+- Images used in tasks are now pulled directly from the GitHub registry. "Wrapping" the images in the OpenShift/K8s cluster is not required anymore. If tasks need to trust a private certificate, it needs to be present as a K8s secret, which will then be mounted as a file in the pods. To add the secret to an existing installation, pass `--private-cert <host>` to `./install.sh`. For more details, see [#621](https://github.com/opendevstack/ods-pipeline/issues/621).
 
 ### Fixed
 

Makefile

@@ -144,19 +144,3 @@ ifeq ($(strip $(namespace)),)
 endif
 	cd scripts && ./install-inside-kind.sh -n $(namespace)
 .PHONY: deploy
-
-##@ OpenShift
-
-start-ods-builds: ## Start builds for each ODS BuildConfig
-	oc start-build ods-package-image
-	oc start-build ods-finish
-	oc start-build ods-go-toolset
-	oc start-build ods-gradle-toolset
-	oc start-build ods-helm
-	oc start-build ods-node16-npm-toolset
-	oc start-build ods-node18-npm-toolset
-	oc start-build ods-pipeline-manager
-	oc start-build ods-python-toolset
-	oc start-build ods-sonar
-	oc start-build ods-start
-.PHONY: start-ods-builds

build/package/Dockerfile.gradle-toolset

@@ -37,13 +37,8 @@ COPY build/package/scripts/copy-build-if-cached.sh /usr/local/bin/copy-build-if-
 COPY build/package/scripts/copy-artifacts.sh /usr/local/bin/copy-artifacts
 COPY build/package/scripts/build-gradle.sh /usr/local/bin/build-gradle
 COPY build/package/scripts/supply-sonar-project-properties-default.sh /usr/local/bin/supply-sonar-project-properties-default
-COPY build/package/scripts/set-gradle-proxy.sh /usr/local/bin/set-gradle-proxy
-RUN chmod +x /usr/local/bin/build-gradle && \
-    chmod +x /usr/local/bin/cache-build && \
-    chmod +x /usr/local/bin/copy-build-if-cached && \
-    chmod +x /usr/local/bin/copy-artifacts && \
-    chmod +x /usr/local/bin/supply-sonar-project-properties-default && \
-    chmod +x /usr/local/bin/set-gradle-proxy
+COPY build/package/scripts/configure-gradle.sh /usr/local/bin/configure-gradle
+COPY build/package/scripts/configure-truststore.sh /usr/local/bin/configure-truststore
 
 # Add sonar-project.properties
 COPY build/package/sonar-project.properties.d/gradle.properties /usr/local/default-sonar-project.properties

build/package/Dockerfile.package-image

@@ -51,6 +51,9 @@ RUN echo -e "build:1:1000\nbuild:1002:64535" > /etc/subuid \
 # Install Trivy
 RUN curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin "v${TRIVY_VERSION}"
 
+# Add scripts
+COPY build/package/scripts/download-aqua-scanner.sh /usr/local/bin/download-aqua-scanner
+
 VOLUME /var/lib/containers
 VOLUME /home/build/.local/share/containers
 

build/package/Dockerfile.sonar

@@ -32,13 +32,16 @@ RUN cd /tmp \
 FROM registry.access.redhat.com/ubi8/ubi-minimal:8.4
 
 ENV SONAR_EDITION="community" \
-    OPENSSL_VERSION=1.1
+    OPENSSL_VERSION=1.1 \
+    JAVA_HOME=/usr/lib/jvm/jre-11
 
 RUN microdnf install --nodocs openssl-${OPENSSL_VERSION}* java-11-openjdk-headless which && microdnf clean all
 
 COPY --from=builder /usr/local/bin/sonar /usr/local/bin/sonar
 COPY --from=builder /usr/local/sonar-scanner-cli /usr/local/sonar-scanner-cli
 COPY --from=builder /usr/local/cnes/cnesreport.jar /usr/local/cnes/cnesreport.jar
+COPY build/package/scripts/configure-truststore.sh /usr/local/bin/configure-truststore
+
 ENV PATH=/usr/local/sonar-scanner-cli/bin:$PATH
 
 USER 1001

build/package/scripts/build-gradle.sh

@@ -43,8 +43,9 @@ echo "Using NEXUS_URL=$NEXUS_URL"
 echo "Using GRADLE_OPTS=$GRADLE_OPTS"
 echo "Using GRADLE_USER_HOME=$GRADLE_USER_HOME"
 echo "Using ARTIFACTS_DIR=$ARTIFACTS_DIR"
+mkdir -p "${GRADLE_USER_HOME}"
 
-set-gradle-proxy
+configure-gradle
 
 echo
 cd "${WORKING_DIR}"

build/package/scripts/build-npm.sh

@@ -65,17 +65,17 @@ if [ "${WORKING_DIR}" != "." ]; then
   ARTIFACT_PREFIX="${WORKING_DIR/\//-}-"
 fi
 
-echo "Configuring npm to use Nexus ..."
-# Remove the protocol segment from NEXUS_URL
-NEXUS_HOST=$(echo "${NEXUS_URL}" | sed -E 's/^\s*.*:\/\///g')
-if [ -n "${NEXUS_HOST}" ] && [ -n "${NEXUS_USERNAME}" ] && [ -n "${NEXUS_PASSWORD}" ]; then
+echo "Configuring npm to use Nexus (${NEXUS_URL}) ..."
+if [ -n "${NEXUS_URL}" ] && [ -n "${NEXUS_USERNAME}" ] && [ -n "${NEXUS_PASSWORD}" ]; then
     NEXUS_AUTH="$(urlencode "${NEXUS_USERNAME}"):$(urlencode "${NEXUS_PASSWORD}")"
     npm config set registry="$NEXUS_URL"/repository/npmjs/
     npm config set always-auth=true
     npm config set _auth="$(echo -n "$NEXUS_AUTH" | base64)"
     npm config set [email protected]
-    npm config set ca=null
-    npm config set strict-ssl=false
+    if [ -f /etc/ssl/certs/private-cert.pem ]; then
+      echo "Configuring private cert ..."
+      npm config set cafile=/etc/ssl/certs/private-cert.pem
+    fi
 fi;
 
 echo "package-*.json checks ..."

build/package/scripts/build-python.sh

@@ -55,7 +55,7 @@ if [ "${WORKING_DIR}" != "." ]; then
   ARTIFACT_PREFIX="${WORKING_DIR/\//-}-"
 fi
 
-echo "Configuring pip to use Nexus ..."
+echo "Configuring pip to use Nexus (${NEXUS_URL}) ..."
 # Remove the protocol segment from NEXUS_URL
 NEXUS_HOST=$(echo "${NEXUS_URL}" | sed -E 's/^\s*.*:\/\///g')
 if [ -n "${NEXUS_HOST}" ] && [ -n "${NEXUS_USERNAME}" ] && [ -n "${NEXUS_PASSWORD}" ]; then

build/package/scripts/set-gradle-proxy.sh → build/package/scripts/configure-gradle.sh

@@ -3,8 +3,20 @@
 # This script checks for env variable HTTP_PROXY and adds them to gradle.properties.
 CONTENT=""
 
-if [[ $HTTP_PROXY != "" ]]; then
+if [ -f /etc/ssl/certs/private-cert.pem ]; then
+  echo "Configuring Gradle to trust private cert ..."
+  configure-truststore --dest-store ".ods-cache/truststore/cacerts"
+  # shellcheck disable=SC2181
+  if [ $? -ne 0 ]; then
+    exit 1
+  fi
+  # Configure Gradle to use the modified trust store.
+  CONTENT+="systemProp.javax.net.ssl.trustStore=.ods-cache/keystore/cacerts\n"
+  CONTENT+="systemProp.javax.net.ssl.trustStorePassword=password\n"
+fi
 
+if [ "${HTTP_PROXY}" != "" ]; then
+	echo "Configuring Gradle to honor HTTP_PROXY ..."
 	proxy=$(echo "$HTTP_PROXY" | sed -e "s|https://||g" | sed -e "s|http://||g")
 	proxy_hostp=$(echo "$proxy" | cut -d "@" -f2)
 
@@ -32,7 +44,8 @@ if [[ $HTTP_PROXY != "" ]]; then
  	fi
 fi
 
-if [[ $NO_PROXY != "" ]]; then
+if [ "${NO_PROXY}" != "" ]; then
+	echo "Configuring Gradle to honor NO_PROXY ..."
 	# shellcheck disable=SC2001
 	noproxy_host=$(echo "$NO_PROXY" | sed -e 's|\,\.|\,\*\.|g')
 	# shellcheck disable=SC2001
@@ -41,6 +54,6 @@ if [[ $NO_PROXY != "" ]]; then
 	CONTENT+="systemProp.https.nonProxyHosts=$noproxy_host\n"
 fi
 
-if [[ $CONTENT != "" ]]; then
+if [ "${CONTENT}" != "" ]; then
   echo -e "$CONTENT" > "${GRADLE_USER_HOME}/gradle.properties"
 fi

build/package/scripts/configure-truststore.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+set -u
+
+md5_bin="${MD5_BIN:-"md5sum --tag"}"
+private_cert="/etc/ssl/certs/private-cert.pem"
+src_truststore="${JAVA_HOME}/lib/security/cacerts"
+src_pass="changeit"
+dest_pass="changeit"
+
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+
+    --src-store) src_truststore="$2"; shift;;
+    --src-store=*) src_truststore="${1#*=}";;
+
+    --src-storepass) src_pass="$2"; shift;;
+    --src-storepass=*) src_pass="${1#*=}";;
+
+    --dest-store) dest_truststore="$2"; shift;;
+    --dest-store=*) dest_truststore="${1#*=}";;
+
+    --dest-storepass) dest_pass="$2"; shift;;
+    --dest-storepass=*) dest_pass="${1#*=}";;
+
+    --debug) set -x; shift;;
+
+  *) echo "Unknown parameter passed: $1"; exit 1;;
+esac; shift; done
+
+dest_truststore_dir="${dest_truststore%/*}"
+mkdir -p "${dest_truststore_dir}"
+md5_private_cert_path="${dest_truststore_dir}/.md5-private-cert"
+md5_private_cert=$(${md5_bin} "${private_cert}")
+
+if [ ! -f "${dest_truststore}" ] || [ "${md5_private_cert}" != "$(cat "${md5_private_cert_path}")" ]; then
+    echo "Creating truststore with private cert ..."
+    # Copy global keystone to location where we can write to (hide output containing warnings).
+    keytool -importkeystore \
+        -srckeystore "${src_truststore}" -destkeystore "${dest_truststore}" \
+        -deststorepass "${dest_pass}" -srcstorepass "${src_pass}" &> keytool-output.txt
+    # shellcheck disable=SC2181
+    if [ $? -ne 0 ]; then
+        cat keytool-output.txt; exit 1
+    fi
+    # Trust private cert (hide output containing warnings).
+    keytool -importcert -noprompt -trustcacerts \
+        -alias private-cert -file "${private_cert}" \
+        -keystore "${dest_truststore}" -storepass "${dest_pass}" &> keytool-output.txt
+    # shellcheck disable=SC2181
+    if [ $? -ne 0 ]; then
+        cat keytool-output.txt; exit 1
+    fi
+    echo "${md5_private_cert}" > "${md5_private_cert_path}"
+fi

build/package/scripts/download-aqua-scanner.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+set -eu
+
+md5_bin="${MD5_BIN:-"md5sum --tag"}"
+aqua_scanner_url=""
+bin_dir=".ods-cache/bin"
+
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+
+    --bin-dir) bin_dir="$2"; shift;;
+    --bin-dir=*) bin_dir="${1#*=}";;
+
+    --aqua-scanner-url) aqua_scanner_url="$2"; shift;;
+    --aqua-scanner-url=*) aqua_scanner_url="${1#*=}";;
+
+    --debug) set -x; shift;;
+
+  *) echo "Unknown parameter passed: $1"; exit 1;;
+esac; shift; done
+
+aqua_scanner_path="${bin_dir}/aquasec"
+md5_aqua_scanner_url_path="${bin_dir}/.md5-aquasec"
+
+# Optionally install Aqua scanner.
+# If the binary already exists and was downloaded from the
+# URL given by aqua_scanner_url, skip download.
+if [ -n "${aqua_scanner_url}" ] && [ "${aqua_scanner_url}" != "none" ]; then
+    md5_aqua_scanner_url=$(${md5_bin} -s "${aqua_scanner_url}")
+    if [ ! -f "${md5_aqua_scanner_url_path}" ] || [ "${md5_aqua_scanner_url}" != "$(cat "${md5_aqua_scanner_url_path}")" ]; then
+        echo 'Installing Aqua scanner...'
+        curl -v -sSf -L "${aqua_scanner_url}" -o aquasec
+        mv aquasec "${aqua_scanner_path}"
+        chmod +x "${aqua_scanner_path}"
+        echo "${md5_aqua_scanner_url}" > "${md5_aqua_scanner_url_path}"
+        echo 'Installed Aqua scanner version:'
+        "${aqua_scanner_path}" version
+    fi
+fi

cmd/artifact-download/main.go

@@ -5,10 +5,14 @@
 //
 // There are two main modes of the program:
 // (1) users supply (OpenShift) namespace, (Bitbucket) project, (Git) repository
-//     and a tag such as "v1.0.0".
+//
+//	and a tag such as "v1.0.0".
+//
 // (2) users run this program from the root of a Git repository and only supply
-//     (OpenShift) namespace and tag=WIP. In this case the latest artifacts are
-//     downloaded.
+//
+//	(OpenShift) namespace and tag=WIP. In this case the latest artifacts are
+//	downloaded.
+//
 // Mode (1) is the main use case, mode (2) is provided as a convenience feature
 // for developers.
 package main
@@ -45,6 +49,7 @@ type options struct {
 	version         bool
 	tag             string
 	outputDirectory string
+	privateCert     string
 	debug           bool
 }
 
@@ -72,6 +77,7 @@ func main() {
 	flag.StringVar(&opts.repository, "repository", "", "Bitbucket repository key")
 	flag.StringVar(&opts.tag, "tag", "", "Git tag to retrieve artifacts for, e.g. v1.0.0 (required)")
 	flag.StringVar(&opts.outputDirectory, "output", "artifacts-out", "Directory to place outputs into")
+	flag.StringVar(&opts.privateCert, "private-cert", "", "Path to private certification (in PEM format)")
 	flag.BoolVar(&opts.debug, "debug", (os.Getenv("DEBUG") == "true"), "Enable debug mode")
 	flag.BoolVar(&opts.version, "version", false, "Display version of binary")
 	flag.Parse()
@@ -129,7 +135,7 @@ func main() {
 	}
 
 	// Bitbucket client
-	bcc, err := installation.NewBitbucketClientConfig(c, opts.namespace, logger)
+	bcc, err := installation.NewBitbucketClientConfig(c, opts.namespace, logger, opts.privateCert)
 	if err != nil {
 		log.Fatalf("Could not create Bitbucket client config: %s. Are you logged into the cluster?", err)
 	}

cmd/package-image/aqua.go

@@ -10,7 +10,7 @@ import (
 )
 
 const (
-	aquasecBin                           = "aquasec"
+	aquasecBin                           = "./.ods-cache/bin/aquasec"
 	scanComplianceFailureExitCode        = 4
 	scanLicenseValidationFailureExitCode = 5
 )

cmd/sonar/main.go

@@ -15,13 +15,26 @@ import (
 )
 
 type options struct {
-	sonarAuthToken string
-	sonarURL       string
-	sonarEdition   string
-	workingDir     string
-	rootPath       string
-	qualityGate    bool
-	debug          bool
+	sonarAuthToken     string
+	sonarURL           string
+	sonarEdition       string
+	workingDir         string
+	rootPath           string
+	qualityGate        bool
+	trustStore         string
+	trustStorePassword string
+	debug              bool
+}
+
+var defaultOptions = options{
+	sonarAuthToken:     os.Getenv("SONAR_AUTH_TOKEN"),
+	sonarURL:           os.Getenv("SONAR_URL"),
+	sonarEdition:       os.Getenv("SONAR_EDITION"),
+	workingDir:         ".",
+	qualityGate:        false,
+	trustStore:         "${JAVA_HOME}/lib/security/cacerts",
+	trustStorePassword: "changeit",
+	debug:              (os.Getenv("DEBUG") == "true"),
 }
 
 func main() {
@@ -31,12 +44,14 @@ func main() {
 	}
 
 	opts := options{rootPath: rootPath}
-	flag.StringVar(&opts.sonarAuthToken, "sonar-auth-token", os.Getenv("SONAR_AUTH_TOKEN"), "sonar-auth-token")
-	flag.StringVar(&opts.sonarURL, "sonar-url", os.Getenv("SONAR_URL"), "sonar-url")
-	flag.StringVar(&opts.sonarEdition, "sonar-edition", os.Getenv("SONAR_EDITION"), "sonar-edition")
-	flag.StringVar(&opts.workingDir, "working-dir", ".", "working directory")
-	flag.BoolVar(&opts.qualityGate, "quality-gate", false, "require quality gate pass")
-	flag.BoolVar(&opts.debug, "debug", (os.Getenv("DEBUG") == "true"), "debug mode")
+	flag.StringVar(&opts.sonarAuthToken, "sonar-auth-token", defaultOptions.sonarAuthToken, "sonar-auth-token")
+	flag.StringVar(&opts.sonarURL, "sonar-url", defaultOptions.sonarURL, "sonar-url")
+	flag.StringVar(&opts.sonarEdition, "sonar-edition", defaultOptions.sonarEdition, "sonar-edition")
+	flag.StringVar(&opts.workingDir, "working-dir", defaultOptions.workingDir, "working directory")
+	flag.BoolVar(&opts.qualityGate, "quality-gate", defaultOptions.qualityGate, "require quality gate pass")
+	flag.StringVar(&opts.trustStore, "truststore", defaultOptions.trustStore, "JKS truststore")
+	flag.StringVar(&opts.trustStorePassword, "truststore-pass", defaultOptions.trustStorePassword, "JKS truststore password")
+	flag.BoolVar(&opts.debug, "debug", defaultOptions.debug, "debug mode")
 	flag.Parse()
 
 	var logger logging.LeveledLoggerInterface
@@ -58,11 +73,13 @@ func main() {
 	}
 
 	sonarClient, err := sonar.NewClient(&sonar.ClientConfig{
-		APIToken:      opts.sonarAuthToken,
-		BaseURL:       opts.sonarURL,
-		ServerEdition: opts.sonarEdition,
-		Debug:         opts.debug,
-		Logger:        logger,
+		APIToken:           opts.sonarAuthToken,
+		BaseURL:            opts.sonarURL,
+		ServerEdition:      opts.sonarEdition,
+		TrustStore:         opts.trustStore,
+		TrustStorePassword: opts.trustStorePassword,
+		Debug:              opts.debug,
+		Logger:             logger,
 	})
 	if err != nil {
 		log.Fatal("sonar client:", err)
@@ -96,17 +113,17 @@ func sonarScan(
 			Base:   ctxt.PullRequestBase,
 		}
 	}
-	scanStdout, err := sonarClient.Scan(
+	err := sonarClient.Scan(
 		sonarProject,
 		ctxt.GitRef,
 		ctxt.GitCommitSHA,
 		prInfo,
+		os.Stdout,
+		os.Stdin,
 	)
 	if err != nil {
-		logger.Infof(scanStdout)
 		return fmt.Errorf("scan failed: %w", err)
 	}
-	logger.Infof(scanStdout)
 
 	logger.Infof("Wait until compute engine task finishes ...")
 	err = waitUntilComputeEngineTaskIsSuccessful(logger, sonarClient)