WIP: Update test images for ARM and OCP 3.11

client/serviceinterface_update_test.go

@@ -130,7 +130,7 @@ var httpDeployment *appsv1.Deployment = &appsv1.Deployment{
 				Containers: []corev1.Container{
 					{
 						Name:            "nginx",
-						Image:           "quay.io/skupper/nginx-unprivileged",
+						Image:           "quay.io/nginx/nginx-unprivileged",
 						ImagePullPolicy: corev1.PullIfNotPresent,
 						Ports: []corev1.ContainerPort{
 							{

pkg/domain/podman/main_test.go

@@ -32,7 +32,7 @@ var (
 const (
 	ENV_PODMAN_ENDPOINT = "SKUPPER_TEST_PODMAN_ENDPOINT"
 	NS                  = "podman-system-test"
-	NGINX_IMAGE         = "quay.io/skupper/nginx-unprivileged:stable-alpine"
+	NGINX_IMAGE         = "quay.io/nginx/nginx-unprivileged:stable-alpine"
 )
 
 // newBasicSite returns a new instance of a basic Site instance

pkg/domain/podman/site_test.go

@@ -460,7 +460,7 @@ func mockContainers() []*container.Container {
 		}, {
 			ID:     strings.Replace(uuid.New().String(), "-", "", -1),
 			Name:   "nginx",
-			Image:  "docker.io/nginxinc/nginx-unprivileged:stable-alpine",
+			Image:  "quay.io/nginx/nginx-unprivileged:stable-alpine",
 			Labels: map[string]string{},
 			Networks: map[string]container.ContainerNetworkInfo{
 				"skupper": {

test/images/Containerfile.hey

@@ -0,0 +1,22 @@
+ARG TARGETPLATFORM
+
+FROM --platform=$TARGETPLATFORM golang:1.21 AS builder
+
+ARG TARGETOS
+ARG TARGETARCH
+
+WORKDIR /go/src/app
+RUN git clone https://github.com/rakyll/hey.git ./
+RUN GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o hey ./hey.go
+
+FROM --platform=$TARGETPLATFORM registry.access.redhat.com/ubi9-minimal
+
+# Create user and group and switch to user's context
+RUN microdnf -y install shadow-utils \
+&& microdnf clean all
+RUN useradd --uid 10000 runner
+USER 10000
+
+WORKDIR /app
+COPY --from=builder /go/src/app/hey .
+CMD ["/app/hey"]

test/images/Containerfile.iperf3

@@ -0,0 +1,18 @@
+ARG TARGETPLATFORM
+
+FROM --platform=$TARGETPLATFORM quay.io/fedora/fedora-minimal
+
+LABEL description="A supporting test image"
+LABEL maintainer="Skupper project"
+LABEL origin=https://github.com/skupperproject/skupper/blob/main/test/images
+
+EXPOSE 5201
+
+# Create user and group and switch to user's context
+RUN microdnf -y install iperf3 shadow-utils \
+&& microdnf clean all
+RUN useradd --uid 10000 runner
+USER 10000
+
+CMD []
+ENTRYPOINT ["/usr/bin/iperf3"]

test/images/Dockerfile.nghttp2 → test/images/Containerfile.nghttp2

@@ -1,4 +1,6 @@
-FROM alpine:latest
+ARG TARGETPLATFORM
+
+FROM --platform=$TARGETPLATFORM alpine:latest
 
 LABEL description="A supporting test image containing nghttp2 (client, server and load tester), for Skupper testing.  By default, this image does not start any nghttp2 project; the user needs to overwrite CMD with the command they want"
 LABEL maintainer="Skupper project"

test/images/Containerfile.wrk

@@ -0,0 +1,17 @@
+ARG TARGETPLATFORM
+
+FROM --platform=$TARGETPLATFORM registry.fedoraproject.org/fedora-minimal AS build
+
+RUN microdnf -y install gcc git make unzip openssl-devel && microdnf -y clean all
+
+WORKDIR /home/root
+RUN git clone https://github.com/wg/wrk.git
+
+WORKDIR /home/root/wrk
+RUN make WITH_OPENSSL=/usr
+
+FROM registry.fedoraproject.org/fedora-minimal
+
+COPY --from=build /home/root/wrk/wrk /usr/bin/wrk
+
+CMD ["/usr/bin/wrk"]

test/images/Containerfile.wrk2

@@ -0,0 +1,13 @@
+ARG TARGETPLATFORM
+
+FROM --platform=$TARGETPLATFORM alpine:latest as builder
+
+RUN apk add --update alpine-sdk openssl-dev zlib-dev && apk add --no-cache git && git clone https://github.com/giltene/wrk2.git && cd wrk2 && make && mv wrk /bin/
+
+
+FROM alpine:latest
+
+RUN apk add --update libgcc openssl zlib
+COPY --from=builder /bin/wrk /bin/
+
+CMD ["/bin/sh"]

test/images/Makefile

@@ -1,8 +1,8 @@
 # This make file provides three operations:
 #
 # - Copy a list of images (hardcoded below) to a main repository (by default,
-#   quay.io/skupper)
-# - Generate and push images to that same repository, based on Dockerfiles
+#   quay.io/skupper), possibly modifying and reassembling them
+# - Generate and push images to that same repository, based on Containerfiles
 # - Copy both lists of images above (external and generated) to another,
 #   configurable repository
 #
@@ -12,19 +12,22 @@
 # the MAIN_REPO:
 #
 #   make docker.io/svagi/nghttp2
+#   make docker.io/nginxinc/nginx-unprivileged
 #   make nghttp2
 #
-# The possible values are those listed on EXTERNAL_IMAGES and GENERATED_IMAGES.
+# The possible values are those listed on EXTERNAL_IMAGES, REASSEMBLE_IMAGES
+# and GENERATED_IMAGES.
 #
 # You can also only build an image (and not push it).  Just add _build to the
 # image name listed in GENERATED_IMAGES:
 #
 #   make nghttp2_build
 #
-# Alternativelly, one can build/push or copy all generated, all external, or
-# all of them:
+# Alternativelly, one can build/push or copy all generated, all external, all
+# reassembled, or all of them:
 #
 #   make external_images
+#   make reassembled_images
 #   make generated_images
 #   make everything
 #
@@ -59,6 +62,7 @@
 #
 #   make COPY_REPO=192.168.0.1:5000/internal nghttp2_copy
 #   make COPY_REPO=192.168.0.1:5000/internal docker.io/ssorj/quiver_copy
+#   make COPY_REPO=192.168.0.1:5000/internal docker.io/nginxinc/nginx-unprivileged_copy
 #
 # This Makefile is intended to be executed manually, with the login of
 # individual users, and not by a robot account on a CI.
@@ -80,6 +84,7 @@ SKOPEO = skopeo
 # That is required for their use with Openshift 3.11
 FORMAT_OPTIONS = --format docker
 TRANSFORM_OPTIONS = --format v2s2
+PLATFORM = linux/amd64,linux/arm64
 
 # Repositories
 MAIN_REPO = quay.io/skupper
@@ -90,22 +95,43 @@ COPY_REPO = localhost:5000/local
 # (such as in docker.io/library/mongo\:5.0).  Otherwise, you'll get an error
 # like `multiple target patterns.  Stop.`
 EXTERNAL_IMAGES := \
-	docker.io/nginxinc/nginx-unprivileged \
 	docker.io/ssorj/quiver \
-	docker.io/library/postgres \
-	docker.io/library/redis \
+	docker.io/library/postgres\:9.5 \
 	docker.io/maistra/examples-bookinfo-productpage-v1\:0.12.0 \
 	docker.io/maistra/examples-bookinfo-ratings-v1\:0.12.0 \
 	docker.io/maistra/examples-bookinfo-details-v1\:0.12.0 \
 	docker.io/maistra/examples-bookinfo-reviews-v3\:0.12.0 \
-	docker.io/library/mongo\:5.0 \
+	docker.io/library/mongo\:5.0
+
+# These manifest lists require special handling; they contain 'images' with
+# os/arch/variant "unknown" and the following entry in LayersData.Annotations:
+#
+#    in-toto.io/predicate-type": "https://slsa.dev/provenance/v0.2
+#
+# That makes a plain skopeo copy --all --format v2s2 to fail with an error
+# like the one below:
+#
+#    copying image 8/14 from manifest list: creating an updated image manifest: Unknown media type during manifest conversion: "application/vnd.in-toto+json"
+#
+# For that reason, we need to first copy the image locally, change its
+# manifest, and only then copy it to its destination.
+REASSEMBLE_IMAGES := \
+	docker.io/library/postgres \
+	docker.io/library/redis \
+	docker.io/nginxinc/nginx-unprivileged \
 	docker.io/nginxinc/nginx-unprivileged\:stable-alpine
 
 # These are the images that are generated by this Makefile (as opposed to
 # created elsewhere and just copied here).  To add a new image, simply create
-# Dockerfile.IMAGENAME and add the IMAGENAME below.
+# Containerfile.IMAGENAME and add the IMAGENAME below.
 GENERATED_IMAGES := \
-	nghttp2
+	hey \
+	iperf3 \
+	nghttp2 \
+	wrk
+
+	# wrk2 does not support ARM: https://github.com/giltene/wrk2/issues/104
+	# wrk2
 
 # --- end of configuration ---
 
@@ -117,6 +143,7 @@ generated_build := $(patsubst %,%_build,$(GENERATED_IMAGES))
 # The copy target names, to be used with the copy operation
 generated_copy := $(patsubst %,%_copy,$(GENERATED_IMAGES))
 external_copy   := $(patsubst %,%_copy,$(EXTERNAL_IMAGES))
+reassembled_copy := $(patsubst %,%_copy,$(REASSEMBLE_IMAGES))
 
 # We don't want someone to just run `make` and start copying stuff
 # around, so instead the default operation is to just give some
@@ -129,21 +156,25 @@ all:
 	@echo Normal use, however, is to select individual targets.
 	@echo Check Makefile contents for documentation.
 
-# For a generated image, we simply have a Dockerfile named after it and call
+# For a generated image, we simply have a Containerfile named after it and call
 # `podman build` on it, tagging it on the MAIN_REPO
 $(generated_build): TARGET = $(patsubst %_build,%,$@)
 $(generated_build):
-	$(PODMAN) build $(FORMAT_OPTIONS) --file Dockerfile.$(TARGET) --tag $(MAIN_REPO)/$(TARGET)
+	# First, we need to remove any existing manifest, as the command below is additive
+	# (it would append builds to an existing manifest)
+	-$(PODMAN) manifest rm $(MAIN_REPO)/$(TARGET)
+	$(PODMAN) build --platform $(PLATFORM) --no-cache $(FORMAT_OPTIONS) --file Containerfile.$(TARGET) --manifest $(MAIN_REPO)/$(TARGET)
 
-# This is generic the push target, for images built here (GENERATED_IMAGES)
+# This is the generic push target, for images built here (GENERATED_IMAGES)
 # Each image depends on its respective _build target
 %: %_build
-	$(PODMAN) push $(MAIN_REPO)/$@
+	$(PODMAN) manifest push $(MAIN_REPO)/$@
 
 # Shortcut targets
 external_images: $(EXTERNAL_IMAGES)
+reassembled_images: $(REASSEMBLE_IMAGES)
 generated_images: $(GENERATED_IMAGES)
-everything: external_images generated_images
+everything: external_images generated_images reassembled_images
 
 # This is the main target for the external images; it copies them from their
 # original locations _into_ MAIN_REPO.  Do not confuse this for the `copy` operation
@@ -153,28 +184,62 @@ everything: external_images generated_images
 # the last part of the URL
 $(EXTERNAL_IMAGES): TARGET = $(shell echo "$@" | sed s_.*/__ )
 $(EXTERNAL_IMAGES):
-	$(SKOPEO) copy $(TRANSFORM_OPTIONS) \
+	$(SKOPEO) copy --all $(TRANSFORM_OPTIONS) \
 		docker://$@ \
 		docker://$(MAIN_REPO)/$(TARGET)
 
+# This is the main target for the reassembled images; it copies them from their
+# original locations to a temporary directory, modifies the manifest.json and then
+# copies them _into_ MAIN_REPO.
+#
+# The first line prepares a target-local TARGET variable, which contains only
+# the last part of the URL
+$(REASSEMBLE_IMAGES): TARGET = $(shell echo "$@" | sed s_.*/__ )
+$(REASSEMBLE_IMAGES): REASSEMBLE_TMPDIR ::= /tmp/$(shell mktemp -d image-reassembly-XXX )
+$(REASSEMBLE_IMAGES):
+	$(SKOPEO) copy --all \
+		docker://$@ \
+		dir://$(REASSEMBLE_TMPDIR)
+
+	ls $(REASSEMBLE_TMPDIR)
+
+	jq \
+		'(.manifests[] | select (.platform.os == "unknown")) |= del (.) | del (..|nulls)' \
+		$(REASSEMBLE_TMPDIR)/manifest.json > $(REASSEMBLE_TMPDIR)/manifest.json.new
+
+	mv $(REASSEMBLE_TMPDIR)/manifest.json.new $(REASSEMBLE_TMPDIR)/manifest.json
+
+	$(SKOPEO) copy --all $(TRANSFORM_OPTIONS) \
+		dir://$(REASSEMBLE_TMPDIR) \
+		docker://$(MAIN_REPO)/$(TARGET)
+
+	rm -rf $(REASSEMBLE_TMPDIR)
+
 #
 # Copy operation
 #
-copy: $(generated_copy) $(external_copy)
+copy: $(generated_copy) $(external_copy) $(reassembled_copy)
 
 # The targets for external and generated images are basically the same; the
 # only difference is the way the original/main target needs to be manipulated
 # to generate the image name+tag (saved in the target-local variable TARGET).
 
 $(generated_copy): TARGET = $(patsubst %_copy,%,$@)
 $(generated_copy):
-	$(SKOPEO) copy $(TRANSFORM_OPTIONS) \
+	$(SKOPEO) copy --all $(TRANSFORM_OPTIONS) \
 		docker://$(MAIN_REPO)/$(TARGET) \
 		docker://$(COPY_REPO)/$(TARGET)
 
 # Remove _copy, but also everything till the last /
 $(external_copy): TARGET = $(shell echo "$@" | sed -e "s/_copy$$//" -e 's_.*/__' )
 $(external_copy):
-	$(SKOPEO) copy $(TRANSFORM_OPTIONS) \
+	$(SKOPEO) copy --all $(TRANSFORM_OPTIONS) \
+		docker://$(MAIN_REPO)/$(TARGET) \
+		docker://$(COPY_REPO)/$(TARGET)
+
+# Remove _copy, but also everything till the last /
+$(reassembled_copy): TARGET = $(shell echo "$@" | sed -e "s/_copy$$//" -e 's_.*/__' )
+$(reassembled_copy):
+	$(SKOPEO) copy --all $(TRANSFORM_OPTIONS) \
 		docker://$(MAIN_REPO)/$(TARGET) \
 		docker://$(COPY_REPO)/$(TARGET)

test/images/README.md

@@ -1,4 +1,4 @@
-# Images for external dependencies
+# Images for external test dependencies
 
 The Makefile on this directory allows for the manipulation of container images
 pertaining to external test dependencies, such as Mongo or quiver.
@@ -26,12 +26,16 @@ Note that this is not a full repo copy; it's restricted to the list of images
 contained on the Makefile.  Skupper images (such as the router or controller)
 are not copied by this operation.
 
+Note also that it is not a simple copy.  As Skupper can run on some older
+Kubernetes that do not support the OCI format, there are some transformations
+done during the copy as well.
+
 See the Makefile contents for information on how to execute the different
 operations.
 
 
 # skupper-test image
 
 Note that the `skupper-test` image used by Skupper integration is part of
-Skupper's own test code, so it's build by the main Makefile at ../.., and not
+Skupper's own test code, so it's built by the main Makefile at ../.., and not
 here.

test/integration/acceptance/annotation/annotated_resource.go

@@ -250,7 +250,7 @@ func createDeployment(cluster *client.VanClient, annotations map[string]string)
 				},
 				Spec: corev1.PodSpec{
 					Containers: []corev1.Container{
-						{Name: "nginx", Image: "quay.io/skupper/nginx-unprivileged:stable-alpine", Ports: []corev1.ContainerPort{{Name: "web", ContainerPort: 8080}}, ImagePullPolicy: corev1.PullIfNotPresent},
+						{Name: "nginx", Image: "quay.io/nginx/nginx-unprivileged:stable-alpine", Ports: []corev1.ContainerPort{{Name: "web", ContainerPort: 8080}}, ImagePullPolicy: corev1.PullIfNotPresent},
 					},
 					RestartPolicy: corev1.RestartPolicyAlways,
 				},
@@ -325,7 +325,7 @@ func createStatefulSet(cluster *client.VanClient, annotations map[string]string)
 				},
 				Spec: corev1.PodSpec{
 					Containers: []corev1.Container{
-						{Name: "nginx", Image: "quay.io/skupper/nginx-unprivileged:stable-alpine", Ports: []corev1.ContainerPort{{Name: "web", ContainerPort: 8080}}, ImagePullPolicy: corev1.PullIfNotPresent},
+						{Name: "nginx", Image: "quay.io/nginx/nginx-unprivileged:stable-alpine", Ports: []corev1.ContainerPort{{Name: "web", ContainerPort: 8080}}, ImagePullPolicy: corev1.PullIfNotPresent},
 					},
 					RestartPolicy: corev1.RestartPolicyAlways,
 				},
@@ -373,7 +373,7 @@ func createDaemonSet(cluster *client.VanClient, annotations map[string]string) (
 				},
 				Spec: corev1.PodSpec{
 					Containers: []corev1.Container{
-						{Name: "nginx", Image: "quay.io/skupper/nginx-unprivileged:stable-alpine", Ports: []corev1.ContainerPort{{Name: "web", ContainerPort: 8080}}, ImagePullPolicy: corev1.PullIfNotPresent},
+						{Name: "nginx", Image: "quay.io/nginx/nginx-unprivileged:stable-alpine", Ports: []corev1.ContainerPort{{Name: "web", ContainerPort: 8080}}, ImagePullPolicy: corev1.PullIfNotPresent},
 					},
 					RestartPolicy: corev1.RestartPolicyAlways,
 				},

test/integration/acceptance/custom/headless/headless.go

@@ -26,7 +26,7 @@ const (
 	appPort     = "8080"
 	appPortInt  = 8080
 	appPortName = "web"
-	appImage    = "quay.io/skupper/nginx-unprivileged:stable-alpine"
+	appImage    = "quay.io/nginx/nginx-unprivileged:stable-alpine"
 	pod0Name    = appName + "-0"
 	pod1Name    = appName + "-1"
 )

test/integration/examples/custom/hipstershop/hipstershop.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/skupperproject/skupper/api/types"
 	"github.com/skupperproject/skupper/pkg/utils"
+	"github.com/skupperproject/skupper/test/utils/arch"
 	"github.com/skupperproject/skupper/test/utils/base"
 	"github.com/skupperproject/skupper/test/utils/constants"
 	"github.com/skupperproject/skupper/test/utils/k8s"
@@ -50,6 +51,11 @@ func Setup(t *testing.T, testRunner base.ClusterTestRunner) {
 	prv1, err := testRunner.GetPrivateContext(1)
 	assert.Assert(t, err)
 
+	// Hipstershop is currently supported only on amd64
+	//
+	// https://github.com/GoogleCloudPlatform/microservices-demo/issues/622#issuecomment-2066712947
+	assert.Assert(t, arch.Skip(t, pub1, pub2, prv1))
+
 	// creating namespaces
 	assert.Assert(t, pub1.CreateNamespace())
 	assert.Assert(t, pub2.CreateNamespace())