feat: performance test scaffolding

Makefile

@@ -121,6 +121,11 @@ test.integration: manifests generate fmt vet
 	go clean -testcache
 	GOFLAGS="-tags=integration_tests" go test -race -v ./test/integration/...
 
+.PHONY: test.performance
+test.performance: manifests generate fmt vet
+	go clean -testcache
+	GOFLAGS="-tags=performance_tests" go test -race -v ./test/performance/...
+
 ##@ Build
 
 .PHONY: build

config/tests/performance/kustomization.yaml

@@ -0,0 +1,9 @@
+bases:
+- ../../default
+- perf-server.yaml
+
+images:
+- name: ghcr.io/kong/blixt-dataplane
+  newTag: performance-tests
+- name: ghcr.io/kong/blixt-controlplane
+  newTag: performance-tests

config/tests/performance/perf-server.yaml

@@ -0,0 +1,39 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: blixt-perf-server
+  labels:
+    app: blixt-perf-server
+spec:
+  selector:
+    matchLabels:
+      app: blixt-perf-server
+  template:
+    metadata:
+      labels:
+        app: blixt-perf-server
+    spec:
+      containers:
+      - name: server
+        image: ghcr.io/kong/blixt-perf-test-server:latest
+        imagePullPolicy: IfNotPresent
+        ports:
+        - containerPort: 5201
+          protocol: UDP
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: blixt-perf-server
+  labels:
+    app: blixt-perf-server
+spec:
+  type: ClusterIP
+  selector:
+    app: blixt-perf-server
+  ports:
+  - name: udp-perf
+    port: 5201
+    protocol: UDP
+    targetPort: 5201

internal/test/utils/utils.go

@@ -2,10 +2,16 @@ package utils
 
 import (
 	"bytes"
+	"context"
+	"fmt"
 	"log"
+	"testing"
 
 	"github.com/go-logr/logr"
 	"github.com/go-logr/stdr"
+	"github.com/kong/kubernetes-testing-framework/pkg/clusters"
+	"github.com/kong/kubernetes-testing-framework/pkg/environments"
+	"github.com/stretchr/testify/assert"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -62,3 +68,51 @@ func NewFakeClientWithGatewayClasses(initObjects ...client.Object) (gatewayv1bet
 		gatewayv1beta1.ObjectName(unmanagedGatewayClass.Name),
 		fakeClient
 }
+
+// WaitForBlixtReadiness waits for Blixt to be ready in the provided testing
+// environment (but deploying Blixt is expected to have already been handled
+// elsewhere).
+func WaitForBlixtReadiness(ctx context.Context, env environments.Environment) error {
+	for {
+		select {
+		case <-ctx.Done():
+			if err := ctx.Err(); err != nil {
+				return fmt.Errorf("context completed while waiting for components: %w", err)
+			}
+			return fmt.Errorf("context completed while waiting for components")
+		default:
+			var controlplaneReady, dataplaneReady bool
+
+			controlplane, err := env.Cluster().Client().AppsV1().Deployments(vars.DefaultNamespace).Get(ctx, vars.DefaultControlPlaneDeploymentName, metav1.GetOptions{})
+			if err != nil {
+				return err
+			}
+			if controlplane.Status.AvailableReplicas > 0 {
+				controlplaneReady = true
+			}
+
+			dataplane, err := env.Cluster().Client().AppsV1().DaemonSets(vars.DefaultNamespace).Get(ctx, vars.DefaultDataPlaneDaemonSetName, metav1.GetOptions{})
+			if err != nil {
+				return err
+			}
+			if dataplane.Status.NumberAvailable > 0 {
+				dataplaneReady = true
+			}
+
+			if controlplaneReady && dataplaneReady {
+				return nil
+			}
+		}
+	}
+}
+
+// DumpDiagnosticsIfFailed dumps the diagnostics if the test failed.
+func DumpDiagnosticsIfFailed(ctx context.Context, t *testing.T, clusters clusters.Cluster) {
+	t.Helper()
+
+	if t.Failed() {
+		output, err := clusters.DumpDiagnostics(ctx, t.Name())
+		t.Logf("%s failed, dumped diagnostics to %s", t.Name(), output)
+		assert.NoError(t, err)
+	}
+}

test/integration/suite_test.go

@@ -14,16 +14,15 @@ import (
 	"github.com/kong/kubernetes-testing-framework/pkg/clusters/addons/metallb"
 	"github.com/kong/kubernetes-testing-framework/pkg/clusters/types/kind"
 	"github.com/kong/kubernetes-testing-framework/pkg/environments"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/gateway-api/pkg/client/clientset/versioned"
 
-	"github.com/kong/blixt/pkg/vars"
+	testutils "github.com/kong/blixt/internal/test/utils"
 )
 
 var (
-	ctx    context.Context
-	cancel context.CancelFunc
-	env    environments.Environment
+	ctx     context.Context
+	cancel  context.CancelFunc
+	env     environments.Environment
 	cleanup map[string]([]func(context.Context) error)
 
 	gwclient *versioned.Clientset
@@ -41,7 +40,7 @@ var (
 
 const (
 	gwCRDsKustomize = "https://github.com/kubernetes-sigs/gateway-api/config/crd/experimental?ref=v0.5.1"
-	testKustomize   = "../../config/tests/blixt"
+	testKustomize   = "../../config/tests/integration"
 )
 
 func TestMain(m *testing.M) {
@@ -117,12 +116,12 @@ func TestMain(m *testing.M) {
 			return clusters.KustomizeDeleteForCluster(ctx, env.Cluster(), testKustomize)
 		})
 	}
-	exitOnErr(waitForBlixtReadiness(ctx, env))
+	exitOnErr(testutils.WaitForBlixtReadiness(ctx, env))
 
 	exit := m.Run()
 
 	exitOnErr(runCleanup(mainCleanupKey))
-	
+
 	os.Exit(exit)
 }
 
@@ -178,37 +177,3 @@ func runCleanup(cleanupKey string) (cleanupErr error) {
 	delete(cleanup, cleanupKey)
 	return
 }
-
-func waitForBlixtReadiness(ctx context.Context, env environments.Environment) error {
-	for {
-		select {
-		case <-ctx.Done():
-			if err := ctx.Err(); err != nil {
-				return fmt.Errorf("context completed while waiting for components: %w", err)
-			}
-			return fmt.Errorf("context completed while waiting for components")
-		default:
-			var controlplaneReady, dataplaneReady bool
-
-			controlplane, err := env.Cluster().Client().AppsV1().Deployments(vars.DefaultNamespace).Get(ctx, vars.DefaultControlPlaneDeploymentName, metav1.GetOptions{})
-			if err != nil {
-				return err
-			}
-			if controlplane.Status.AvailableReplicas > 0 {
-				controlplaneReady = true
-			}
-
-			dataplane, err := env.Cluster().Client().AppsV1().DaemonSets(vars.DefaultNamespace).Get(ctx, vars.DefaultDataPlaneDaemonSetName, metav1.GetOptions{})
-			if err != nil {
-				return err
-			}
-			if dataplane.Status.NumberAvailable > 0 {
-				dataplaneReady = true
-			}
-
-			if controlplaneReady && dataplaneReady {
-				return nil
-			}
-		}
-	}
-}

test/integration/udproute_test.go

@@ -14,12 +14,13 @@ import (
 	"time"
 
 	"github.com/google/uuid"
-	"github.com/kong/blixt/test/utils"
 	"github.com/kong/kubernetes-testing-framework/pkg/clusters"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
+
+	testutils "github.com/kong/blixt/internal/test/utils"
 )
 
 const (
@@ -31,7 +32,7 @@ const (
 func TestUDPRouteBasics(t *testing.T) {
 	udpRouteBasicsCleanupKey := "udproutebasics"
 	defer func() {
-		utils.DumpDiagnosticsIfFailed(ctx, t, env.Cluster())
+		testutils.DumpDiagnosticsIfFailed(ctx, t, env.Cluster())
 		runCleanup(udpRouteBasicsCleanupKey)
 	}()
 
@@ -94,7 +95,7 @@ func TestUDPRouteBasics(t *testing.T) {
 func TestUDPRouteNoReach(t *testing.T) {
 	udpRouteNoReachCleanupKey := "udproutenoreach"
 	defer func() {
-		utils.DumpDiagnosticsIfFailed(ctx, t, env.Cluster())
+		testutils.DumpDiagnosticsIfFailed(ctx, t, env.Cluster())
 		runCleanup(udpRouteNoReachCleanupKey)
 	}()
 

test/performance/suite_test.go

@@ -0,0 +1,83 @@
+//go:build performance_tests
+// +build performance_tests
+
+package performance
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/kong/kubernetes-testing-framework/pkg/clusters"
+	"github.com/kong/kubernetes-testing-framework/pkg/clusters/addons/loadimage"
+	"github.com/kong/kubernetes-testing-framework/pkg/clusters/addons/metallb"
+	"github.com/kong/kubernetes-testing-framework/pkg/environments"
+	"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	"sigs.k8s.io/gateway-api/pkg/client/clientset/versioned"
+
+	testutils "github.com/kong/blixt/internal/test/utils"
+)
+
+var (
+	ctx     context.Context
+	cancel  context.CancelFunc
+	env     environments.Environment
+	cleanup map[string]([]func(context.Context) error)
+
+	gwclient  *versioned.Clientset
+	k8sclient *clientset.Clientset
+
+	controlplaneImage = os.Getenv("BLIXT_CONTROLPLANE_IMAGE")
+	dataplaneImage    = os.Getenv("BLIXT_DATAPLANE_IMAGE")
+)
+
+const (
+	gwCRDsKustomize = "https://github.com/kubernetes-sigs/gateway-api/config/crd/experimental?ref=v0.5.1"
+	testKustomize   = "../../config/tests/performance"
+)
+
+func TestMain(m *testing.M) {
+	if controlplaneImage == "" || dataplaneImage == "" {
+		exitOnErr(fmt.Errorf("BLIXT_CONTROLPLANE_IMAGE and BLIXT_DATAPLANE_IMAGE must be provided"))
+	}
+
+	ctx, cancel = context.WithCancel(context.Background())
+	defer cancel()
+
+	fmt.Println("INFO: creating a new kind cluster")
+	loadImages, err := loadimage.NewBuilder().WithImage(controlplaneImage)
+	exitOnErr(err)
+	loadImages, err = loadImages.WithImage(dataplaneImage)
+	exitOnErr(err)
+
+	env, err = environments.NewBuilder().WithAddons(metallb.New(), loadImages.Build()).Build(ctx)
+	exitOnErr(err)
+
+	fmt.Printf("INFO: new kind cluster %s was created\n", env.Cluster().Name())
+	gwclient, err = versioned.NewForConfig(env.Cluster().Config())
+	exitOnErr(err)
+	k8sclient, err = clientset.NewForConfig(env.Cluster().Config())
+	exitOnErr(err)
+
+	fmt.Println("INFO: deploying Gateway API CRDs")
+	exitOnErr(clusters.KustomizeDeployForCluster(ctx, env.Cluster(), gwCRDsKustomize))
+
+	fmt.Println("INFO: deploying blixt via config/test kustomize")
+	exitOnErr(clusters.KustomizeDeployForCluster(ctx, env.Cluster(), testKustomize))
+	exitOnErr(testutils.WaitForBlixtReadiness(ctx, env))
+
+	fmt.Println("INFO: running performance tests")
+	exit := m.Run()
+
+	exitOnErr(env.Cluster().Cleanup(ctx))
+
+	os.Exit(exit)
+}
+
+func exitOnErr(err error) {
+	if err != nil {
+		fmt.Fprint(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+}

tools/performance/Dockerfile

@@ -0,0 +1,11 @@
+FROM alpine
+
+LABEL org.opencontainers.image.source https://github.com/kong/blixt
+
+RUN apk --no-cache add iperf3
+
+RUN adduser -D -s /bin/sh iperf3
+
+USER iperf3
+
+ENTRYPOINT ["iperf3", "--server"]

tools/performance/Makefile

@@ -0,0 +1,12 @@
+IMAGE ?= ghcr.io/kong/blixt-perf-test-server
+TAG ?= latest
+
+all: build.image
+
+.PHONY: build.image
+build.image:
+	docker build -t $(IMAGE):$(TAG) ./
+
+.PHONY: push.image
+push.image: build.image
+	docker push $(IMAGE):$(TAG)

tools/performance/README.md

@@ -0,0 +1,10 @@
+# Performance Tooling
+
+Here you will find tooling for running performance tests of the Blixt dataplane.
+
+We use [iPerf3][iperf3] for performance tests and provide our own container
+image for that here as [the iPerf3 community is opposed to providing one
+themselves][iperf3-no-image].
+
+[iperf3]:https://github.com/esnet/iperf
+[iperf3-no-image]:https://github.com/esnet/iperf/issues/690#issuecomment-367879269