Address comments

Signed-off-by: Wenqi Qiu <[email protected]>

Makefile

@@ -370,7 +370,8 @@ build-agent-ubuntu:
 .PHONY: build-ubuntu
 build-ubuntu: build-agent-ubuntu build-controller-ubuntu
 
-antrea-scale-image:
+.PHONY: build-antrea-scale
+build-antrea-scale:
 	@echo "===> Building antrea/antrea-scale Docker image <==="
 	docker build -t antrea/antrea-scale:$(DOCKER_IMG_VERSION) -f build/images/Dockerfile.scale .
 	docker tag antrea/antrea-scale:$(DOCKER_IMG_VERSION) antrea/antrea-scale

build/images/Dockerfile.scale

@@ -1,9 +1,26 @@
-FROM ubuntu:20.04
+# Copyright 2024 Antrea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG BASE_IMAGE=ubuntu:22.04
+FROM ${BASE_IMAGE}
 
 LABEL maintainer="Antrea <[email protected]>"
 LABEL description="Antrea scale test image."
 
 USER root
 
+WORKDIR /antrea
+
 COPY test/performance/scale.yml .
 COPY bin/antrea-scale /usr/local/bin

build/yamls/antrea-scale-exec.yml → build/yamls/antrea-scale.yml

@@ -61,7 +61,7 @@ rules:
       - delete
       - deletecollection
   - apiGroups:
-      - clusterinformation.antrea.tanzu.vmware.com
+      - crd.antrea.io
     resources:
       - antreaagentinfos
     verbs:
@@ -70,8 +70,7 @@ rules:
       - update
       - delete
   - apiGroups:
-      - controlplane.antrea.tanzu.vmware.com
-      - networking.antrea.tanzu.vmware.com
+      - controlplane.antrea.io
     resources:
       - networkpolicies
       - appliedtogroups
@@ -81,7 +80,7 @@ rules:
       - watch
       - list
   - apiGroups:
-      - controlplane.antrea.tanzu.vmware.com
+      - controlplane.antrea.io
     resources:
       - nodestatssummaries
     verbs:

ci/jenkins/test-scale.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Copyright 2023 Antrea Authors
+# Copyright 2024 Antrea Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,18 +27,11 @@ WORKDIR=$DEFAULT_WORKDIR
 KUBECONFIG_PATH=$DEFAULT_KUBECONFIG_PATH
 TESTCASE=""
 TEST_FAILURE=false
-MODE="report"
 DOCKER_REGISTRY=$(head -n1 "${WORKSPACE}/ci/docker-registry")
 TESTBED_TYPE="legacy"
-GO_VERSION=$(head -n1 "${WORKSPACE}/build/images/deps/go-version")
 IMAGE_PULL_POLICY="Always"
-DEFAULT_IP_MODE="ipv4"
-IP_MODE=""
 GOLANG_RELEASE_DIR=${WORKDIR}/golang-releases
 
-CONFORMANCE_SKIP="\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[sig-cli\]|\[sig-storage\]|\[sig-auth\]|\[sig-api-machinery\]|\[sig-apps\]|\[sig-node\]"
-NETWORKPOLICY_SKIP="NetworkPolicyLegacy|should allow egress access to server in CIDR block|should enforce except clause while egress access to server in CIDR block"
-
 CONTROL_PLANE_NODE_ROLE="master|control-plane"
 
 CLEAN_STALE_IMAGES="docker system prune --force --all --filter until=4h"
@@ -53,9 +46,6 @@ Run K8s e2e community tests (Conformance & Network Policy) or Antrea e2e tests o
 
         --kubeconfig             Path of cluster kubeconfig.
         --workdir                Home path for Go, vSphere information and antrea_logs during cluster setup. Default is $WORKDIR.
-        --testcase               Windows install OVS, Conformance and Network Policy or Antrea e2e testcases on a Windows or Linux cluster. It can also be flexible ipam or multicast e2e test.
-        --registry               The docker registry to use instead of dockerhub.
-        --testbed-type           The testbed type to run tests. It can be flexible-ipam, jumper or legacy." 
 
 function print_usage {
     echoerr "$_usage"
@@ -82,18 +72,6 @@ case $key in
     WORKDIR="$2"
     shift 2
     ;;
-    --testcase)
-    TESTCASE="$2"
-    shift 2
-    ;;
-    --registry)
-    DOCKER_REGISTRY="$2"
-    shift 2
-    ;;
-    --testbed-type)
-    TESTBED_TYPE="$2"
-    shift 2
-    ;;
     -h|--help)
     print_usage
     exit 0
@@ -118,8 +96,6 @@ if [[ "$DOCKER_REGISTRY" != "" ]]; then
 fi
 export NO_PULL
 
-E2ETEST_PATH=${WORKDIR}/kubernetes/_output/dockerized/bin/linux/amd64/e2e.test
-
 function export_govc_env_var {
     env_govc="${WORKDIR}/govc.env"
     if [ -f "$env_govc" ]; then
@@ -136,15 +112,7 @@ function export_govc_env_var {
 
 function clean_antrea {
     echo "====== Cleanup Antrea Installation ======"
-    clean_ns "monitoring"
-    clean_ns "antrea-ipam-test"
-    clean_ns "antrea-test"
-    echo "====== Cleanup Conformance Namespaces ======"
-    clean_ns "net"
-    clean_ns "service"
-    clean_ns "x-"
-    clean_ns "y-"
-    clean_ns "z-"
+    clean_ns "antrea-scale-ns"
 
     # Delete antrea-prometheus first for k8s>=1.22 to avoid Pod stuck in Terminating state.
     kubectl delete -f ${WORKDIR}/antrea-prometheus.yml --ignore-not-found=true || true
@@ -201,10 +169,7 @@ function deliver_antrea_scale {
     make clean
     ${CLEAN_STALE_IMAGES}
     ${PRINT_DOCKER_STATUS}
-    if [[ ! "${TESTCASE}" =~ "e2e" && "${DOCKER_REGISTRY}" != "" ]]; then
-        docker pull "${DOCKER_REGISTRY}/antrea/sonobuoy-systemd-logs:v0.3"
-        docker tag "${DOCKER_REGISTRY}/antrea/sonobuoy-systemd-logs:v0.3" "sonobuoy/systemd-logs:v0.3"
-    fi
+
     chmod -R g-w build/images/ovs
     chmod -R g-w build/images/base
     DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull
@@ -275,12 +240,19 @@ function generate_ssh_config {
 }
 
 function prepare_scale_simulator {
+    ## Try best to clean up old config.
+    kubectl delete -f "${WORKSPACE}/build/yamls/antrea-agent-simulator.yml" || true
+    kubectl delete secret kubeconfig || true
+
+    # Create simulators.
     kubectl taint -l 'antrea/instance=simulator' node mocknode=true:NoExecute
-    kubectl create secret generic kubeconfig --type=Opaque --namespace=kube-system --from-file=${WORKDIR}/.kube
+    kubectl create secret generic kubeconfig --type=Opaque --namespace=kube-system --from-file=admin.conf=${WORKDIR}/.kube
+
+    kubectl apply -f "${WORKSPACE}/build/yamls/antrea-agent-simulator.yml"
 }
 
 function run_scale_test {
-    echo "====== Running Antrea E2E Tests ======"
+    echo "====== Running Antrea Scale Tests ======"
     export GO111MODULE=on
     export GOPATH=${WORKDIR}/go
     export GOROOT=${GOLANG_RELEASE_DIR}/go
@@ -293,6 +265,8 @@ function run_scale_test {
     generate_ssh_config
 
     set +e
+    ls ${WORKSPACE}
+    make bin
     ${WORKSPACE}/bin/antrea-scale --config ./test/performance/scale.yml
     set -e
 
@@ -318,8 +292,8 @@ source $WORKSPACE/ci/jenkins/utils.sh
 check_and_upgrade_golang
 clean_tmp
 
-trap clean_antrea EXIT
-deliver_antrea_scale
+#trap clean_antrea EXIT
+#deliver_antrea_scale
 prepare_scale_simulator
 run_scale_test
 

cmd/antrea-scale/main.go

@@ -1,4 +1,4 @@
-// Copyright 2021 Antrea Authors
+// Copyright 2024 Antrea Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -28,19 +28,21 @@ import (
 )
 
 type options struct {
-	kubeConfigPath string
-	configPath     string
-	timeout        int
-	loglevel       string
+	kubeConfigPath    string
+	configPath        string
+	templateFilesPath string
+	timeout           int
+	loglevel          string
 }
 
 var (
 	option = options{}
 )
 
 func init() {
-	flag.StringVar(&option.kubeConfigPath, "kubeConfigPath", "", "Cluster config path")
-	flag.StringVar(&option.configPath, "config", "", "Config file of scale test cases list")
+	flag.StringVar(&option.kubeConfigPath, "kubeConfigPath", "", "Path of cluster kubeconfig")
+	flag.StringVar(&option.configPath, "config", "test/performance/scale.yml", "Config file of scale test cases list")
+	flag.StringVar(&option.templateFilesPath, "templateFilesPath", "test/performance/assets", "Template YAML files path of test cases")
 	flag.IntVar(&option.timeout, "timeout", 10, "Timeout limit (minutes) of the whole scale test")
 	flag.StringVar(&option.loglevel, "v", "2", "")
 	flag.Parse()
@@ -66,7 +68,7 @@ func run() error {
 	globalCtx, globalCancelFunc := context.WithTimeout(context.Background(), time.Duration(option.timeout)*time.Minute)
 	defer globalCancelFunc()
 
-	testData, err := framework.ScaleUp(globalCtx, option.kubeConfigPath, option.configPath)
+	testData, err := framework.Initialize(globalCtx, option.kubeConfigPath, option.configPath, option.templateFilesPath)
 	if err != nil {
 		return fmt.Errorf("error when creating TestData: %w", err)
 	}
@@ -94,7 +96,7 @@ func run() error {
 			}
 			time.Sleep(config.WaitInterval)
 		}
-		klog.Infoln(strings.Repeat("==", 100))
+		klog.Infoln(strings.Repeat("==", 72))
 	}
 	return nil
 }

go.mod

@@ -39,9 +39,9 @@ require (
 	github.com/mdlayher/packet v1.1.2
 	github.com/miekg/dns v1.1.58
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822
+	github.com/olekukonko/tablewriter v0.0.4
 	github.com/onsi/ginkgo/v2 v2.17.1
 	github.com/onsi/gomega v1.32.0
-	github.com/olekukonko/tablewriter v0.0.4
 	github.com/pkg/sftp v1.13.6
 	github.com/prometheus/client_golang v1.18.0
 	github.com/prometheus/common v0.47.0