Refactor run_tests.sh script (rancher#6280)

* Improve run_tests script Signed-off-by: Manuel Buil <[email protected]> * Upgrade tests to ubuntu2310 and use nodeOS var Signed-off-by: Manuel Buil <[email protected]> --------- Signed-off-by: Manuel Buil <[email protected]>
dereknola · Nov 6, 2024 · e305a73 · e305a73
1 parent eaa2357
commit e305a73
Show file tree

Hide file tree

Showing 19 changed files with 76 additions and 62 deletions.
diff --git a/tests/e2e/createreport/launchresults.html b/tests/e2e/createreport/launchresults.html
@@ -14,8 +14,8 @@
    <body>
       <h1>
       </h1>
-	<p class="center"> Ubuntu 20.04 Test Results </p>
-   <iframe src="./rke2_ubuntu2004_results.html"
+	<p class="center"> Ubuntu 23.10 Test Results </p>
+   <iframe src="./rke2_ubuntu2310_results.html"
               height="500px" width="1850px">
       </iframe>
 	<p class="center"> Rocky Linux8.7 Test Results </p>

diff --git a/tests/e2e/dnscache/Vagrantfile b/tests/e2e/dnscache/Vagrantfile
@@ -1,6 +1,6 @@
 ENV['VAGRANT_NO_PARALLEL'] = ENV['E2E_STANDUP_PARALLEL'] ? nil : 'no'
 NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "agent-0" ])
-NODE_BOXES = (ENV['E2E_NODE_BOXES'] || ['generic/ubuntu2004', 'generic/ubuntu2004'])
+NODE_BOXES = (ENV['E2E_NODE_BOXES'] || ['generic/ubuntu2310', 'generic/ubuntu2310'])
 GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")
 RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
 NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i

diff --git a/tests/e2e/dnscache/dnscache_test.go b/tests/e2e/dnscache/dnscache_test.go
@@ -13,8 +13,8 @@ import (
 	"github.com/rancher/rke2/tests/e2e"
 )
 
-// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64
-var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
+// Valid nodeOS: generic/ubuntu2310, opensuse/Leap-15.3.x86_64
+var nodeOS = flag.String("nodeOS", "generic/ubuntu2310", "VM operating system")
 var serverCount = flag.Int("serverCount", 1, "number of server nodes")
 var agentCount = flag.Int("agentCount", 1, "number of agent nodes")
 var ci = flag.Bool("ci", false, "running on CI")

diff --git a/tests/e2e/dualstack/Vagrantfile b/tests/e2e/dualstack/Vagrantfile
@@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = ENV['E2E_STANDUP_PARALLEL'] ? nil : 'no'
 NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
   ["server-0", "server-1", "server-2", "agent-0" ])
 NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
-  ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004'])
+  ['generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310'])
 GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")
 RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
 NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i

diff --git a/tests/e2e/dualstack/dualstack_test.go b/tests/e2e/dualstack/dualstack_test.go
@@ -13,8 +13,8 @@ import (
 	"github.com/rancher/rke2/tests/e2e"
 )
 
-// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64
-var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
+// Valid nodeOS: generic/ubuntu2310, opensuse/Leap-15.3.x86_64
+var nodeOS = flag.String("nodeOS", "generic/ubuntu2310", "VM operating system")
 var serverCount = flag.Int("serverCount", 3, "number of server nodes")
 var agentCount = flag.Int("agentCount", 1, "number of agent nodes")
 var ci = flag.Bool("ci", false, "running on CI")

diff --git a/tests/e2e/mixedos/Vagrantfile b/tests/e2e/mixedos/Vagrantfile
@@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = ENV['E2E_STANDUP_PARALLEL'] ? nil : 'no'
 NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
 ["server-0", "linux-agent-0", "windows-agent-0" ])
 NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
-['generic/ubuntu2004', 'generic/ubuntu2004', 'jborean93/WindowsServer2022'])
+['generic/ubuntu2310', 'generic/ubuntu2310', 'jborean93/WindowsServer2022'])
 GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")
 RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
 NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i

diff --git a/tests/e2e/mixedos/mixedos_test.go b/tests/e2e/mixedos/mixedos_test.go
@@ -12,8 +12,8 @@ import (
 	"github.com/rancher/rke2/tests/e2e"
 )
 
-// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64
-var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "operating system for linux nodes")
+// Valid nodeOS: generic/ubuntu2310, opensuse/Leap-15.3.x86_64
+var nodeOS = flag.String("nodeOS", "generic/ubuntu2310", "operating system for linux nodes")
 var serverCount = flag.Int("serverCount", 1, "number of server nodes")
 var linuxAgentCount = flag.Int("linuxAgentCount", 1, "number of linux agent nodes")
 var windowsAgentCount = flag.Int("windowsAgentCount", 1, "number of windows agent nodes")

diff --git a/tests/e2e/mixedosbgp/Vagrantfile b/tests/e2e/mixedosbgp/Vagrantfile
@@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = ENV['E2E_STANDUP_PARALLEL'] ? nil : 'no'
 NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
 ["server-0", "linux-agent-0", "windows-agent-0" ])
 NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
-['generic/ubuntu2004', 'generic/ubuntu2004', 'jborean93/WindowsServer2022'])
+['generic/ubuntu2310', 'generic/ubuntu2310', 'jborean93/WindowsServer2022'])
 GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")
 RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
 NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i

diff --git a/tests/e2e/mixedosbgp/mixedosbgp_test.go b/tests/e2e/mixedosbgp/mixedosbgp_test.go
@@ -12,8 +12,8 @@ import (
 	"github.com/rancher/rke2/tests/e2e"
 )
 
-// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64
-var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "operating system for linux nodes")
+// Valid nodeOS: generic/ubuntu2310, opensuse/Leap-15.3.x86_64
+var nodeOS = flag.String("nodeOS", "generic/ubuntu2310", "operating system for linux nodes")
 var serverCount = flag.Int("serverCount", 1, "number of server nodes")
 var linuxAgentCount = flag.Int("linuxAgentCount", 1, "number of linux agent nodes")
 var windowsAgentCount = flag.Int("windowsAgentCount", 1, "number of windows agent nodes")

diff --git a/tests/e2e/multus/Vagrantfile b/tests/e2e/multus/Vagrantfile
@@ -1,6 +1,6 @@
 ENV['VAGRANT_NO_PARALLEL'] = ENV['E2E_STANDUP_PARALLEL'] ? nil : 'no'
 NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "agent-0" ])
-NODE_BOXES = (ENV['E2E_NODE_BOXES'] || ['generic/ubuntu2004', 'generic/ubuntu2004'])
+NODE_BOXES = (ENV['E2E_NODE_BOXES'] || ['generic/ubuntu2310', 'generic/ubuntu2310'])
 GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")
 RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
 NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i

diff --git a/tests/e2e/multus/multus_test.go b/tests/e2e/multus/multus_test.go
@@ -13,8 +13,8 @@ import (
 	"github.com/rancher/rke2/tests/e2e"
 )
 
-// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64
-var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
+// Valid nodeOS: generic/ubuntu2310, opensuse/Leap-15.3.x86_64
+var nodeOS = flag.String("nodeOS", "generic/ubuntu2310", "VM operating system")
 var serverCount = flag.Int("serverCount", 1, "number of server nodes")
 var agentCount = flag.Int("agentCount", 1, "number of agent nodes")
 var ci = flag.Bool("ci", false, "running on CI")

diff --git a/tests/e2e/scripts/latest_commit.sh b/tests/e2e/scripts/latest_commit.sh
@@ -1,7 +1,22 @@
 #!/bin/bash
 # Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build
 iterations=0
-curl -s -H 'Accept: application/vnd.github.v3+json' "https://api.github.com/repos/rancher/rke2/commits?per_page=5&sha=$1" | jq -r '.[] | .sha'  &> "$2"
+response=$(curl -s -H 'Accept: application/vnd.github.v3+json' "https://api.github.com/repos/rancher/rke2/commits?per_page=5&sha=$1")
+type=$(echo "$response" | jq -r type)
+
+# Verify if the response is an array with the rke2 commits
+if [[ $type == "object" ]]; then
+    message=$(echo "$response" | jq -r .message)
+    if [[ $message == "API rate limit exceeded for "* ]]; then
+        echo "Github API rate limit exceeded"
+	exit 1
+    fi
+    echo "Github API returned a non-expected response ${message}"
+    exit 1
+elif [[ $type == "array" ]]; then
+    echo ${response} | jq -r '.[] | .sha'  &> "$2"
+fi
+
 curl -s --fail https://rke2-ci-builds.s3.amazonaws.com/rke2-images.linux-amd64-$(head -n 1 $2).tar.zst.sha256sum
 while [ $? -ne 0 ]; do
     ((iterations++))

diff --git a/tests/e2e/scripts/run_tests.sh b/tests/e2e/scripts/run_tests.sh
@@ -1,62 +1,61 @@
 #!/bin/bash
-nodeOS=${1:-"generic/ubuntu2004"}
-servercount=${2:-3}
-agentcount=${3:-1}
-db=${4:-"etcd"}
-hardened=${5:-""}
-rke2_version=${rke2_version}
-rke2_channel=${rke2_channel:-"commit"}
-
-E2E_EXTERNAL_DB=$db && export E2E_EXTERNAL_DB
+# Usage: ./run_tests.sh
+# This script runs all the rke2 e2e tests and generates a report with the log
+# The generated log is placed in createreport/rke2_${OS}.log
+#
+# This script must be run inside the rke2 directory where the tests exist
+#
+# Example:
+#   To run the script with default settings:
+#     ./run_tests.sh
+#
+set -x
+
+# tests to run
+tests=("ciliumnokp" "dnscache" "dualstack" "mixedos" "mixedosbgp" "multus" "secretsencryption" "splitserver" "upgradecluster" "validatecluster")
+nodeOS=${1:-"generic/ubuntu2310"}
+OS=$(echo "$nodeOS"|cut -d'/' -f2)
+
 E2E_REGISTRY=true && export E2E_REGISTRY
 
-cd
-cd rke2 && git pull --rebase origin master
+git pull --rebase origin master
 /usr/local/go/bin/go mod tidy
-
 cd tests/e2e
-OS=$(echo "$nodeOS"|cut -d'/' -f2)
-echo "$OS"
 
 # create directory to store reports if it does not exists
 if [ ! -d createreport ]
 then
 	mkdir createreport
 fi
 
+# Remove VMs which are in invalid state
+vagrant global-status --prune
+
 count=0
 run_tests(){
-	count=$(( count + 1 ))
-	vagrant global-status | awk '/running/'|cut -c1-7| xargs -r -d '\n' -n 1 -- vagrant destroy -f
-
-	E2E_RELEASE_VERSION=$rke2_version && export E2E_RELEASE_VERSION
-	E2E_RELEASE_CHANNEL=$rke2_channel && export E2E_RELEASE_CHANNEL
-
-	echo 'RUNNING CLUSTER UPGRADE TEST'
-	E2E_REGISTRY=true /usr/local/go/bin/go test -v ./upgradecluster/upgradecluster_test.go -nodeOS="$nodeOS" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=1h -json -ci |tee createreport/rke2_"$OS".log
-
-	echo 'RUNNING DUALSTACK VALIDATION TEST'
-	E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v dualstack/dualstack_test.go -nodeOS="$nodeOS" -serverCount=1 -agentCount=1  -timeout=30m -json -ci |tee -a createreport/rke2_"$OS".log
 
-	echo 'RUNNING CLUSTER VALIDATION TEST'
-	E2E_REGISTRY=true E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v validatecluster/validatecluster_test.go -nodeOS="$nodeOS" -serverCount=$((servercount)) -agentCount=$((agentcount))  -timeout=30m -json -ci |tee -a createreport/rke2_"$OS".log
+    count=$(( count + 1 ))
 
-	echo 'RUNNING MIXEDOS TEST'
-	/usr/local/go/bin/go test -v mixedos/mixedos_test.go -nodeOS="$nodeOS" -serverCount=$((servercount)) -timeout=1h -json -ci |tee -a  createreport/rke2_"$OS".log
+    for i in ${!tests[@]}; do
+	pushd ${tests[$i]}
+	vagrant destroy -f
 
-	echo 'RUNNING SPLIT SERVER VALIDATION TEST'
-	E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v splitserver/splitserver_test.go -nodeOS="$nodeOS" -timeout=30m -json -ci |tee -a createreport/rke2_"$OS".log
+        echo "RUNNING ${tests[$i]} TEST"
+        /usr/local/go/bin/go test -v ${tests[$i]}_test.go -timeout=2h -nodeOS="$nodeOS" -json -ci |tee -a ../createreport/rke2_${OS}.log
+
+	popd
+    done
 }
 
-ls createreport/rke2_"$OS".log 2>/dev/null && rm createreport/rke2_"$OS".log
+ls createreport/rke2_${OS}.log 2>/dev/null && rm createreport/rke2_${OS}.log
 run_tests
 
 # re-run test if first run fails and keep record of repeatedly failed test to debug
-while [ -f createreport/rke2_"$OS".log ] && grep -w ":fail" createreport/rke2_"$OS".log && [ $count -le 2 ]
+while [ -f createreport/rke2_${OS}.log ] && grep -w " FAIL:" createreport/rke2_${OS}.log && [ $count -le 2 ]
 do
-        cp createreport/rke2_"$OS".log createreport/rke2_"$OS"_"$count".log
+        cp createreport/rke2_${OS}.log createreport/rke2_${OS}_${count}.log
         run_tests
 done
 
 # Generate report and upload to s3 bucket
-cd createreport && /usr/local/go/bin/go run -v report-template-bindata.go generate_report.go -f rke2_"OS".log
+cd createreport && /usr/local/go/bin/go run -v report-template-bindata.go generate_report.go -f rke2_${OS}.log
diff --git a/tests/e2e/splitserver/Vagrantfile b/tests/e2e/splitserver/Vagrantfile
@@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = ENV['E2E_STANDUP_PARALLEL'] ? nil : 'no'
 NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
   ["server-etcd-0", "server-cp-0", "server-cp-1", "agent-0"])
 NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
-  ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004'])
+  ['generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310'])
 GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")
 RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
 NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i

diff --git a/tests/e2e/splitserver/splitserver_test.go b/tests/e2e/splitserver/splitserver_test.go
@@ -13,8 +13,8 @@ import (
 	"github.com/rancher/rke2/tests/e2e"
 )
 
-// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64
-var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
+// Valid nodeOS: generic/ubuntu2310, opensuse/Leap-15.3.x86_64
+var nodeOS = flag.String("nodeOS", "generic/ubuntu2310", "VM operating system")
 var etcdCount = flag.Int("etcdCount", 1, "number of server nodes only deploying etcd")
 var controlPlaneCount = flag.Int("controlPlaneCount", 1, "number of server nodes acting as control plane")
 var agentCount = flag.Int("agentCount", 1, "number of agent nodes")

diff --git a/tests/e2e/upgradecluster/Vagrantfile b/tests/e2e/upgradecluster/Vagrantfile
@@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = ENV['E2E_STANDUP_PARALLEL'] ? nil : 'no'
 NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
   ["server-0", "server-1", "server-2", "agent-0"])
 NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
-  ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004'])
+  ['generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310'])
 RELEASE_CHANNEL = (ENV['E2E_RELEASE_CHANNEL'] || "latest")
 RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
 NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i

diff --git a/tests/e2e/upgradecluster/upgradecluster_test.go b/tests/e2e/upgradecluster/upgradecluster_test.go
@@ -12,8 +12,8 @@ import (
 	"github.com/rancher/rke2/tests/e2e"
 )
 
-// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64
-var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
+// Valid nodeOS: generic/ubuntu2310, opensuse/Leap-15.3.x86_64
+var nodeOS = flag.String("nodeOS", "generic/ubuntu2310", "VM operating system")
 var serverCount = flag.Int("serverCount", 3, "number of server nodes")
 var agentCount = flag.Int("agentCount", 1, "number of agent nodes")
 var ci = flag.Bool("ci", false, "running on CI")

diff --git a/tests/e2e/validatecluster/Vagrantfile b/tests/e2e/validatecluster/Vagrantfile
@@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = ENV['E2E_STANDUP_PARALLEL'] ? nil : 'no'
 NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
   ["server-0", "server-1", "server-2", "agent-0" ])
 NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
-  ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004'])
+  ['generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310'])
 GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")
 RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
 NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i

diff --git a/tests/e2e/validatecluster/validatecluster_test.go b/tests/e2e/validatecluster/validatecluster_test.go
@@ -13,8 +13,8 @@ import (
 	"github.com/rancher/rke2/tests/e2e"
 )
 
-// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64
-var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
+// Valid nodeOS: generic/ubuntu2310, opensuse/Leap-15.3.x86_64
+var nodeOS = flag.String("nodeOS", "generic/ubuntu2310", "VM operating system")
 var serverCount = flag.Int("serverCount", 3, "number of server nodes")
 var agentCount = flag.Int("agentCount", 1, "number of agent nodes")
 var ci = flag.Bool("ci", false, "running on CI")