From 30e66d600373d4911e2256fc398d08f72d8785f1 Mon Sep 17 00:00:00 2001 From: Urvashi Mohnani Date: Thu, 15 Sep 2022 16:19:46 -0400 Subject: [PATCH] Set up minikube for k8s testing Install and set up minikube so that we can create a k8s cluster for testing. Signed-off-by: Urvashi Mohnani --- .cirrus.yml | 20 +++++++++- contrib/cirrus/runner.sh | 6 +++ contrib/cirrus/setup_environment.sh | 8 ++++ test/minikube/001-kube.bats | 61 +++++++++++++++++++++++++++++ test/minikube/helpers.bash | 60 ++++++++++++++++++++++++++++ 5 files changed, 154 insertions(+), 1 deletion(-) create mode 100755 test/minikube/001-kube.bats create mode 100644 test/minikube/helpers.bash diff --git a/.cirrus.yml b/.cirrus.yml index 0b61d979d7..d0780fe66b 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -33,7 +33,7 @@ env: UBUNTU_NAME: "ubuntu-2204" # Image identifiers - IMAGE_SUFFIX: "c5823947156488192" + IMAGE_SUFFIX: "c4678746211876864" # EC2 images FEDORA_AMI: "fedora-aws-${IMAGE_SUFFIX}" FEDORA_AARCH64_AMI: "fedora-podman-aws-arm64-${IMAGE_SUFFIX}" @@ -770,6 +770,23 @@ rootless_system_test_task: main_script: *main always: *logs_artifacts +minikube_test_task: + name: *std_name_fmt + alias: minikube_test + # Docs: ./contrib/cirrus/CIModes.md + only_if: *not_tag_build_docs_multiarch + depends_on: + - build + - rootless_system_test + gce_instance: *standardvm + env: + <<: *stdenvars + TEST_FLAVOR: minikube + PRIV_NAME: rootless + clone_script: *get_gosrc + setup_script: *setup + main_script: *main + always: *logs_artifacts buildah_bud_test_task: name: *std_name_fmt @@ -963,6 +980,7 @@ success_task: - remote_system_test_aarch64 - rootless_system_test - rootless_remote_system_test + - minikube_test - buildah_bud_test - rootless_gitlab_test - upgrade_test diff --git a/contrib/cirrus/runner.sh b/contrib/cirrus/runner.sh index d360f6a04c..2c8b7289bd 100755 --- a/contrib/cirrus/runner.sh +++ b/contrib/cirrus/runner.sh @@ -118,6 +118,12 @@ function _run_endpoint() { make endpoint } +function _run_minikube() { + _bail_if_test_can_be_skipped test/minikube + msg "Testing minikube." + bats test/minikube |& logformatter +} + exec_container() { local var_val local cmd diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index 2775094843..9c01aec6fe 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -297,6 +297,14 @@ case "$TEST_FLAVOR" in die "Invalid value for \$TEST_ENVIRON=$TEST_ENVIRON" fi + install_test_configs + ;; + minikube) + dnf install -y $PACKAGE_DOWNLOAD_DIR/minikube-latest* + remove_packaged_podman_files + make install.tools + make install PREFIX=/usr ETCDIR=/etc + minikube config set driver podman install_test_configs ;; machine) diff --git a/test/minikube/001-kube.bats b/test/minikube/001-kube.bats new file mode 100755 index 0000000000..4dd8df12ab --- /dev/null +++ b/test/minikube/001-kube.bats @@ -0,0 +1,61 @@ +#!/usr/bin/env bats +# +# Tests of podman kube commands with minikube +# + +load helpers.bash + +############################################################################### +# BEGIN tests + +@test "minikube - check cluster is up" { + run minikube kubectl get nodes + assert "$status" -eq 0 "get status of nodes" + assert "$output" =~ "Ready" + run minikube kubectl get pods + assert "$status" -eq 0 "get pods in the default namespace" + assert "$output" == "No resources found in default namespace." + wait_for_default_sa +} + +@test "minikube - deploy generated container yaml to minikube" { + cname="test-ctr" + fname="/tmp/minikube_deploy_$(random_string 6).yaml" + run_podman container create --name $cname $IMAGE top + run_podman kube generate -f $fname $cname + + # deploy to the minikube cluster + project="ctr-ns" + run minikube kubectl create namespace $project + assert "$status" -eq 0 "create new namespace $project" + run minikube kubectl -- apply -f $fname + echo $output >&2 + assert "$status" -eq 0 "deploy $fname to the cluster" + assert "$output" == "pod/$cname-pod created" + wait_for_pods_to_start + run minikube kubectl delete namespace $project + assert $status -eq 0 "delete namespace $project" +} + +@test "minikube - deploy generated pod yaml to minikube" { + pname="test-pod" + cname1="test-ctr1" + cname2="test-ctr2" + fname="/tmp/minikube_deploy_$(random_string 6).yaml" + + run_podman pod create --name $pname --publish 9999:8888 + run_podman container create --name $cname1 --pod $pname $IMAGE sleep 1000 + run_podman container create --name $cname2 --pod $pname $IMAGE sleep 2000 + run_podman kube generate -f $fname $pname + + # deploy to the minikube cluster + project="pod-ns" + run minikube kubectl create namespace $project + assert "$status" -eq 0 "create new namespace $project" + run minikube kubectl -- apply -f $fname + assert "$status" -eq 0 "deploy $fname to the cluster" + assert "$output" == "pod/$pname created" + wait_for_pods_to_start + run minikube kubectl delete namespace $project + assert $status -eq 0 "delete namespace $project" +} diff --git a/test/minikube/helpers.bash b/test/minikube/helpers.bash new file mode 100644 index 0000000000..a89594622e --- /dev/null +++ b/test/minikube/helpers.bash @@ -0,0 +1,60 @@ +# -*- bash -*- + +load ../system/helpers.bash + +function setup(){ + # only set up the minikube cluster before the first test + if [[ "$BATS_TEST_NUMBER" -eq 1 ]]; then + minikube start + fi + basic_setup +} + +function teardown(){ + # only delete the minikube cluster if we are done with the last test + # the $DEBUG_MINIKUBE env can be set to preserve the cluster to debug if needed + if [[ "$BATS_TEST_NUMBER" -eq ${#BATS_TEST_NAMES[@]} ]] && [[ "$DEBUG_MINIKUBE" == "" ]]; then + minikube delete + fi + basic_teardown +} + +function wait_for_default_sa(){ + count=0 + sa_ready=false + # timeout after 30 seconds + # if the default service account hasn't been created yet, there is something else wrong + while [[ $count -lt 30 ]] && [[ $sa_ready == false ]] + do + run minikube kubectl get sa + assert "$status" -eq 0 + if [[ "$output" != "No resources found in default namespace." ]]; then + sa_ready=true + fi + count=$((count + 1)) + sleep 1 + done + if [[ $sa_ready == false ]]; then + die "Timed out waiting for default service account to be created" + fi +} + +function wait_for_pods_to_start(){ + count=0 + running=false + # timeout after 30 seconds + # if the pod hasn't started running after 30 seconds, there is something else wrong + while [[ $count -lt 30 ]] && [[ $running == false ]] + do + run minikube kubectl get pods + assert "$status" -eq 0 + if [[ "$output" =~ "Running" ]]; then + running=true + fi + count=$((count + 1)) + sleep 1 + done + if [[ $running == false ]]; then + die "Timed out waiting for pod to move to 'Running' state" + fi +}