From a3dec77073b9e86ebf05d8f9062bad6d88f088f7 Mon Sep 17 00:00:00 2001 From: Sushanta Das Date: Wed, 18 Sep 2024 15:34:53 +0530 Subject: [PATCH] tests for inidividual tasks/stepactions --- .github/workflows/run-task-tests.yaml | 88 ++++++ hack/run-test.sh | 59 ++++ .../0.1/tests/pre-apply-task-hook.sh | 4 + .../0.1/tests/pre-apply-taskrun-hook.sh | 3 + task/git-clone/0.1/tests/run.yaml | 30 ++ test/common.sh | 261 ++++++++++++++++++ test/run-test.sh | 47 ++++ test/test-tasks.sh | 68 +++++ 8 files changed, 560 insertions(+) create mode 100644 .github/workflows/run-task-tests.yaml create mode 100644 hack/run-test.sh create mode 100644 task/git-clone/0.1/tests/pre-apply-task-hook.sh create mode 100644 task/git-clone/0.1/tests/pre-apply-taskrun-hook.sh create mode 100644 task/git-clone/0.1/tests/run.yaml create mode 100644 test/common.sh create mode 100755 test/run-test.sh create mode 100755 test/test-tasks.sh diff --git a/.github/workflows/run-task-tests.yaml b/.github/workflows/run-task-tests.yaml new file mode 100644 index 000000000..e75f73c9f --- /dev/null +++ b/.github/workflows/run-task-tests.yaml @@ -0,0 +1,88 @@ +name: Run Task Tests + +"on": + pull_request + +jobs: + run-task-tests: + runs-on: ubuntu-22.04 + steps: + - name: Get all changed files in the PR from task or test directories + id: changed-files + uses: tj-actions/changed-files@v45 + with: + # Avoid using single or double quotes for multiline patterns + files: | + task/** + test/** + + - name: Free Disk Space + if: steps.changed-files.outputs.any_changed == 'true' + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + docker-images: false + + - name: Checkout build-defintions Repository + if: steps.changed-files.outputs.any_changed == 'true' + uses: actions/checkout@v3 + with: + ref: "${{ github.event.pull_request.head.sha }}" + path: build-definitions + + - name: Checkout konflux-ci/konflux-ci Repository + if: steps.changed-files.outputs.any_changed == 'true' + uses: actions/checkout@v3 + with: + repository: 'konflux-ci/konflux-ci' + path: konflux-ci + + - name: Create k8s Kind Cluster + if: steps.changed-files.outputs.any_changed == 'true' + uses: helm/kind-action@v1 + with: + config: konflux-ci/kind-config.yaml + + - name: Show version information + if: steps.changed-files.outputs.any_changed == 'true' + run: | + kubectl version + kind version + + - name: Deploying Dependencies + if: steps.changed-files.outputs.any_changed == 'true' + run: | + cd $GITHUB_WORKSPACE/konflux-ci + ./deploy-deps.sh + + - name: Wait for the dependencies to be ready + if: steps.changed-files.outputs.any_changed == 'true' + run: | + cd $GITHUB_WORKSPACE/konflux-ci + ./wait-for-all.sh + + - name: Deploying Konflux + if: steps.changed-files.outputs.any_changed == 'true' + run: | + cd $GITHUB_WORKSPACE/konflux-ci + ./deploy-konflux.sh + + - name: List namespaces + if: steps.changed-files.outputs.any_changed == 'true' + run: | + kubectl get namespace + + - name: Deploy test resources + if: steps.changed-files.outputs.any_changed == 'true' + run: | + cd $GITHUB_WORKSPACE/konflux-ci + ./deploy-test-resources.sh + + - name: Run the task tests + if: steps.changed-files.outputs.any_changed == 'true' + env: + CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} + run: | + echo "CHANGED_FILES: ${CHANGED_FILES}" + cd $GITHUB_WORKSPACE/build-definitions + ./test/test-tasks.sh diff --git a/hack/run-test.sh b/hack/run-test.sh new file mode 100644 index 000000000..91ccd3d20 --- /dev/null +++ b/hack/run-test.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +set -ex + +cd $(git rev-parse --show-toplevel) +source test/common.sh + +if [[ -z ${@} || ${1} == "-h" ]];then + cat </dev/null 2>/dev/null || : + +${KUBECTL_CMD} create namespace ${tns} + +#create appstudio-pipeline SA in the test namespace if not already +if ! ${KUBECTL_CMD} get sa appstudio-pipeline -n ${tns} | grep 'appstudio-pipeline'; then + $KUBECTL_CMD create sa appstudio-pipeline -n ${tns} +fi + +test_resource_creation ${RESOURCE}/${NAME}/${VERSION}/tests \ No newline at end of file diff --git a/task/git-clone/0.1/tests/pre-apply-task-hook.sh b/task/git-clone/0.1/tests/pre-apply-task-hook.sh new file mode 100644 index 000000000..54467b9c7 --- /dev/null +++ b/task/git-clone/0.1/tests/pre-apply-task-hook.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +echo "example of pre-apply-task-hook, this script can be used to run kubernetes commands" +${KUBECTL_CMD} get ns diff --git a/task/git-clone/0.1/tests/pre-apply-taskrun-hook.sh b/task/git-clone/0.1/tests/pre-apply-taskrun-hook.sh new file mode 100644 index 000000000..232ece36d --- /dev/null +++ b/task/git-clone/0.1/tests/pre-apply-taskrun-hook.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "an example of pre-apply-taskrun-hook script" \ No newline at end of file diff --git a/task/git-clone/0.1/tests/run.yaml b/task/git-clone/0.1/tests/run.yaml new file mode 100644 index 000000000..a2ed5cba2 --- /dev/null +++ b/task/git-clone/0.1/tests/run.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: TaskRun +metadata: + name: git-clone-run-noargs +spec: + workspaces: + - name: output + emptyDir: {} + taskRef: + name: git-clone + params: + - name: url + value: https://github.com/kelseyhightower/nocode +--- +apiVersion: tekton.dev/v1beta1 +kind: TaskRun +metadata: + name: git-clone-run-tag +spec: + workspaces: + - name: output + emptyDir: {} + taskRef: + name: git-clone + params: + - name: url + value: https://github.com/kelseyhightower/nocode + - name: revision + value: 1.0.0 diff --git a/test/common.sh b/test/common.sh new file mode 100644 index 000000000..ae6f5e0b1 --- /dev/null +++ b/test/common.sh @@ -0,0 +1,261 @@ + +#!/bin/bash + +CATALOG_TEST_SKIP_CLEANUP=true + +# Define a custom kubectl path if you like +KUBECTL_CMD=${KUBECTL_CMD:-kubectl} + +TEST_RESULT_FILE=/tmp/test-result + +# Checks whether the given function exists. +function function_exists() { + [[ "$(type -t $1)" == "function" ]] +} + +# Set the return code that the test script will return. +# Parameters: $1 - return code (0-255) +function set_test_return_code() { + # kubetest teardown might fail and thus incorrectly report failure of the + # script, even if the tests pass. + # We store the real test result to return it later, ignoring any teardown + # failure in kubetest. + # TODO(adrcunha): Get rid of this workaround. + echo -n "$1"> ${TEST_RESULT_FILE} +} + +function detect_changed_e2e_test() { + # check if any file from test/ directory is changed + echo ${CHANGED_FILES} |grep "test[/^/]" +} + +function detect_new_changed_resources() { + # detect for changes in tests dir of the task + echo ${CHANGED_FILES} |grep 'task/[^\/]*/[^\/]*/tests/[^/]*'|xargs -I {} dirname {}|sed 's/\(tests\).*/\1/g' + # detect for changes in the task manifest + echo ${CHANGED_FILES} |grep 'task/[^\/]*/[^\/]*/*[^/]*.yaml'|xargs -I {} dirname {}|awk '{print $1"/tests"}' +} + +# Signal (as return code and in the logs) that all E2E tests passed. +function success() { + set_test_return_code 0 + echo "**************************************" + echo "*** E2E TESTS PASSED ***" + echo "**************************************" + exit 0 +} + +function show_failure() { + local testname=$1 tns=$2 + + echo "FAILED: ${testname} task has failed to comeback properly" ; + ${KUBECTL_CMD} api-resources + echo "Namespace: ${tns}" + echo "--- TaskRun Dump" + ${KUBECTL_CMD} get --ignore-not-found=true -n ${tns} taskrun -o yaml + echo "--- Task Dump" + ${KUBECTL_CMD} get --ignore-not-found=true -n ${tns} task -o yaml + echo "--- PipelineRun Dump" + ${KUBECTL_CMD} get --ignore-not-found=true -n ${tns} pipelinerun -o yaml + echo "--- Pipeline Dump" + ${KUBECTL_CMD} get --ignore-not-found=true -n ${tns} pipeline -o yaml + echo "--- StepAction Dump" + ${KUBECTL_CMD} get --ignore-not-found=true -n ${tns} stepaction -o yaml + echo "--- Container Logs" + for pod in $(${KUBECTL_CMD} get pod -o name -n ${tns}); do + echo "----POD_NAME: ${pod}---" + ${KUBECTL_CMD} logs --all-containers -n ${tns} ${pod} || true + done + exit 1 + +} + +function test_yaml_can_install() { + # Validate that all the StepAction/Task CRDs in this repo are valid by creating them in a NS. + ns="test-yaml-ns" + all_tasks="$*" + ${KUBECTL_CMD} create ns "${ns}" || true + + local runtest + for runtest in ${all_tasks}; do + # remove task/ or stepaction/ from beginning + local runtestdir=${runtest#*/} + # remove /0.1/tests from end + local testname=${runtestdir%%/*} + runtest=${runtest//tests} + + # in case a stepaction/task is being removed then it's directory + # doesn't exists, so skip the test for YAML + [ ! -d "${runtest%%/*}/${testname}" ] && continue + + runtest="${runtest}${testname}.yaml" + skipit= + for ignore in ${TEST_YAML_IGNORES};do + [[ ${ignore} == "${testname}" ]] && skipit=True + done + + # don't test the stepactions/tasks which are deprecated + cat ${runtest} | grep 'tekton.dev/deprecated: \"true\"' && skipit=True + + # In case if PLATFORM env variable is specified, do the tests only for matching stepactions/tasks + [[ -n ${PLATFORM} ]] && [[ $(grep "tekton.dev/platforms" ${runtest} 2>/dev/null) != *"${PLATFORM}"* ]] && skipit=True + + [[ -n ${skipit} ]] && continue + echo "Checking ${testname}" + ${KUBECTL_CMD} -n ${ns} apply -f <(sed "s/namespace:.*/namespace: ${ns}/" "${runtest}") + done +} + +function test_resource_creation() { + local runtest + declare -A resource_to_wait_for + + for runtest in $@;do + # remove task/ from beginning + local runtestdir=${runtest#*/} + # remove //tests from end + local testname=${runtestdir%%/*} + local version=$(basename $(basename $(dirname $runtest))) + # check version is in given format + [[ ${version} =~ ^[0-9]+\.[0-9]+$ ]] || { echo "ERROR: version of the task is not set properly"; exit 1;} + # replace . with - in version as not supported in namespace name + version="$( echo $version | tr '.' '-' )" + + local tns="${testname}-${version}" + local skipit= + + for ignore in ${TEST_TASKRUN_IGNORES};do + [[ ${ignore} == ${testname} ]] && skipit=True + done + + # remove /tests from end + local resourcedir=${runtest%/*} + + # check whether tests folder exists or not inside task dir + # if not then run the tests for next task (if any) + [[ ! -d $runtest ]] && skipit=True + + ls ${resourcedir}/*.yaml 2>/dev/null >/dev/null || skipit=True + + cat ${resourcedir}/*.yaml | grep 'tekton.dev/deprecated: \"true\"' && skipit=True + + # In case if PLATFORM env variable is specified, do the tests only for matching tasks + [[ -n ${PLATFORM} ]] && [[ $(grep "tekton.dev/platforms" ${resourcedir}/*.yaml 2>/dev/null) != *"${PLATFORM}"* ]] && skipit=True + + [[ -n ${skipit} ]] && continue + + # Delete the tns if already exists + ${KUBECTL_CMD} delete ns ${tns} >/dev/null 2>/dev/null || : + + # create the test namespace + ${KUBECTL_CMD} create namespace ${tns} >/dev/null 2>/dev/null + + # create the service account appstudio-pipeline (konflux spedific requirement) + $KUBECTL_CMD create sa appstudio-pipeline -n ${tns} + + # Install the task itself first. We can only have one YAML file + yaml=$(printf ${resourcedir}/*.yaml) + started=$(date '+%Hh%M:%S') + echo "${started} STARTING: ${testname}/${version}" + + # dry-run this YAML to validate and also get formatting side-effects. + # TODO: need to add tekton linter here + ${KUBECTL_CMD} -n ${tns} create -f ${yaml} --dry-run=client -o yaml >${TMPF} + + [[ -f ${resourcedir}/tests/pre-apply-task-hook.sh ]] && source ${resourcedir}/tests/pre-apply-task-hook.sh + function_exists pre-apply-task-hook && pre-apply-task-hook + + # Make sure we have deleted the content, this is in case of rerun + # and namespace hasn't been cleaned up or there is some Cluster* + # stuff, which really should not be allowed. + ${KUBECTL_CMD} -n ${tns} delete -f ${TMPF} >/dev/null 2>/dev/null || true + ${KUBECTL_CMD} -n ${tns} create -f ${TMPF} + + # Install resource and run + for yaml in ${runtest}/*.yaml;do + cp ${yaml} ${TMPF} + [[ -f ${resourcedir}/tests/pre-apply-taskrun-hook.sh ]] && source ${resourcedir}/tests/pre-apply-taskrun-hook.sh + function_exists pre-apply-taskrun-hook && pre-apply-taskrun-hook + + # Make sure we have deleted the content, this is in case of rerun + # and namespace hasn't been cleaned up or there is some Cluster* + # stuff, which really should not be allowed. + ${KUBECTL_CMD} -n ${tns} delete -f ${TMPF} >/dev/null 2>/dev/null || true + ${KUBECTL_CMD} -n ${tns} create -f ${TMPF} + done + + resource_to_wait_for["$testname/${version}"]="${tns}|$started" + + done + + # I would refactor this to a function but bash limitation is too great, really need a rewrite the sooner + # the uglness to pass a hashmap to a function https://stackoverflow.com/a/17557904/145125 + local cnt=0 + local all_status='' + local reason='' + local maxloop=60 # 10 minutes max + + set +x + while true;do + # If we have timed out then show failures of what's remaining in + # resource_to_wait_for we assume only first one fails this + [[ ${cnt} == "${maxloop}" ]] && { + for testname in "${!resource_to_wait_for[@]}";do + target_ns=${resource_to_wait_for[$testname]} + show_failure "${testname}" "${target_ns}" + done + } + [[ -z ${resource_to_wait_for[*]} ]] && { + break + } + + for testname in "${!resource_to_wait_for[@]}";do + target_ns=${resource_to_wait_for[$testname]%|*} + started=${resource_to_wait_for[$testname]#*|} + # sometimes we don't get all_status and reason in one go so + # wait until we get the reason and all_status for 5 iterations + for tektontype in pipelinerun taskrun;do + for _ in {1..10}; do + all_status=$(${KUBECTL_CMD} get -n ${target_ns} ${tektontype} --output=jsonpath='{.items[*].status.conditions[*].status}') + reason=$(${KUBECTL_CMD} get -n ${target_ns} ${tektontype} --output=jsonpath='{.items[*].status.conditions[*].reason}') + result_values=$(${KUBECTL_CMD} get -n ${target_ns} ${tektontype} --output=jsonpath='{.items[*].status.results[*].value}') + [[ ! -z ${all_status} ]] && [[ ! -z ${reason} ]] && break + sleep 1 + done + # No need to check taskrun if pipelinerun has been set + [[ ! -z ${all_status} ]] && [[ ! -z ${reason} ]] && break + + done + + if [[ -z ${all_status} || -z ${reason} ]];then + echo "Could not find a created taskrun or pipelinerun in ${target_ns}" + fi + + breakit=True + for status in ${all_status};do + [[ ${status} == *ERROR || ${reason} == *Fail* || ${reason} == Couldnt* ]] && show_failure ${testname} ${target_ns} + + if [[ ${status} != True ]];then + breakit= + fi + done + + if [[ -z ${result_values} ]]; then + for value in ${result_values}; do + [[ ${value} == "" ]] && show_failure ${testname} ${target_ns} + done + fi + + if [[ ${breakit} == True ]];then + unset resource_to_wait_for[$testname] + [[ -z ${CATALOG_TEST_SKIP_CLEANUP} ]] && ${KUBECTL_CMD} delete ns ${target_ns} >/dev/null + echo "${started}::$(date '+%Hh%M:%S') SUCCESS: ${testname} testrun has successfully executed" ; + fi + + done + + sleep 10 + cnt=$((cnt+1)) + done + set -x +} \ No newline at end of file diff --git a/test/run-test.sh b/test/run-test.sh new file mode 100755 index 000000000..921b8a374 --- /dev/null +++ b/test/run-test.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +set -ex + +cd $(git rev-parse --show-toplevel) +source test/common.sh + +if [[ -z ${@} || ${1} == "-h" ]];then + cat < 8 + +# You can ignore some yaml tests by providing the TEST_YAML_IGNORES variable +# with the test name separated by a space, for example: +# +# TEST_YAML_IGNORES="kaniko s2i" +# +# will ignore the kaniko and s2i tests, +# +TEST_YAML_IGNORES=${TEST_YAML_IGNORES:-""} + +# Allow ignoring some yaml tests, space separated, should be the basename of the +# test for example "s2i" +TEST_TASKRUN_IGNORES=${TEST_TASKRUN_IGNORES:-""} + +# Define this variable if you want to run all tests and not just the modified one. +TEST_RUN_ALL_TESTS=${TEST_RUN_ALL_TESTS:-""} + +set -ex +set -o pipefail + +all_tests=$(echo task/*/*/tests) + +[[ -z ${TEST_RUN_ALL_TESTS} ]] && [[ ! -z $(detect_changed_e2e_test) ]] && TEST_RUN_ALL_TESTS=1 + +if [[ -z ${TEST_RUN_ALL_TESTS} ]];then + all_tests=$(detect_new_changed_resources|sort -u || true) + [[ -z ${all_tests} ]] && { + echo "No tests has been detected in this PR. exiting." + success + } +fi + +test_yaml_can_install "${all_tests}" + +function test_resources { + local cnt=0 + local resource_to_tests="" + + for runtest in $@;do + resource_to_tests="${resource_to_tests} ${runtest}" + if [[ ${cnt} == "${MAX_NUMBERS_OF_PARALLEL_TASKS}" ]];then + test_resource_creation "${resource_to_tests}" + cnt=0 + resource_to_tests="" + continue + fi + cnt=$((cnt+1)) + done + + # in case if there are some remaining resources + if [[ -n ${resource_to_tests} ]];then + test_resource_creation "${resource_to_tests}" + fi +} + +test_resources "${all_tests}" + +success \ No newline at end of file