Skip to content

Commit

Permalink
Merge pull request #175 from wallarm/NODE-5583
Browse files Browse the repository at this point in the history
Node-5583 Required variables for smoke-tests
  • Loading branch information
braek-neck authored Aug 9, 2024
2 parents 14f8ab4 + 80e4272 commit 265784f
Show file tree
Hide file tree
Showing 6 changed files with 241 additions and 163 deletions.
19 changes: 19 additions & 0 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,22 @@ jobs:
- name: Checkout
uses: actions/checkout@v4

- name: Import secrets
uses: hashicorp/vault-action@130d1f5f4fe645bb6c83e4225c04d64cfb62de6e # v2.5.0
id: secrets
with:
exportEnv: true
url: ${{ secrets.VAULT_URL }}
role: ${{ secrets.VAULT_ROLE }}
method: kubernetes
path: kubernetes-ci
secrets: |
kv-gitlab-ci/data/github/shared/dockerhub-creds user | DOCKERHUB_USER ;
kv-gitlab-ci/data/github/shared/dockerhub-creds password | DOCKERHUB_PASSWORD ;
- name: Build
run: |
echo ${DOCKERHUB_PASSWORD} | docker login -u ${DOCKERHUB_USER} --password-stdin
IMAGE="wallarm/sidecar-controller"
make build TAG=${GITHUB_SHA} IMAGE=${IMAGE} PLATFORMS=linux/${{ matrix.ARCH }} BUILDX_ARGS=--load
docker save -o sidecar-${{ matrix.ARCH }}.tar ${IMAGE}:${GITHUB_SHA}
Expand Down Expand Up @@ -117,6 +131,8 @@ jobs:
kv-gitlab-ci/data/github/${{ env.env_code }} allure_server_token ;
kv-gitlab-ci/data/github/shared/smoke-tests-registry-creds token_name ;
kv-gitlab-ci/data/github/shared/smoke-tests-registry-creds token_secret ;
kv-gitlab-ci/data/github/shared/dockerhub-creds user ;
kv-gitlab-ci/data/github/shared/dockerhub-creds password ;
- name: Checkout
uses: actions/checkout@v4
Expand All @@ -136,6 +152,8 @@ jobs:
env:
SKIP_CLUSTER_CREATION: true
SKIP_IMAGE_CREATION: true
DOCKERHUB_USER: ${{ steps.secrets.outputs.user }}
DOCKERHUB_PASSWORD: ${{ steps.secrets.outputs.password }}
INJECTION_STRATEGY: ${{ matrix.INJECTION_STRATEGY }}
WALLARM_API_TOKEN: ${{ steps.secrets.outputs.api_token }}
WALLARM_API_HOST: ${{ steps.secrets.outputs.api_host }}
Expand All @@ -145,6 +163,7 @@ jobs:
SMOKE_REGISTRY_SECRET: ${{ steps.secrets.outputs.token_secret }}
WEBHOOK_API_KEY: ${{ steps.secrets.outputs.webhook_api_key }}
WEBHOOK_UUID: ${{ steps.secrets.outputs.webhook_uuid }}
ALLURE_PROJECT_ID: 10
ALLURE_UPLOAD_REPORT: true
ALLURE_GENERATE_REPORT: true
ALLURE_TOKEN: ${{ steps.secrets.outputs.allure_server_token }}
Expand Down
16 changes: 13 additions & 3 deletions example-env.ini
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,16 @@
### the main docker containers via docker-compose.
### Please rename this file to "env.ini" for makeing
### it work.
###
WALLARM_API_TOKEN=xxxxx-base64-base64-base64-base64-xxxxx
WALLARM_API_HOST=api.wallarm.com
# Wallarm API settings
WALLARM_API_TOKEN=...
WALLARM_API_HOST=us1.api.wallarm.com
WALLARM_API_PRESET=us1
CLIENT_ID=...
USER_TOKEN=...
SMOKE_REGISTRY_TOKEN=...
SMOKE_REGISTRY_SECRET=glpat-...

WEBHOOK_API_KEY=...
WEBHOOK_UUID=...

PYTEST_WORKERS=10
167 changes: 167 additions & 0 deletions test/smoke/functions.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
# This file used for import in other files

RED='\033[0;31m'
NC='\033[0m'

function check_mandatory_vars() {

declare -a mandatory
declare -a allure_mandatory

mandatory=(
WALLARM_API_TOKEN
WALLARM_API_HOST
WALLARM_API_PRESET
CLIENT_ID
USER_TOKEN
WEBHOOK_API_KEY
WEBHOOK_UUID
SMOKE_REGISTRY_TOKEN
SMOKE_REGISTRY_SECRET
)

env_list=""

for var in "${mandatory[@]}"; do
if [[ -z "${!var:-}" ]]; then
env_list+=" $var"
fi
done

if [[ "${ALLURE_UPLOAD_REPORT:-false}" == "true" ]]; then
allure_mandatory=(
ALLURE_TOKEN
ALLURE_ENVIRONMENT_ARCH
ALLURE_PROJECT_ID
ALLURE_GENERATE_REPORT
ALLURE_ENVIRONMENT_K8S
)

for var in "${allure_mandatory[@]}"; do
if [[ -z "${!var:-}" ]]; then
env_list+=" $var"
fi
done
fi

if [[ -n "$env_list" ]]; then
for var in ${env_list}; do
echo -e "${RED}Environment variable $var must be set${NC}"
done
exit 1
fi

}

function cleanup() {
if [[ "${KUBETEST_IN_DOCKER:-}" == "true" ]]; then
kind "export" logs --name ${KIND_CLUSTER_NAME} "${ARTIFACTS}/logs" || true
fi
if [[ "${CI:-}" == "true" ]]; then
kind delete cluster \
--verbosity=${KIND_LOG_LEVEL} \
--name ${KIND_CLUSTER_NAME}
fi
}

function get_controller_logs_and_fail() {
echo "#################################"
echo "######## Controller logs ########"
echo "#################################"
kubectl logs -l "app.kubernetes.io/component=controller" --tail=-1 || true
echo -e "#################################\n"

echo "#####################################"
echo "######## Post-analytics logs ########"
echo -e "#####################################\n"
for CONTAINER in antibot appstructure supervisord tarantool ; do
echo "#######################################"
echo "###### ${CONTAINER} container logs ######"
echo -e "#######################################\n"
kubectl logs -l "app.kubernetes.io/component=postanalytics" -c ${CONTAINER} --tail=-1 || true
echo -e "#######################################\n"
done

for COMPONENT in controller postanalytics ; do
echo "#######################################"
echo "###### Describe ${COMPONENT} pod ######"
echo -e "#######################################\n"
kubectl describe po -l "app.kubernetes.io/component=${COMPONENT}"
echo -e "#######################################\n"
done

exit 1
}

###

function get_logs_and_fail() {
get_logs
extra_debug_logs
clean_allure_report
exit 1
}

function get_logs() {
echo "#################################"
echo "######## Controller logs ########"
echo "#################################"
kubectl logs -l "app.kubernetes.io/component=controller" --tail=-1
echo -e "#################################\n"

echo "#################################"
echo "######## Post-analytics Pod #####"
echo "#################################"
for CONTAINER in antibot appstructure supervisord tarantool ; do
echo "#######################################"
echo "###### ${CONTAINER} container logs ######"
echo -e "#######################################\n"
kubectl logs -l "app.kubernetes.io/component=postanalytics" -c ${CONTAINER} --tail=-1
echo -e "#######################################\n"
done

echo "#################################"
echo "######## Application Pod ########"
echo -e "#################################\n"

echo "####################################################"
echo "###### List directory /opt/wallarm/etc/wallarm #####"
echo "####################################################"
kubectl exec "${POD}" -c sidecar-proxy -- sh -c "ls -laht /opt/wallarm/etc/wallarm && cat /opt/wallarm/etc/wallarm/node.yaml" || true
echo -e "#####################################################\n"

echo "############################################"
echo "###### List directory /var/lib/nginx/wallarm"
echo "############################################"
kubectl exec "${POD}" -c sidecar-proxy -- sh -c "ls -laht /opt/wallarm/var/lib/nginx/wallarm && ls -laht /opt/wallarm/var/lib/nginx/wallarm/shm" || true
echo -e "############################################\n"

echo "############################################################"
echo "###### List directory /opt/wallarm/var/lib/wallarm-acl #####"
echo "############################################################"
kubectl exec "${POD}" -c sidecar-proxy -- sh -c "ls -laht /opt/wallarm/var/lib/wallarm-acl" || true
echo -e "############################################################\n"

echo "#################################"
echo "######## Application Pod Logs ###"
echo -e "#################################\n"
kubectl logs -l "wallarm-sidecar=enabled" --all-containers --ignore-errors --since=1h
echo -e "#################################\n"
}

function extra_debug_logs {
echo "############################################"
echo "###### Extra cluster debug info ############"
echo "############################################"

echo "Grepping cluster OOMKilled events..."
kubectl get events -A | grep -i OOMKill || true

echo "Displaying pods state in default namespace..."
kubectl get pods

}

function clean_allure_report() {
[[ "$ALLURE_GENERATE_REPORT" == false && -d "allure_report" ]] && rm -rf allure_report/* 2>/dev/null || true
}
102 changes: 7 additions & 95 deletions test/smoke/run-smoke-suite.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@ set -o errexit
set -o nounset
set -o pipefail

# check if all mandatory vars was defined
source "${PWD}/test/smoke/functions.sh"
check_mandatory_vars

export KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME:-ingress-smoke-test}
export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/kind-config-$KIND_CLUSTER_NAME}"

Expand All @@ -20,13 +24,13 @@ SMOKE_IMAGE_TAG="${SMOKE_IMAGE_TAG:-latest}"

# Allure related variables
ALLURE_ENDPOINT="${ALLURE_ENDPOINT:-https://allure.wallarm.com}"
ALLURE_PROJECT_ID=${ALLURE_PROJECT_ID:-10}
ALLURE_PROJECT_ID=${ALLURE_PROJECT_ID}
ALLURE_RESULTS="${ALLURE_RESULTS:-/tests/_out/allure_report}"
ALLURE_UPLOAD_REPORT="${ALLURE_UPLOAD_REPORT:-false}"
ALLURE_GENERATE_REPORT="${ALLURE_GENERATE_REPORT:-false}"

# Pytest related variables
CLIENT_ID="${CLIENT_ID:-5}"
CLIENT_ID="${CLIENT_ID}"
WALLARM_API_CA_VERIFY="${WALLARM_API_CA_VERIFY:-true}"
WALLARM_API_HOST="${WALLARM_API_HOST:-api.wallarm.com}"
WALLARM_API_PRESET="${WALLARM_API_PRESET:-eu1}"
Expand All @@ -36,98 +40,6 @@ PYTEST_WORKERS="${PYTEST_WORKERS:-15}"
#TODO We need it here just to don't let test fail. Remove this variable when test will be fixed.
HOSTNAME_OLD_NODE="smoke-tests-old-node"

function clean_allure_report() {
[[ "$ALLURE_GENERATE_REPORT" == false && -d "allure_report" ]] && rm -rf allure_report/* 2>/dev/null || true
}

function get_logs_and_fail() {
get_logs
extra_debug_logs
clean_allure_report
exit 1
}

function get_logs() {
echo "#################################"
echo "######## Controller logs ########"
echo "#################################"
kubectl logs -l "app.kubernetes.io/component=controller" --tail=-1
echo -e "#################################\n"

echo "#################################"
echo "######## Post-analytics Pod #####"
echo "#################################"
for CONTAINER in appstructure supervisord tarantool ; do
echo "#######################################"
echo "###### ${CONTAINER} container logs ######"
echo -e "#######################################\n"
kubectl logs -l "app.kubernetes.io/component=postanalytics" -c ${CONTAINER} --tail=-1
echo -e "#######################################\n"
done

echo "#################################"
echo "######## Application Pod ########"
echo -e "#################################\n"

echo "####################################################"
echo "###### List directory /opt/wallarm/etc/wallarm #####"
echo "####################################################"
kubectl exec "${POD}" -c sidecar-proxy -- sh -c "ls -laht /opt/wallarm/etc/wallarm && cat /opt/wallarm/etc/wallarm/node.yaml" || true
echo -e "#####################################################\n"

echo "############################################"
echo "###### List directory /var/lib/nginx/wallarm"
echo "############################################"
kubectl exec "${POD}" -c sidecar-proxy -- sh -c "ls -laht /opt/wallarm/var/lib/nginx/wallarm && ls -laht /opt/wallarm/var/lib/nginx/wallarm/shm" || true
echo -e "############################################\n"

echo "############################################################"
echo "###### List directory /opt/wallarm/var/lib/wallarm-acl #####"
echo "############################################################"
kubectl exec "${POD}" -c sidecar-proxy -- sh -c "ls -laht /opt/wallarm/var/lib/wallarm-acl" || true
echo -e "############################################################\n"

echo "#################################"
echo "######## Application Pod Logs ###"
echo -e "#################################\n"
kubectl logs -l "wallarm-sidecar=enabled" --all-containers --ignore-errors --since=1h
echo -e "#################################\n"
}


function extra_debug_logs {
echo "############################################"
echo "###### Extra cluster debug info ############"
echo "############################################"

echo "Grepping cluster OOMKilled events..."
kubectl get events -A | grep -i OOMKill || true

echo "Displaying pods state in default namespace..."
kubectl get pods

}

declare -a mandatory
mandatory=(
CLIENT_ID
USER_TOKEN
SMOKE_REGISTRY_TOKEN
SMOKE_REGISTRY_SECRET
)

missing=false
for var in "${mandatory[@]}"; do
if [[ -z "${!var:-}" ]]; then
echo "Environment variable $var must be set"
missing=true
fi
done

if [ "$missing" = true ]; then
exit 1
fi

if [[ "${CI:-false}" == "false" ]]; then
trap 'kubectl delete pod pytest --now --ignore-not-found' EXIT ERR
# Colorize pytest output if run locally
Expand Down Expand Up @@ -229,4 +141,4 @@ EXEC_CMD="env $GITHUB_VARS $RUN_TESTS -n ${PYTEST_WORKERS} ${PYTEST_ARGS}"
# shellcheck disable=SC2086
kubectl exec pytest ${EXEC_ARGS} -- ${EXEC_CMD} || get_logs_and_fail
extra_debug_logs
clean_allure_report
clean_allure_report
Loading

0 comments on commit 265784f

Please sign in to comment.