From a1a7924e5984b438cfb4d5be1e3964d7fca28fb6 Mon Sep 17 00:00:00 2001 From: Yongming Ding Date: Fri, 11 Feb 2022 11:19:12 -0800 Subject: [PATCH] Add CI validation job for flow visibility Signed-off-by: Yongming Ding --- ci/jenkins/jobs/macros.yaml | 17 ++++ ci/jenkins/jobs/projects.yaml | 15 ++++ ci/test-flow-visibility.sh | 153 ++++++++++++++++++++++++++++++++++ 3 files changed, 185 insertions(+) create mode 100755 ci/test-flow-visibility.sh diff --git a/ci/jenkins/jobs/macros.yaml b/ci/jenkins/jobs/macros.yaml index ddfa33943db..7492aa3184b 100644 --- a/ci/jenkins/jobs/macros.yaml +++ b/ci/jenkins/jobs/macros.yaml @@ -141,6 +141,23 @@ echo "=== FAILURE !!! ===" exit 1 +- builder: + name: builder-flow-visibility + builders: + - shell: |- + #!/bin/bash + set -ex + ./ci/jenkins/test-vmc.sh --cluster-name "${JOB_NAME}-${BUILD_NUMBER}" --setup-only + ./ci/test-flow-visibility.sh --kubeconfig jenkins/out/kubeconfig + ./ci/jenkins/test-vmc.sh --cluster-name "${JOB_NAME}-${BUILD_NUMBER}" --cleanup-only + + if !(test -f TEST_FAILURE); then + echo "=== SUCCESS !!! ===" + exit 0 + fi + echo "=== FAILURE !!! ===" + exit 1 + - builder: name: builder-matrix builders: diff --git a/ci/jenkins/jobs/projects.yaml b/ci/jenkins/jobs/projects.yaml index 873965c1318..32baeb08480 100644 --- a/ci/jenkins/jobs/projects.yaml +++ b/ci/jenkins/jobs/projects.yaml @@ -73,6 +73,21 @@ - email: recipients: projectantrea-dev@googlegroups.com wrappers: [] + - '{name}-{test_name}-for-period': + test_name: daily-flow-visibility-validate + node: 'antrea-test-node' + description: 'This is for validating the flow visibility manifest daily.' + builders: + - builder-flow-visibility + branches: + - '*/main' + included_regions: [] + cron: 'H H * * *' + ignore_post_commit_hooks: false + publishers: + - email: + recipients: projectantrea-dev@googlegroups.com + wrappers: [] - '{name}-{test_name}-for-gc': test_name: workload-cluster node: 'antrea-test-node' diff --git a/ci/test-flow-visibility.sh b/ci/test-flow-visibility.sh new file mode 100755 index 00000000000..f3c7ce9e6bb --- /dev/null +++ b/ci/test-flow-visibility.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash + +# Copyright 2022 Antrea Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +FAILURE=false +THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +GIT_CHECKOUT_DIR=${THIS_DIR}/.. + +_usage="Usage: $0 [--kubeconfig ] + +Setup Grafana, ClickHouse and Antrea Flow Aggregator to validate flow-aggregator.yml and flow-visibility.yml. + + --kubeconfig Path to kubeconfig. + +" + +echoerr() { + >&2 echo "$@" +} + +print_usage() { + echoerr "$_usage" +} + + +while [[ $# -gt 0 ]] +do +key="$1" +case $key in + --kubeconfig) + export KUBECONFIG="$2" + shift 2 + ;; + -h|--help) + print_usage + exit 0 + ;; + *) + echoerr "Unknown option $1" + exit 1 + ;; +esac +done + +start_antrea() { + echo "=== Starting Antrea ===" + kubectl apply -f ${GIT_CHECKOUT_DIR}/build/yamls/antrea.yml + kubectl rollout status --timeout=5m deployment/coredns -n kube-system + kubectl rollout status --timeout=5m deployment.apps/antrea-controller -n kube-system + kubectl rollout status --timeout=5m daemonset/antrea-agent -n kube-system +} + +setup_flow_visibility() { + echo "=== Starting Flow Visibility ===" + # install ClickHouse operator + kubectl apply -f https://raw.githubusercontent.com/Altinity/clickhouse-operator/master/deploy/operator/clickhouse-operator-install-bundle.yaml + kubectl apply -f ${GIT_CHECKOUT_DIR}/build/yamls/flow-visibility.yml + echo "=== Waiting for ClickHouse and Grafana to be ready ===" + sleep 10 + kubectl wait --for=condition=ready pod -l app=clickhouse-operator -n kube-system --timeout=600s + kubectl wait --for=condition=ready pod -l app=clickhouse -n flow-visibility --timeout=600s + kubectl wait --for=condition=ready pod -l app=grafana -n flow-visibility --timeout=600s +} + +check_grafana() { + echo "=== Check the installation of Grafana plugins ===" + grafana_logs=$(kubectl logs -n flow-visibility $(kubectl -n flow-visibility get pod -l app=grafana -o jsonpath="{.items[0].metadata.name}")) + if ( echo ${grafana_logs} | grep -q 'msg="Plugin registered" logger=plugin.manager pluginId=antreaflowvisibility-grafana-sankey-plugin' ); then + echo "=== antreaflowvisibility-grafana-sankey-plugin installed correctly ===" + else + echo "=== antreaflowvisibility-grafana-sankey-plugin is NOT installed correctly ===" + FAILURE=true + fi + if ( echo ${grafana_logs} | grep -q 'msg="Plugin registered" logger=plugin.manager pluginId=grafana-clickhouse-datasource' ); then + echo "=== grafana-clickhouse-datasource installed correctly ===" + else + echo "=== grafana-clickhouse-datasource is NOT installed correctly ===" + FAILURE=true + fi +} + +check_clickhouse_monitor() { + echo "=== Check the status of ClickHouse monitor cron job status===" + sleep 60 + CLICKHOUSE_MONITOR=$(kubectl get job -l app=clickhouse-monitor -n flow-visibility -o jsonpath="{.items[-1].metadata.name}") + kubectl wait --for=condition=complete job/$CLICKHOUSE_MONITOR -n flow-visibility --timeout=60s +} + +config_antrea() { + echo "=== Stopping Antrea === " + kubectl delete -f ${GIT_CHECKOUT_DIR}/build/yamls/antrea.yml + echo "=== Enable Antrea Flow Exporter ===" + sed -i -e "s/# FlowExporter: false/ FlowExporter: true/g" ${GIT_CHECKOUT_DIR}/build/yamls/antrea.yml +} + +setup_flow_aggregator() { + echo "=== Config Antrea Flow Aggregator ===" + sed -i -e ":a;N;s/ # Enable is the switch to enable exporting flow records to ClickHouse.\n #enable: false/ # Enable is the switch to enable exporting flow records to ClickHouse.\n enable: true/g;ta" ${GIT_CHECKOUT_DIR}/build/yamls/flow-aggregator.yml + echo "=== Start Antrea Flow Aggregator ===" + kubectl apply -f ${GIT_CHECKOUT_DIR}/build/yamls/flow-aggregator.yml + echo "=== Waiting for Antrea Flow Aggregator to be ready ===" + kubectl wait --for=condition=ready pod -l app=flow-aggregator -n flow-aggregator --timeout=600s +} + +# Antrea Flow Aggregator starts to insert flow records into ClickHouse database. +# This function will check if there are flow records inserted in desired table of database. +check_record() { + echo "=== Wait for up to 5 minutes to receive flow records ===" + for i in `seq 5` + do + sleep 60 + echo "=== Get flow record (try for 1m) ===" + clickhouse_output=$(kubectl exec -it chi-clickhouse-clickhouse-0-0-0 -n flow-visibility -- clickhouse-client -h clickhouse-clickhouse.flow-visibility.svc -u clickhouse_operator --password clickhouse_operator_password --query="select count(*) from flows") + # Remove all new line, return, tab from the output string, to allow integer comparison. + row_number="${clickhouse_output//[$'\t\r\n ']}" + echo $row_number + if [ $row_number -gt 0 ]; then + echo "=== Record is received correctly ===" + break + elif [ $i == 5 ]; then + echo "=== Record is NOT received correctly ===" + FAILURE=true + fi + done +} + +start_antrea +setup_flow_visibility +check_grafana +check_clickhouse_monitor +config_antrea +start_antrea +setup_flow_aggregator +check_record + +if ( ${FAILURE} == true ); then + echo "=== TEST FAILURE !! ===" + touch TEST_FAILURE +fi