Skip to content
This repository has been archived by the owner on May 22, 2021. It is now read-only.

Commit

Permalink
[AIRFLOW-6564] Additional diagnostics information on CI check failure (…
Browse files Browse the repository at this point in the history 
…apache#7172)
  • Loading branch information
@potiuk @galuszkak
potiuk authored and galuszkak committed Mar 5, 2020
1 parent d8b849d commit c3722dc
Show file tree
Hide file tree
Showing 5 changed files with 262 additions and 0 deletions.
6 changes: 6 additions & 0 deletions airflow/bin/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -837,6 +837,12 @@ class CLIFactory:
'help': "Runs a shell to access the database",
'args': tuple(),
},
{
'func': lazy_load_command('airflow.cli.commands.db_command.check'),
'name': 'check',
'help': "Check if the database can be reached.",
'args': tuple(),
},
)
CONNECTIONS_COMMANDS = (
{
Expand Down
6 changes: 6 additions & 0 deletions airflow/cli/commands/db_command.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,3 +81,9 @@ def shell(args):
subprocess.Popen(["psql"], env=env).wait()
else:
raise AirflowException(f"Unknown driver: {url.drivername}")


@cli_utils.action_logging
def check(_):
"""Runs a check command that checks if db is available."""
db.check()
10 changes: 10 additions & 0 deletions airflow/utils/db.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -501,3 +501,13 @@ def resetdb():
Base.metadata.drop_all(connection) # pylint: disable=no-member

initdb()


@provide_session
def check(session=None):
"""
Checks if the database works.
:param session: session of the sqlalchemy
"""
session.execute('select 1 as is_alive;')
log.info("Connection successful.")
238 changes: 238 additions & 0 deletions scripts/ci/in_container/check_environment.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,238 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

# Script to check licences for all code. Can be started from any working directory
set -euo pipefail

MY_DIR=$(cd "$(dirname "$0")" || exit 1; pwd)

# shellcheck source=scripts/ci/in_container/_in_container_utils.sh
. "${MY_DIR}/_in_container_utils.sh"

in_container_basic_sanity_check

in_container_script_start


function on_exit() {
#shellcheck disable=2181
EXIT_CODE=$?
if [[ ${EXIT_CODE} != 0 ]]; then
echo "###########################################################################################"
echo " EXITING WITH STATUS CODE ${EXIT_CODE}"
echo "###########################################################################################"
echo " Docker processes:"
echo "###########################################################################################"
docker ps --no-trunc
echo "###########################################################################################"
for CONTAINER in $(docker ps -qa)
do
CONTAINER_NAME=$(docker inspect --format "{{.Name}}" "${CONTAINER}")
echo "-------------------------------------------------------------------------------------------"
echo " Docker inspect: ${CONTAINER_NAME}"
echo "-------------------------------------------------------------------------------------------"
echo
docker inspect "${CONTAINER}"
echo
echo "-------------------------------------------------------------------------------------------"
echo " Docker logs: ${CONTAINER_NAME}"
echo "-------------------------------------------------------------------------------------------"
echo
docker logs "${CONTAINER}"
echo
echo "###########################################################################################"
done
fi
}

export EXIT_CODE=0

function check_integration {
INTEGRATION_NAME=$1
CALL=$2
MAX_CHECK=${3:=1}

echo "==============================================================================================="
echo " Checking integration ${INTEGRATION_NAME}"
echo "==============================================================================================="

ENV_VAR_NAME=INTEGRATION_${INTEGRATION_NAME^^}
if [[ ${!ENV_VAR_NAME:=} != "true" ]]; then
echo " Integration ${INTEGRATION_NAME} disabled. Not checking"
echo "==============================================================================================="
return
fi

while true
do
echo "Executing: ${CALL}"
echo "-----------------------------------------------------------------------------------------------"
set +e
eval "${CALL}"
RES=$?
set -e
echo "-----------------------------------------------------------------------------------------------"
if [[ ${RES} == 0 ]]; then
echo " Integration ${INTEGRATION_NAME} OK!"
break
else
echo " ${INTEGRATION_NAME} is not yet ready -> exit code ${RES}"
MAX_CHECK=$((MAX_CHECK-1))
fi
if [[ ${MAX_CHECK} == 0 ]]; then
echo
echo "ERROR! Maximum number of retries while checking ${INTEGRATION_NAME} integration. Exiting"
echo
break
else
echo
echo "Sleeping! ${MAX_CHECK} retries left!"
echo
sleep 1
fi
done
if [[ ${RES} != 0 ]]; then
echo " ERROR: Integration ${INTEGRATION_NAME} could not be started!"
export EXIT_CODE=${RES}
fi
echo "==============================================================================================="
}

trap on_exit EXIT
echo
echo "Check CI environment sanity!"
echo

export EXIT_CODE=0

if [[ -n ${BACKEND:=} ]]; then
echo "==============================================================================================="
echo " Checking backend: ${BACKEND}"
echo "==============================================================================================="

set +e
if [[ ${BACKEND} == "mysql" ]]; then
# Wait until mysql is ready!
MYSQL_CONTAINER=$(docker ps -qf "name=mysql")
echo "MySQL container: ${MYSQL_CONTAINER}"
if [[ -z ${MYSQL_CONTAINER} ]]; then
echo
echo "ERROR! MYSQL container is not started. Exiting!"
echo
exit 1
fi
MAX_CHECK=60
while true
do
echo
echo "Checking if MySQL is ready for connections"
CONNECTION_READY_MESSAGES=$(docker logs "${MYSQL_CONTAINER}" 2>&1 | \
grep -c "mysqld: ready for connections" )
# MySQL when starting from dockerfile starts a temporary server first because it
# starts with an empty database first and it will create the airflow database and then
# it will start a second server to serve this newly created database
# That's why we should wait until docker logs contain "ready for connections" twice
# more info: https://github.com/docker-library/mysql/issues/527
if [[ ${CONNECTION_READY_MESSAGES} == 2 ]];
then
echo
echo "MySQL is ready for connections!"
echo
break
else
echo
echo "Number of 'ready for connections' in MySQL logs: ${CONNECTION_READY_MESSAGES}"
echo
fi
MAX_CHECK=$((MAX_CHECK-1))
if [[ ${MAX_CHECK} == 0 ]]; then
echo
echo "ERROR! Maximum number of retries while waiting for MySQL. Exiting"
echo
exit 1
else
echo
echo "Sleeping! ${MAX_CHECK} retries left!"
echo
sleep 1
fi
done
fi

MAX_CHECK=3
while true
do
AIRFLOW__CORE__LOGGING_LEVEL=error airflow db check
RES=$?
if [[ ${RES} == 0 ]]; then
break
fi
MAX_CHECK=$((MAX_CHECK-1))
if [[ ${MAX_CHECK} == 0 ]]; then
echo
echo "ERROR! Maximum number of retries while connecting to DB. Exiting"
echo
exit 1
else
echo
echo "Sleeping! ${MAX_CHECK} retries left!"
echo
sleep 1
fi
done
set -e
else
echo "==============================================================================================="
echo " Skip checking backend - BACKEND not set"
echo "==============================================================================================="
if [[ ${RES} == 0 ]]; then
echo "-----------------------------------------------------------------------------------------------"
echo
echo "Backend database is sane"
echo
echo "-----------------------------------------------------------------------------------------------"
else
echo "-----------------------------------------------------------------------------------------------"
echo
echo "Error when checking backend database"
echo
echo "-----------------------------------------------------------------------------------------------"
fi
export EXIT_CODE=${RES}
fi


check_integration kerberos "kinit -Vkt '${KRB5_KTNAME:=}' airflow" 30
check_integration mongo "nc -zvv mongo 27017" 20
check_integration redis "nc -zvv redis 6379" 20
check_integration rabbitmq "nc -zvv rabbitmq 5672" 20
check_integration cassandra "nc -zvv cassandra 9042" 20
check_integration openldap "nc -zvv openldap 389" 20

if [[ ${EXIT_CODE} != 0 ]]; then
echo
echo "CI environment is not sane!"
echo
exit ${EXIT_CODE}
fi

echo
echo "CI environment is sane!"
echo

in_container_script_end
2 changes: 2 additions & 0 deletions scripts/ci/in_container/entrypoint_ci.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,8 @@ set -u

KUBERNETES_VERSION=${KUBERNETES_VERSION:=""}

"${MY_DIR}/check_environment.sh"

if [[ "${TRAVIS}" == "true" ]]; then
CI_ARGS=(
"--verbosity=0"
Expand Down

0 comments on commit c3722dc

Please sign in to comment.