From 4ba60f4dc58422043f47dfd16172d52da35d5974 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 13 Mar 2020 08:10:44 +0100 Subject: [PATCH] [AIRFLOW-7054] Breeze has an option now to reset db at entry --- BREEZE.rst | 7 +++++++ breeze | 15 +++++++++++++++ breeze-complete | 4 ++-- scripts/ci/docker-compose/base.yml | 1 + scripts/ci/in_container/check_environment.sh | 13 +++++++++++++ scripts/ci/in_container/entrypoint_ci.sh | 3 +++ tests/test_utils/system_tests_class.py | 2 +- 7 files changed, 42 insertions(+), 3 deletions(-) diff --git a/BREEZE.rst b/BREEZE.rst index 833633b2b0d717..8a8648f945d698 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -848,6 +848,13 @@ This is the current syntax for `./breeze <./breeze>`_: Default: sqlite + -d, --db-reset + Resets the database at entry to the envvironment. It will drop all the tables + and data and recreate the DB from scratch even if 'restart' command was not used. + Combined with 'restart' command it enters the environment in the state that is + ready to start airflow webserver/scheduler/worker. Without the switch, the database + does not have any tables and you need to run reset db manually. + -i, --integration Integration to start during tests - it determines which integrations are started for integration tests. There can be more than one integration started, or all to diff --git a/breeze b/breeze index 0675e06093139b..03596157f1d371 100755 --- a/breeze +++ b/breeze @@ -120,6 +120,9 @@ function setup_default_breeze_variables() { # Forward credentials to docker export FORWARD_CREDENTIALS="false" + # Reset DB at entry + export DB_RESET="false" + # If install released airflow is set to specified version, then the source version of airflow # is removed and the specified version of airflow is installed from pypi export INSTALL_AIRFLOW_VERSION=${INSTALL_AIRFLOW_VERSION:="current"} @@ -552,6 +555,11 @@ function parse_arguments() { echo "Installs version of Airflow: ${INSTALL_AIRFLOW_VERSION}" echo shift 2 ;; + -d|--db-reset) + echo "Resetting the DB!" + echo + export DB_RESET="true" + shift 1 ;; -v|--verbose) export VERBOSE="true" echo "Verbose output" @@ -1079,6 +1087,13 @@ ${ALLOWED_BACKENDS} Default: ${_BREEZE_DEFAULT_BACKEND:=} +-d, --db-reset + Resets the database at entry to the envvironment. It will drop all the tables + and data and recreate the DB from scratch even if 'restart' command was not used. + Combined with 'restart' command it enters the environment in the state that is + ready to start airflow webserver/scheduler/worker. Without the switch, the database + does not have any tables and you need to run reset db manually. + -i, --integration Integration to start during tests - it determines which integrations are started for integration tests. There can be more than one integration started, or all to diff --git a/breeze-complete b/breeze-complete index 813dc35f8d125b..9c931e4e829290 100644 --- a/breeze-complete +++ b/breeze-complete @@ -69,7 +69,7 @@ _BREEZE_SHORT_OPTIONS=" h p: b: i: K: V: s x r -l a: +l a: d: v y n q f F P C L D: R: -u @@ -79,7 +79,7 @@ _BREEZE_LONG_OPTIONS=" help python: backend: integration: kubernetes-mode: kubernetes-version: kind-cluster-start kind-cluster-stop kind-cluster-recreate -skip-mounting-local-sources install-airflow-version: +skip-mounting-local-sources install-airflow-version: db-reset verbose assume-yes assume-no assume-quit forward-credentials force-build-images force-pull-images force-clean-images use-local-cache dockerhub-user: dockerhub-repo: push-images diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml index 30f0ce4aac8809..7ec5864532cd16 100644 --- a/scripts/ci/docker-compose/base.yml +++ b/scripts/ci/docker-compose/base.yml @@ -56,6 +56,7 @@ services: - RUN_INTEGRATION_TESTS - BREEZE - INSTALL_AIRFLOW_VERSION + - DB_RESET - ENABLED_SYSTEMS - RUN_SYSTEM_TESTS volumes: diff --git a/scripts/ci/in_container/check_environment.sh b/scripts/ci/in_container/check_environment.sh index 07bcb18dfe7a73..9debed5d34ad7e 100755 --- a/scripts/ci/in_container/check_environment.sh +++ b/scripts/ci/in_container/check_environment.sh @@ -167,6 +167,16 @@ function check_mysql_logs { done } +function resetdb() { + if [[ ${DB_RESET:="false"} == "true" ]]; then + if [[ ${RUN_AIRFLOW_1_10} == "true" ]]; then + airflow resetdb -y + else + airflow db reset -y + fi + fi +} + if [[ -n ${BACKEND:=} ]]; then echo "===============================================================================================" echo " Checking backend: ${BACKEND}" @@ -200,6 +210,8 @@ check_integration rabbitmq "nc -zvv rabbitmq 5672" 20 check_integration cassandra "nc -zvv cassandra 9042" 20 check_integration openldap "nc -zvv openldap 389" 20 +resetdb + if [[ ${EXIT_CODE} != 0 ]]; then echo echo "Error: some of the CI environment failed to initialize!" @@ -207,6 +219,7 @@ if [[ ${EXIT_CODE} != 0 ]]; then exit ${EXIT_CODE} fi + if [[ ${DISABLED_INTEGRATIONS} != "" ]]; then echo echo "Disabled integrations:${DISABLED_INTEGRATIONS}" diff --git a/scripts/ci/in_container/entrypoint_ci.sh b/scripts/ci/in_container/entrypoint_ci.sh index 58c257b75a447f..1958d0ac2a9703 100755 --- a/scripts/ci/in_container/entrypoint_ci.sh +++ b/scripts/ci/in_container/entrypoint_ci.sh @@ -89,6 +89,9 @@ else install_released_airflow_version "${INSTALL_AIRFLOW_VERSION}" fi + +export RUN_AIRFLOW_1_10=${RUN_AIRFLOW_1_10:="false"} + export HADOOP_DISTRO="${HADOOP_DISTRO:="cdh"}" export HADOOP_HOME="${HADOOP_HOME:="/opt/hadoop-cdh"}" diff --git a/tests/test_utils/system_tests_class.py b/tests/test_utils/system_tests_class.py index 691efc0c2f3818..1cb8a2505a3299 100644 --- a/tests/test_utils/system_tests_class.py +++ b/tests/test_utils/system_tests_class.py @@ -121,7 +121,7 @@ def run_dag(self, dag_id: str, dag_folder: str = DEFAULT_DAG_FOLDER) -> None: :param dag_folder: directory where to look for the specific DAG. Relative to AIRFLOW_HOME. :type dag_folder: str """ - if os.environ.get("RUN_AIRFLOW_1_10"): + if os.environ.get("RUN_AIRFLOW_1_10") == "true": # For system tests purpose we are changing airflow/providers # to side packages path of the installed providers package python = f"python{sys.version_info.major}.{sys.version_info.minor}"