From 1efc9e20a5e8b4878e0b1ee271acbc32d5c65614 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 10 Dec 2019 14:49:00 +0100 Subject: [PATCH] [AIRFLOW-6216] Allow pytests to be run without "tests" (#6770) With this change you should be able to simply run `pytest` to run all the tests in the main airflow directory. This consist of two changes: * moving pytest.ini to the main airflow directory * skipping collecting kubernetes tests when ENV != kubernetes (cherry picked from commit 239d51ed31f9607e192d1e1c5a997dd03304b870) --- .dockerignore | 1 + Dockerfile | 2 +- tests/pytest.ini => pytest.ini | 2 ++ scripts/ci/_utils.sh | 11 ++--------- scripts/ci/docker-compose-local.yml | 1 + .../kubernetes/test_kubernetes_executor.py | 2 +- .../kubernetes/test_kubernetes_pod_operator.py | 2 +- 7 files changed, 9 insertions(+), 12 deletions(-) rename tests/pytest.ini => pytest.ini (98%) diff --git a/.dockerignore b/.dockerignore index 8b05a534e55d85..69365b57d40ec6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -39,6 +39,7 @@ !.coveragerc !.rat-excludes !.flake8 +!pytest.ini !LICENSE !MANIFEST.in !NOTICE diff --git a/Dockerfile b/Dockerfile index c7c5335ee5f2a7..065a970e4ad3b8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -340,7 +340,7 @@ COPY docs/ ${AIRFLOW_SOURCES}/docs/ COPY tests/ ${AIRFLOW_SOURCES}/tests/ COPY airflow/ ${AIRFLOW_SOURCES}/airflow/ COPY .coveragerc .rat-excludes .flake8 LICENSE MANIFEST.in NOTICE CHANGELOG.txt \ - .github \ + .github pytest.ini \ setup.cfg setup.py \ ${AIRFLOW_SOURCES}/ diff --git a/tests/pytest.ini b/pytest.ini similarity index 98% rename from tests/pytest.ini rename to pytest.ini index b0fc84c6ebff37..c61637842691b5 100644 --- a/tests/pytest.ini +++ b/pytest.ini @@ -26,6 +26,8 @@ addopts = --ignore=tests/dags/test_impersonation_subdag.py --ignore=tests/dags/test_subdag.py norecursedirs = + .eggs + airflow tests/dags tests/dags_with_system_exit tests/test_utils diff --git a/scripts/ci/_utils.sh b/scripts/ci/_utils.sh index 512e9c2bd459c7..1a59f6bdb5951b 100644 --- a/scripts/ci/_utils.sh +++ b/scripts/ci/_utils.sh @@ -81,7 +81,7 @@ function print_info() { declare -a AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS if [[ ${AIRFLOW_MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS} == "true" ]]; then print_info - print_info "Mount whole sourcce directory for static checks" + print_info "Mount whole airflow source directory for static checks (make sure all files are in container)" print_info AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS=( \ "-v" "${AIRFLOW_SOURCES}:/opt/airflow" \ @@ -104,6 +104,7 @@ elif [[ ${AIRFLOW_MOUNT_HOST_VOLUMES_FOR_STATIC_CHECKS} == "true" ]]; then "-v" "${AIRFLOW_SOURCES}/tmp:/opt/airflow/tmp:cached" \ "-v" "${AIRFLOW_SOURCES}/tests:/opt/airflow/tests:cached" \ "-v" "${AIRFLOW_SOURCES}/.flake8:/opt/airflow/.flake8:cached" \ + "-v" "${AIRFLOW_SOURCES}/pytest.ini:/opt/airflow/pytest.ini:cached" \ "-v" "${AIRFLOW_SOURCES}/setup.cfg:/opt/airflow/setup.cfg:cached" \ "-v" "${AIRFLOW_SOURCES}/setup.py:/opt/airflow/setup.py:cached" \ "-v" "${AIRFLOW_SOURCES}/.rat-excludes:/opt/airflow/.rat-excludes:cached" \ @@ -802,14 +803,6 @@ function build_image_on_ci() { fi elif [[ ${TRAVIS_JOB_NAME} == "Static"* ]]; then rebuild_ci_image_if_needed - elif [[ ${TRAVIS_JOB_NAME} == "Pylint"* ]]; then - rebuild_ci_slim_image_if_needed - match_files_regexp '.*\.py' - if [[ ${FILE_MATCHES} == "true" || ${TRAVIS_PULL_REQUEST:=} == "false" ]]; then - rebuild_ci_image_if_needed - else - touch "${BUILD_CACHE_DIR}"/.skip_tests - fi elif [[ ${TRAVIS_JOB_NAME} == "Build documentation"* ]]; then rebuild_ci_image_if_needed else diff --git a/scripts/ci/docker-compose-local.yml b/scripts/ci/docker-compose-local.yml index 7b782d36b7eb02..c718de36edaffd 100644 --- a/scripts/ci/docker-compose-local.yml +++ b/scripts/ci/docker-compose-local.yml @@ -49,6 +49,7 @@ services: - ../../.bash_aliases:/root/.bash_aliases:cached - ../../.inputrc:/root/.inputrc:cached - ../../.flake8:/opt/airflow/.flake8:cached + - ../../pytest.ini:/opt/airflow/pytest.ini:cached - ../../.rat-excludes:/opt/airflow/.rat-excludes:cached - ../../logs:/root/airflow/logs:cached - ../../tmp:/opt/airflow/tmp:cached diff --git a/tests/integration/kubernetes/test_kubernetes_executor.py b/tests/integration/kubernetes/test_kubernetes_executor.py index c335f16f0de110..14fb2958f348ec 100644 --- a/tests/integration/kubernetes/test_kubernetes_executor.py +++ b/tests/integration/kubernetes/test_kubernetes_executor.py @@ -30,7 +30,7 @@ try: check_call(["/usr/local/bin/kubectl", "get", "pods"]) except Exception as e: # pylint: disable=broad-except - if os.environ.get('KUBERNETES_VERSION'): + if os.environ.get('KUBERNETES_VERSION') and os.environ.get('ENV', 'kubernetes') == 'kubernetes': raise e else: raise unittest.SkipTest( diff --git a/tests/integration/kubernetes/test_kubernetes_pod_operator.py b/tests/integration/kubernetes/test_kubernetes_pod_operator.py index ae51217dc30a7b..49630719647fe0 100644 --- a/tests/integration/kubernetes/test_kubernetes_pod_operator.py +++ b/tests/integration/kubernetes/test_kubernetes_pod_operator.py @@ -36,7 +36,7 @@ try: check_call(["/usr/local/bin/kubectl", "get", "pods"]) except Exception as e: # pylint: disable=broad-except - if os.environ.get('KUBERNETES_VERSION'): + if os.environ.get('KUBERNETES_VERSION') and os.environ.get('ENV', 'kubernetes') == 'kubernetes': raise e else: raise unittest.SkipTest(