From d48ea31b14a2b4d4bc4ba03b52e2ad32611e4a2a Mon Sep 17 00:00:00 2001 From: Kevin Yang Date: Tue, 5 Nov 2019 14:29:43 +0100 Subject: [PATCH] [AIRFLOW-5362] Reorder imports (#5944) (cherry picked from commit d719e1fd6705a93a0dfefef4b46478ade5e006ea) --- .pre-commit-config.yaml | 23 ++++++++++- BREEZE.rst | 2 +- airflow/api/__init__.py | 5 +-- airflow/api/client/local_client.py | 4 +- airflow/api/common/experimental/__init__.py | 2 +- .../api/common/experimental/get_dag_runs.py | 2 +- .../api/common/experimental/trigger_dag.py | 6 +-- airflow/bin/airflow | 2 +- .../airflow_local_settings.py | 2 +- .../default_webserver_config.py | 5 ++- .../auth/backends/github_enterprise_auth.py | 7 +--- airflow/contrib/auth/backends/google_auth.py | 7 +--- airflow/contrib/auth/backends/ldap_auth.py | 22 ++++------- .../contrib/auth/backends/password_auth.py | 17 +++----- .../example_azure_cosmosdb_sensor.py | 2 +- .../example_databricks_operator.py | 2 - .../example_dags/example_dingding_operator.py | 1 - .../example_emr_job_flow_automatic_steps.py | 4 +- .../example_emr_job_flow_manual_steps.py | 9 ++--- .../example_kubernetes_executor_config.py | 1 - .../example_kubernetes_operator.py | 2 +- .../example_papermill_operator.py | 2 - .../example_dags/example_qubole_operator.py | 2 +- .../example_dags/example_twitter_dag.py | 1 - .../example_dags/example_winrm_operator.py | 1 - airflow/contrib/hooks/aws_athena_hook.py | 1 + airflow/contrib/hooks/aws_dynamodb_hook.py | 2 +- airflow/contrib/hooks/aws_hook.py | 3 +- .../hooks/azure_container_instance_hook.py | 7 ++-- .../hooks/azure_container_volume_hook.py | 4 +- airflow/contrib/hooks/azure_fileshare_hook.py | 3 +- airflow/contrib/hooks/cassandra_hook.py | 7 ++-- airflow/contrib/hooks/datadog_hook.py | 7 ++-- airflow/contrib/hooks/discord_webhook_hook.py | 2 +- airflow/contrib/hooks/emr_hook.py | 2 +- airflow/contrib/hooks/ftp_hook.py | 4 +- airflow/contrib/hooks/grpc_hook.py | 7 ++-- airflow/contrib/hooks/imap_hook.py | 2 +- airflow/contrib/hooks/jenkins_hook.py | 5 ++- airflow/contrib/hooks/openfaas_hook.py | 3 +- airflow/contrib/hooks/opsgenie_alert_hook.py | 2 +- airflow/contrib/hooks/redis_hook.py | 1 + airflow/contrib/hooks/sagemaker_hook.py | 6 +-- airflow/contrib/hooks/segment_hook.py | 3 +- airflow/contrib/hooks/sftp_hook.py | 4 +- airflow/contrib/hooks/slack_webhook_hook.py | 2 +- airflow/contrib/hooks/spark_jdbc_hook.py | 1 + airflow/contrib/hooks/spark_jdbc_script.py | 1 + airflow/contrib/hooks/spark_sql_hook.py | 2 +- airflow/contrib/hooks/sqoop_hook.py | 2 +- .../contrib/operators/aws_athena_operator.py | 2 +- .../operators/aws_sqs_publish_operator.py | 2 +- .../azure_container_instances_operator.py | 19 ++++----- .../contrib/operators/databricks_operator.py | 6 +-- airflow/contrib/operators/jira_operator.py | 3 +- airflow/contrib/operators/mongo_to_s3.py | 3 +- .../oracle_to_azure_data_lake_transfer.py | 9 +++-- .../operators/qubole_check_operator.py | 6 +-- airflow/contrib/operators/qubole_operator.py | 6 ++- .../operators/redis_publish_operator.py | 2 +- .../operators/sagemaker_base_operator.py | 1 - .../sagemaker_endpoint_config_operator.py | 2 +- .../operators/sagemaker_endpoint_operator.py | 2 +- .../operators/sagemaker_model_operator.py | 2 +- .../operators/sagemaker_training_operator.py | 2 +- .../operators/sagemaker_transform_operator.py | 2 +- .../operators/sagemaker_tuning_operator.py | 2 +- .../operators/slack_webhook_operator.py | 2 +- .../contrib/operators/spark_jdbc_operator.py | 2 +- .../contrib/operators/spark_sql_operator.py | 2 +- airflow/contrib/operators/vertica_to_hive.py | 5 ++- airflow/contrib/operators/vertica_to_mysql.py | 9 ++--- airflow/contrib/operators/winrm_operator.py | 2 +- airflow/contrib/sensors/aws_athena_sensor.py | 4 +- airflow/contrib/sensors/aws_sqs_sensor.py | 4 +- airflow/contrib/sensors/bash_sensor.py | 9 +++-- airflow/contrib/sensors/jira_sensor.py | 3 +- .../contrib/sensors/redis_pub_sub_sensor.py | 2 +- .../contrib/sensors/sagemaker_base_sensor.py | 2 +- .../sensors/sagemaker_training_sensor.py | 2 +- airflow/contrib/sensors/sftp_sensor.py | 1 + .../contrib/task_runner/cgroup_task_runner.py | 2 +- airflow/executors/__init__.py | 5 ++- airflow/executors/celery_executor.py | 5 +-- airflow/executors/dask_executor.py | 3 +- airflow/hooks/base_hook.py | 7 +--- airflow/hooks/hive_hooks.py | 5 +-- airflow/hooks/http_hook.py | 2 +- airflow/hooks/postgres_hook.py | 3 +- airflow/hooks/samba_hook.py | 3 +- airflow/hooks/slack_hook.py | 3 +- airflow/hooks/zendesk_hook.py | 4 +- airflow/jobs/__init__.py | 4 +- airflow/lineage/__init__.py | 3 +- airflow/lineage/backend/atlas/__init__.py | 6 +-- airflow/migrations/env.py | 7 ++-- ...1210f153_increase_queue_name_size_limit.py | 2 +- .../0a2a5b66e19d_add_task_reschedule_table.py | 2 +- .../0e2a74e0fc9f_add_time_zone_awareness.py | 2 +- .../1507a7289a2f_create_is_encrypted.py | 2 +- ...e3_add_is_encrypted_column_to_variable_.py | 2 +- .../versions/1b38cef5b76e_add_dagrun.py | 2 +- ...24_add_executor_config_to_task_instance.py | 4 +- .../versions/2e541a1dcfed_task_duration.py | 2 +- ...0f54d61_more_logging_into_task_isntance.py | 2 +- ...4_add_kubernetes_resource_checkpointing.py | 2 +- .../versions/40e67319e3a9_dagrun_config.py | 2 +- .../41f5f12752f8_add_superuser_field.py | 3 +- .../versions/4446e08588_dagrun_start_end.py | 2 +- ..._add_fractional_seconds_to_mysql_tables.py | 3 +- .../502898887f84_adding_extra_to_log.py | 2 +- ...61833c1c74b_add_password_column_to_user.py | 2 +- ...d17757c7a_add_pid_field_to_taskinstance.py | 2 +- ...de9cddf6c9_add_task_fails_journal_table.py | 2 +- ...4a4_make_taskinstance_pool_not_nullable.py | 2 +- ...hange_datetime_to_datetime2_6_on_mssql_.py | 1 + .../856955da8476_fix_sqlite_foreign_key.py | 2 +- ...5c0_add_kubernetes_scheduler_uniqueness.py | 2 +- .../a56c9515abdc_remove_dag_stat_table.py | 2 +- ...6_add_a_column_to_track_the_encryption_.py | 2 +- ...dd_notification_sent_column_to_sla_miss.py | 2 +- ...6_make_xcom_value_column_a_large_binary.py | 2 +- .../c8ffec048a3b_add_fields_to_dag.py | 2 +- ...7_add_max_tries_column_to_task_instance.py | 8 ++-- ...ae31099d61_increase_text_size_for_mysql.py | 3 +- ...4ecb8fbee3_add_schedule_interval_to_dag.py | 2 +- .../versions/e3a246e0dc1_current_schema.py | 2 +- .../f2ca10b85618_add_dag_stats_table.py | 2 +- airflow/models/base.py | 1 + airflow/models/errors.py | 2 +- airflow/models/kubernetes.py | 2 +- airflow/models/log.py | 4 +- airflow/models/pool.py | 2 +- airflow/models/slamiss.py | 4 +- airflow/models/taskfail.py | 2 +- airflow/models/taskreschedule.py | 2 +- airflow/models/variable.py | 6 +-- airflow/models/xcom.py | 5 +-- airflow/operators/branch_operator.py | 2 +- airflow/operators/papermill_operator.py | 2 +- airflow/operators/presto_check_operator.py | 3 +- airflow/plugins_manager.py | 10 ++--- airflow/sensors/external_task_sensor.py | 2 +- airflow/sensors/s3_prefix_sensor.py | 1 - airflow/sentry.py | 1 - airflow/task/task_runner/__init__.py | 2 +- airflow/task/task_runner/base_task_runner.py | 4 +- airflow/utils/cli_action_loggers.py | 2 +- airflow/utils/compression.py | 6 +-- airflow/utils/configuration.py | 5 +-- airflow/utils/db.py | 10 ++--- airflow/utils/email.py | 14 +++---- airflow/utils/file.py | 6 +-- airflow/utils/log/es_task_handler.py | 7 ++-- airflow/utils/log/file_processor_handler.py | 2 +- airflow/utils/log/json_formatter.py | 2 +- airflow/utils/log/s3_task_handler.py | 2 +- airflow/utils/log/wasb_task_handler.py | 4 +- airflow/utils/net.py | 3 +- airflow/utils/sqlalchemy.py | 11 ++---- airflow/utils/timeout.py | 7 +--- airflow/utils/timezone.py | 2 +- airflow/www/gunicorn_config.py | 1 + airflow/www/validators.py | 3 +- airflow/www_rbac/decorators.py | 8 ++-- breeze-complete | 2 +- dags/test_dag.py | 3 +- dev/airflow-license | 3 +- docs/conf.py | 2 +- docs/exts/exampleinclude.py | 3 +- docs/exts/removemarktransform.py | 2 +- scripts/ci/ci_isort.sh | 39 +++++++++++++++++++ scripts/perf/scheduler_ops_metrics.py | 3 +- setup.cfg | 10 +++++ .../common/experimental/test_mark_tasks.py | 13 ++++--- tests/api/common/experimental/test_pool.py | 2 +- tests/cli/test_worker_initialisation.py | 6 ++- .../hooks/test_aws_glue_catalog_hook.py | 1 + tests/contrib/hooks/test_aws_hook.py | 2 +- tests/contrib/hooks/test_aws_lambda_hook.py | 2 +- .../test_azure_container_instance_hook.py | 12 +++--- .../test_azure_container_registry_hook.py | 2 +- .../hooks/test_azure_container_volume_hook.py | 2 +- tests/contrib/hooks/test_azure_cosmos_hook.py | 6 +-- tests/contrib/hooks/test_cassandra_hook.py | 5 ++- tests/contrib/hooks/test_databricks_hook.py | 6 +-- tests/contrib/hooks/test_dingding_hook.py | 3 +- .../hooks/test_discord_webhook_hook.py | 3 +- tests/contrib/hooks/test_emr_hook.py | 1 + .../hooks/test_gcp_natural_language_hook.py | 3 +- tests/contrib/hooks/test_grpc_hook.py | 11 +++--- tests/contrib/hooks/test_mongo_hook.py | 10 +++-- tests/contrib/hooks/test_openfaas_hook.py | 6 ++- .../contrib/hooks/test_opsgenie_alert_hook.py | 7 ++-- tests/contrib/hooks/test_qubole_check_hook.py | 1 + tests/contrib/hooks/test_redis_hook.py | 1 + tests/contrib/hooks/test_redshift_hook.py | 3 +- tests/contrib/hooks/test_sagemaker_hook.py | 11 +++--- tests/contrib/hooks/test_spark_jdbc_hook.py | 3 +- .../operators/test_aws_athena_operator.py | 2 +- .../operators/test_awsbatch_operator.py | 2 +- ...test_azure_container_instances_operator.py | 10 ++--- ...st_azure_cosmos_insertdocument_operator.py | 1 - .../operators/test_databricks_operator.py | 7 ++-- .../test_discord_webhook_operator.py | 1 - tests/contrib/operators/test_ecs_operator.py | 2 +- .../operators/test_emr_add_steps_operator.py | 3 +- .../test_emr_create_job_flow_operator.py | 3 +- .../test_gcp_translate_speech_operator.py | 6 ++- .../operators/test_jenkins_operator.py | 4 +- .../operators/test_opsgenie_alert_operator.py | 1 - .../operators/test_qubole_check_operator.py | 10 +++-- .../contrib/operators/test_qubole_operator.py | 9 ++--- .../operators/test_redis_publish_operator.py | 6 ++- .../operators/test_s3_to_gcs_operator.py | 3 +- .../operators/test_s3_to_sftp_operator.py | 8 ++-- ...test_sagemaker_endpoint_config_operator.py | 3 +- .../test_sagemaker_endpoint_operator.py | 3 +- .../test_sagemaker_model_operator.py | 3 +- .../test_sagemaker_training_operator.py | 3 +- .../test_sagemaker_transform_operator.py | 3 +- .../test_sagemaker_tuning_operator.py | 3 +- tests/contrib/operators/test_sftp_operator.py | 3 +- .../operators/test_sftp_to_s3_operator.py | 4 +- .../operators/test_snowflake_operator.py | 4 +- .../operators/test_spark_jdbc_operator.py | 1 - .../operators/test_spark_submit_operator.py | 7 +--- tests/contrib/sensors/test_athena_sensor.py | 3 +- tests/contrib/sensors/test_bash_sensor.py | 3 +- tests/contrib/sensors/test_hdfs_sensor.py | 3 +- tests/contrib/sensors/test_mongo_sensor.py | 1 - tests/contrib/sensors/test_python_sensor.py | 1 - tests/contrib/sensors/test_redis_sensor.py | 1 + .../sensors/test_sagemaker_endpoint_sensor.py | 3 +- .../sensors/test_sagemaker_training_sensor.py | 5 +-- .../test_sagemaker_transform_sensor.py | 3 +- .../sensors/test_sagemaker_tuning_sensor.py | 3 +- tests/contrib/sensors/test_wasb_sensor.py | 6 +-- tests/contrib/sensors/test_weekday_sensor.py | 1 + .../contrib/utils/logging_command_executor.py | 2 +- tests/contrib/utils/test_weekday.py | 1 - tests/dags/test_scheduler_dags.py | 1 + tests/dags/test_task_view_type_check.py | 1 - tests/dags_with_system_exit/a_system_exit.py | 2 +- tests/dags_with_system_exit/c_system_exit.py | 2 +- tests/executors/test_executor.py | 2 +- tests/hooks/test_hive_hook.py | 2 +- tests/hooks/test_oracle_hook.py | 9 +++-- tests/hooks/test_pig_hook.py | 1 + tests/hooks/test_slack_hook.py | 1 + tests/jobs/test_backfill_job.py | 10 ++--- tests/jobs/test_scheduler_job.py | 17 +++----- tests/kubernetes/test_pod_launcher.py | 2 +- tests/lineage/backend/test_atlas.py | 6 +-- tests/macros/test_hive.py | 2 +- tests/models/test_cleartasks.py | 2 +- tests/models/test_dag.py | 4 +- tests/models/test_dagrun.py | 5 +-- tests/models/test_variable.py | 2 +- tests/operators/test_bash_operator.py | 2 +- tests/operators/test_branch_operator.py | 6 +-- tests/operators/test_check_operator.py | 2 +- tests/operators/test_docker_operator.py | 6 +-- tests/operators/test_druid_check_operator.py | 4 +- tests/operators/test_hive_to_druid.py | 3 +- tests/operators/test_http_operator.py | 3 +- tests/operators/test_latest_only_operator.py | 2 +- tests/operators/test_s3_to_hive_operator.py | 22 +++++------ tests/operators/test_slack_operator.py | 3 +- tests/operators/test_virtualenv_operator.py | 8 ++-- tests/security/test_kerberos.py | 2 +- tests/sensors/test_external_task_sensor.py | 7 ++-- tests/sensors/test_hdfs_sensor.py | 1 - .../test_named_hive_partition_sensor.py | 2 +- tests/sensors/test_timedelta_sensor.py | 3 +- tests/sensors/test_timeout_sensor.py | 3 +- tests/test_local_settings.py | 3 +- tests/test_utils/config.py | 2 +- tests/test_utils/db.py | 5 +-- .../deps/test_not_in_retry_period_dep.py | 3 +- .../deps/test_pool_slots_available_dep.py | 3 +- tests/ti_deps/deps/test_trigger_rule_dep.py | 2 +- tests/utils/log/elasticmock/__init__.py | 1 + .../utils/log/test_file_processor_handler.py | 7 ++-- tests/utils/test_dates.py | 6 +-- tests/utils/test_db.py | 6 +-- tests/utils/test_decorators.py | 2 +- tests/utils/test_dot_renderer.py | 1 - tests/utils/test_email.py | 1 + tests/utils/test_helpers.py | 4 +- tests/utils/test_json.py | 2 +- tests/utils/test_log_handlers.py | 9 +++-- tests/utils/test_operator_helpers.py | 3 +- tests/utils/test_tests.py | 1 + tests/utils/test_timezone.py | 3 +- tests/utils/test_trigger_rule.py | 1 + tests/utils/test_weight_rule.py | 1 + 297 files changed, 597 insertions(+), 583 deletions(-) create mode 100755 scripts/ci/ci_isort.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c06c3b1df41c63..92a3ffb2a69b69 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -75,7 +75,7 @@ repos: - id: insert-license name: Add license for all JS files files: \.js$ - exclude: ^\.github/.*$|^airflow/_vendor/.*$|^airflow/www/static/.*$ + exclude: ^\.github/.*$|^airflow/_vendor/.*$|^airflow/www/static/.*|^airflow/www_rbac/static/.*$ args: - --comment-style - "/**| *| */" @@ -190,6 +190,14 @@ repos: entry: "./scripts/ci/pre_commit_lint_dockerfile.sh" files: ^Dockerfile.*$ pass_filenames: true + - id: isort + name: Run isort to sort imports + language: python + entry: isort + files: \.py$ + # To keep consistent with the global isort skip config defined in setup.cfg + exclude: ^airflow/_vendor/.*$|^build/.*$|^.tox/.*$|^venv/.*$ + additional_dependencies: ['isort'] - id: update-breeze-file name: Update output of breeze command in BREEZE.rst entry: "./scripts/ci/pre_commit_breeze_cmd_line.sh" @@ -230,3 +238,16 @@ repos: files: \.py$ pass_filenames: true require_serial: true + - id: mypy + name: Run mypy + language: system + entry: "./scripts/ci/pre_commit_mypy.sh" + files: \.py$ + exclude: ^airflow/_vendor/.*$ + pass_filenames: true + - id: flake8 + name: Run flake8 + language: system + entry: "./scripts/ci/pre_commit_flake8.sh" + files: \.py$ + pass_filenames: true diff --git a/BREEZE.rst b/BREEZE.rst index 2cc0e01f433c80..5efea2759b063f 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -959,7 +959,7 @@ This is the current syntax for `./breeze <./breeze>`_: -K, --kubernetes-version Kubernetes version - only used in case of 'kubernetes' environment. - One of [ v1.13.0 ]. Default: v1.13.0 + One of [ v1.13.0 v1.15.3 ]. Default: v1.13.0 -M, --kubernetes-mode Kubernetes mode - only used in case of 'kubernetes' environment. diff --git a/airflow/api/__init__.py b/airflow/api/__init__.py index 2feb7f4ab3c8b6..ec6b80e8f9a662 100644 --- a/airflow/api/__init__.py +++ b/airflow/api/__init__.py @@ -20,15 +20,14 @@ from __future__ import print_function -from importlib import import_module import warnings +from importlib import import_module import lazy_object_proxy from zope.deprecation import deprecated -from airflow.exceptions import AirflowException, AirflowConfigException from airflow.configuration import conf - +from airflow.exceptions import AirflowConfigException, AirflowException from airflow.utils.log.logging_mixin import LoggingMixin diff --git a/airflow/api/client/local_client.py b/airflow/api/client/local_client.py index fc3c1cecea4692..37c9785740bf51 100644 --- a/airflow/api/client/local_client.py +++ b/airflow/api/client/local_client.py @@ -19,9 +19,7 @@ """Local client API""" from airflow.api.client import api_client -from airflow.api.common.experimental import pool -from airflow.api.common.experimental import trigger_dag -from airflow.api.common.experimental import delete_dag +from airflow.api.common.experimental import delete_dag, pool, trigger_dag class Client(api_client.Client): diff --git a/airflow/api/common/experimental/__init__.py b/airflow/api/common/experimental/__init__.py index 4a2384f61b3ba0..a973e99d316ddf 100644 --- a/airflow/api/common/experimental/__init__.py +++ b/airflow/api/common/experimental/__init__.py @@ -20,7 +20,7 @@ from datetime import datetime from typing import Optional -from airflow.exceptions import DagNotFound, TaskNotFound, DagRunNotFound +from airflow.exceptions import DagNotFound, DagRunNotFound, TaskNotFound from airflow.models import DagBag, DagModel, DagRun diff --git a/airflow/api/common/experimental/get_dag_runs.py b/airflow/api/common/experimental/get_dag_runs.py index 635286aedea3e2..3f7aa68f2ef4d9 100644 --- a/airflow/api/common/experimental/get_dag_runs.py +++ b/airflow/api/common/experimental/get_dag_runs.py @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. """DAG runs APIs.""" -from typing import Optional, List, Dict, Any +from typing import Any, Dict, List, Optional from flask import url_for diff --git a/airflow/api/common/experimental/trigger_dag.py b/airflow/api/common/experimental/trigger_dag.py index 2dd86b804406c0..6ad4575b7a0e4a 100644 --- a/airflow/api/common/experimental/trigger_dag.py +++ b/airflow/api/common/experimental/trigger_dag.py @@ -19,10 +19,10 @@ """Triggering DAG runs APIs.""" import json from datetime import datetime -from typing import Union, Optional, List +from typing import List, Optional, Union -from airflow.exceptions import DagRunAlreadyExists, DagNotFound -from airflow.models import DagRun, DagBag, DagModel +from airflow.exceptions import DagNotFound, DagRunAlreadyExists +from airflow.models import DagBag, DagModel, DagRun from airflow.utils import timezone from airflow.utils.state import State diff --git a/airflow/bin/airflow b/airflow/bin/airflow index d98cf00a582e0f..72adf8367c02e1 100755 --- a/airflow/bin/airflow +++ b/airflow/bin/airflow @@ -22,8 +22,8 @@ import os import argcomplete -from airflow.configuration import conf from airflow.bin.cli import CLIFactory +from airflow.configuration import conf if __name__ == '__main__': diff --git a/airflow/config_templates/airflow_local_settings.py b/airflow/config_templates/airflow_local_settings.py index 4555b9822f7a68..531a8087c62aff 100644 --- a/airflow/config_templates/airflow_local_settings.py +++ b/airflow/config_templates/airflow_local_settings.py @@ -18,7 +18,7 @@ # under the License. import os -from typing import Dict, Any +from typing import Any, Dict import six diff --git a/airflow/config_templates/default_webserver_config.py b/airflow/config_templates/default_webserver_config.py index bc8a6bb1fb6131..65f775842ab807 100644 --- a/airflow/config_templates/default_webserver_config.py +++ b/airflow/config_templates/default_webserver_config.py @@ -18,13 +18,16 @@ # under the License. """Default configuration for the Airflow webserver""" import os + from flask_appbuilder.security.manager import AUTH_DB + +from airflow.configuration import conf + # from flask_appbuilder.security.manager import AUTH_LDAP # from flask_appbuilder.security.manager import AUTH_OAUTH # from flask_appbuilder.security.manager import AUTH_OID # from flask_appbuilder.security.manager import AUTH_REMOTE_USER -from airflow.configuration import conf basedir = os.path.abspath(os.path.dirname(__file__)) diff --git a/airflow/contrib/auth/backends/github_enterprise_auth.py b/airflow/contrib/auth/backends/github_enterprise_auth.py index 28819222e59438..a703441f671202 100644 --- a/airflow/contrib/auth/backends/github_enterprise_auth.py +++ b/airflow/contrib/auth/backends/github_enterprise_auth.py @@ -17,13 +17,10 @@ # specific language governing permissions and limitations # under the License. import flask_login - +from flask import redirect, request, url_for # Need to expose these downstream # flake8: noqa: F401 -from flask_login import current_user, logout_user, login_required, login_user - -from flask import url_for, redirect, request - +from flask_login import current_user, login_required, login_user, logout_user from flask_oauthlib.client import OAuth from airflow import models diff --git a/airflow/contrib/auth/backends/google_auth.py b/airflow/contrib/auth/backends/google_auth.py index 2ce23ae56cd567..efcc2f0aa1c0c5 100644 --- a/airflow/contrib/auth/backends/google_auth.py +++ b/airflow/contrib/auth/backends/google_auth.py @@ -17,13 +17,10 @@ # specific language governing permissions and limitations # under the License. import flask_login - +from flask import redirect, request, url_for # Need to expose these downstream # flake8: noqa: F401 -from flask_login import current_user, logout_user, login_required, login_user - -from flask import url_for, redirect, request - +from flask_login import current_user, login_required, login_user, logout_user from flask_oauthlib.client import OAuth from airflow import models diff --git a/airflow/contrib/auth/backends/ldap_auth.py b/airflow/contrib/auth/backends/ldap_auth.py index 7368deae88dd8c..3d71e2e2683f04 100644 --- a/airflow/contrib/auth/backends/ldap_auth.py +++ b/airflow/contrib/auth/backends/ldap_auth.py @@ -16,27 +16,21 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from future.utils import native +import re +import ssl +import traceback import flask_login -from flask_login import login_required, current_user, logout_user # noqa: F401 -from flask import flash +from flask import flash, redirect, url_for +from flask_login import current_user, login_required, logout_user # noqa: F401 +from future.utils import native +from ldap3 import LEVEL, SUBTREE, Connection, Server, Tls, set_config_parameter from wtforms import Form, PasswordField, StringField from wtforms.validators import InputRequired -from ldap3 import Server, Connection, Tls, set_config_parameter, LEVEL, SUBTREE -import ssl - -from flask import url_for, redirect - from airflow import models -from airflow.configuration import conf -from airflow.configuration import AirflowConfigException +from airflow.configuration import AirflowConfigException, conf from airflow.utils.db import provide_session - -import traceback -import re - from airflow.utils.log.logging_mixin import LoggingMixin LOGIN_MANAGER = flask_login.LoginManager() diff --git a/airflow/contrib/auth/backends/password_auth.py b/airflow/contrib/auth/backends/password_auth.py index e04a40d455b897..85f50b1f29a9da 100644 --- a/airflow/contrib/auth/backends/password_auth.py +++ b/airflow/contrib/auth/backends/password_auth.py @@ -23,24 +23,19 @@ from functools import wraps from sys import version_info -from flask import flash, Response -from flask import url_for, redirect, make_response -from flask_bcrypt import generate_password_hash, check_password_hash - import flask_login +from flask import Response, flash, make_response, redirect, url_for +from flask_bcrypt import check_password_hash, generate_password_hash # noinspection PyUnresolvedReferences # pylint: disable=unused-import -from flask_login import login_required, current_user, logout_user # noqa: F401 -# pylint: enable=unused-import - -from wtforms import Form, PasswordField, StringField -from wtforms.validators import InputRequired - +from flask_login import current_user, login_required, logout_user # noqa: F401 from sqlalchemy import Column, String from sqlalchemy.ext.hybrid import hybrid_property +from wtforms import Form, PasswordField, StringField +from wtforms.validators import InputRequired from airflow import models -from airflow.utils.db import provide_session, create_session +from airflow.utils.db import create_session, provide_session from airflow.utils.log.logging_mixin import LoggingMixin LOGIN_MANAGER = flask_login.LoginManager() diff --git a/airflow/contrib/example_dags/example_azure_cosmosdb_sensor.py b/airflow/contrib/example_dags/example_azure_cosmosdb_sensor.py index 01f7493f3e4b4c..9e1ad0a6b7d99e 100644 --- a/airflow/contrib/example_dags/example_azure_cosmosdb_sensor.py +++ b/airflow/contrib/example_dags/example_azure_cosmosdb_sensor.py @@ -28,8 +28,8 @@ """ from airflow import DAG -from airflow.contrib.sensors.azure_cosmos_sensor import AzureCosmosDocumentSensor from airflow.contrib.operators.azure_cosmos_operator import AzureCosmosInsertDocumentOperator +from airflow.contrib.sensors.azure_cosmos_sensor import AzureCosmosDocumentSensor from airflow.utils import dates default_args = { diff --git a/airflow/contrib/example_dags/example_databricks_operator.py b/airflow/contrib/example_dags/example_databricks_operator.py index 262edbdc2c3250..93989f81e6cf60 100644 --- a/airflow/contrib/example_dags/example_databricks_operator.py +++ b/airflow/contrib/example_dags/example_databricks_operator.py @@ -33,11 +33,9 @@ """ import airflow - from airflow import DAG from airflow.contrib.operators.databricks_operator import DatabricksSubmitRunOperator - default_args = { 'owner': 'Airflow', 'email': ['airflow@example.com'], diff --git a/airflow/contrib/example_dags/example_dingding_operator.py b/airflow/contrib/example_dags/example_dingding_operator.py index d67ad6af2c7bbf..a9639f884d9333 100644 --- a/airflow/contrib/example_dags/example_dingding_operator.py +++ b/airflow/contrib/example_dags/example_dingding_operator.py @@ -25,7 +25,6 @@ from airflow import DAG from airflow.contrib.operators.dingding_operator import DingdingOperator - args = { 'owner': 'Airflow', 'retries': 3, diff --git a/airflow/contrib/example_dags/example_emr_job_flow_automatic_steps.py b/airflow/contrib/example_dags/example_emr_job_flow_automatic_steps.py index 6e6a4ee7c9fa31..03f19e0f770767 100644 --- a/airflow/contrib/example_dags/example_emr_job_flow_automatic_steps.py +++ b/airflow/contrib/example_dags/example_emr_job_flow_automatic_steps.py @@ -20,10 +20,10 @@ This is an example dag for a AWS EMR Pipeline with auto steps. """ from datetime import timedelta + import airflow from airflow import DAG -from airflow.contrib.operators.emr_create_job_flow_operator \ - import EmrCreateJobFlowOperator +from airflow.contrib.operators.emr_create_job_flow_operator import EmrCreateJobFlowOperator from airflow.contrib.sensors.emr_job_flow_sensor import EmrJobFlowSensor DEFAULT_ARGS = { diff --git a/airflow/contrib/example_dags/example_emr_job_flow_manual_steps.py b/airflow/contrib/example_dags/example_emr_job_flow_manual_steps.py index 6eaa4e5ec62adc..b03dd92369c6f3 100644 --- a/airflow/contrib/example_dags/example_emr_job_flow_manual_steps.py +++ b/airflow/contrib/example_dags/example_emr_job_flow_manual_steps.py @@ -26,13 +26,10 @@ import airflow from airflow import DAG -from airflow.contrib.operators.emr_create_job_flow_operator \ - import EmrCreateJobFlowOperator -from airflow.contrib.operators.emr_add_steps_operator \ - import EmrAddStepsOperator +from airflow.contrib.operators.emr_add_steps_operator import EmrAddStepsOperator +from airflow.contrib.operators.emr_create_job_flow_operator import EmrCreateJobFlowOperator +from airflow.contrib.operators.emr_terminate_job_flow_operator import EmrTerminateJobFlowOperator from airflow.contrib.sensors.emr_step_sensor import EmrStepSensor -from airflow.contrib.operators.emr_terminate_job_flow_operator \ - import EmrTerminateJobFlowOperator DEFAULT_ARGS = { 'owner': 'Airflow', diff --git a/airflow/contrib/example_dags/example_kubernetes_executor_config.py b/airflow/contrib/example_dags/example_kubernetes_executor_config.py index cb02c42902cf7b..c05c01002ccd3b 100644 --- a/airflow/contrib/example_dags/example_kubernetes_executor_config.py +++ b/airflow/contrib/example_dags/example_kubernetes_executor_config.py @@ -28,7 +28,6 @@ from airflow.models import DAG from airflow.operators.python_operator import PythonOperator - default_args = { 'owner': 'Airflow', 'start_date': airflow.utils.dates.days_ago(2) diff --git a/airflow/contrib/example_dags/example_kubernetes_operator.py b/airflow/contrib/example_dags/example_kubernetes_operator.py index e9453356bcff0f..ce3c40cf8475e7 100644 --- a/airflow/contrib/example_dags/example_kubernetes_operator.py +++ b/airflow/contrib/example_dags/example_kubernetes_operator.py @@ -19,9 +19,9 @@ """ This is an example dag for using the KubernetesPodOperator. """ +from airflow.models import DAG from airflow.utils.dates import days_ago from airflow.utils.log.logging_mixin import LoggingMixin -from airflow.models import DAG log = LoggingMixin().log diff --git a/airflow/contrib/example_dags/example_papermill_operator.py b/airflow/contrib/example_dags/example_papermill_operator.py index 8710538d4a5f18..6840bfd2f04761 100644 --- a/airflow/contrib/example_dags/example_papermill_operator.py +++ b/airflow/contrib/example_dags/example_papermill_operator.py @@ -25,11 +25,9 @@ from datetime import timedelta import airflow - from airflow.models import DAG from airflow.operators.papermill_operator import PapermillOperator - default_args = { 'owner': 'Airflow', 'start_date': airflow.utils.dates.days_ago(2) diff --git a/airflow/contrib/example_dags/example_qubole_operator.py b/airflow/contrib/example_dags/example_qubole_operator.py index 569bcc6206b661..4808ba9fbcedbc 100644 --- a/airflow/contrib/example_dags/example_qubole_operator.py +++ b/airflow/contrib/example_dags/example_qubole_operator.py @@ -35,7 +35,7 @@ from airflow import DAG from airflow.contrib.operators.qubole_operator import QuboleOperator from airflow.operators.dummy_operator import DummyOperator -from airflow.operators.python_operator import PythonOperator, BranchPythonOperator +from airflow.operators.python_operator import BranchPythonOperator, PythonOperator default_args = { 'owner': 'Airflow', diff --git a/airflow/contrib/example_dags/example_twitter_dag.py b/airflow/contrib/example_dags/example_twitter_dag.py index 648e2fbe7d402d..f7050d4c1a884b 100644 --- a/airflow/contrib/example_dags/example_twitter_dag.py +++ b/airflow/contrib/example_dags/example_twitter_dag.py @@ -37,7 +37,6 @@ from airflow.operators.hive_operator import HiveOperator from airflow.operators.python_operator import PythonOperator - # -------------------------------------------------------------------------------- # Create a few placeholder scripts. In practice these would be different python # script files, which are imported in this section with absolute or relative imports diff --git a/airflow/contrib/example_dags/example_winrm_operator.py b/airflow/contrib/example_dags/example_winrm_operator.py index df28d9e2e576bd..e3fd907fa3583a 100644 --- a/airflow/contrib/example_dags/example_winrm_operator.py +++ b/airflow/contrib/example_dags/example_winrm_operator.py @@ -37,7 +37,6 @@ from airflow.models import DAG from airflow.operators.dummy_operator import DummyOperator - default_args = { 'owner': 'Airflow', 'start_date': airflow.utils.dates.days_ago(2) diff --git a/airflow/contrib/hooks/aws_athena_hook.py b/airflow/contrib/hooks/aws_athena_hook.py index b54cbe64cb77f3..3560367309ec3f 100644 --- a/airflow/contrib/hooks/aws_athena_hook.py +++ b/airflow/contrib/hooks/aws_athena_hook.py @@ -21,6 +21,7 @@ This module contains AWS Athena hook """ from time import sleep + from airflow.contrib.hooks.aws_hook import AwsHook diff --git a/airflow/contrib/hooks/aws_dynamodb_hook.py b/airflow/contrib/hooks/aws_dynamodb_hook.py index a1ced968f254a5..b5a5d9414091b3 100644 --- a/airflow/contrib/hooks/aws_dynamodb_hook.py +++ b/airflow/contrib/hooks/aws_dynamodb_hook.py @@ -21,8 +21,8 @@ """ This module contains the AWS DynamoDB hook """ -from airflow.exceptions import AirflowException from airflow.contrib.hooks.aws_hook import AwsHook +from airflow.exceptions import AirflowException class AwsDynamoDBHook(AwsHook): diff --git a/airflow/contrib/hooks/aws_hook.py b/airflow/contrib/hooks/aws_hook.py index 8cfbf2367aa3fa..cd763277af359f 100644 --- a/airflow/contrib/hooks/aws_hook.py +++ b/airflow/contrib/hooks/aws_hook.py @@ -21,8 +21,9 @@ This module contains Base AWS Hook """ -import logging import configparser +import logging + import boto3 from airflow.exceptions import AirflowException diff --git a/airflow/contrib/hooks/azure_container_instance_hook.py b/airflow/contrib/hooks/azure_container_instance_hook.py index 0fa958875a7628..ea281cf33e2ed2 100644 --- a/airflow/contrib/hooks/azure_container_instance_hook.py +++ b/airflow/contrib/hooks/azure_container_instance_hook.py @@ -18,15 +18,14 @@ import os -from airflow.hooks.base_hook import BaseHook -from airflow.exceptions import AirflowException - from azure.common.client_factory import get_client_from_auth_file from azure.common.credentials import ServicePrincipalCredentials - from azure.mgmt.containerinstance import ContainerInstanceManagementClient from zope.deprecation import deprecation +from airflow.exceptions import AirflowException +from airflow.hooks.base_hook import BaseHook + class AzureContainerInstanceHook(BaseHook): """ diff --git a/airflow/contrib/hooks/azure_container_volume_hook.py b/airflow/contrib/hooks/azure_container_volume_hook.py index c81c5190776c60..62cd62da83e49f 100644 --- a/airflow/contrib/hooks/azure_container_volume_hook.py +++ b/airflow/contrib/hooks/azure_container_volume_hook.py @@ -17,9 +17,9 @@ # specific language governing permissions and limitations # under the License. +from azure.mgmt.containerinstance.models import AzureFileVolume, Volume + from airflow.hooks.base_hook import BaseHook -from azure.mgmt.containerinstance.models import (Volume, - AzureFileVolume) class AzureContainerVolumeHook(BaseHook): diff --git a/airflow/contrib/hooks/azure_fileshare_hook.py b/airflow/contrib/hooks/azure_fileshare_hook.py index 8afa1540d78b41..545a8d3735cc74 100644 --- a/airflow/contrib/hooks/azure_fileshare_hook.py +++ b/airflow/contrib/hooks/azure_fileshare_hook.py @@ -18,9 +18,10 @@ # under the License. # -from airflow.hooks.base_hook import BaseHook from azure.storage.file import FileService +from airflow.hooks.base_hook import BaseHook + class AzureFileShareHook(BaseHook): """ diff --git a/airflow/contrib/hooks/cassandra_hook.py b/airflow/contrib/hooks/cassandra_hook.py index 08def3a5b775cc..c159b8741e2e6b 100644 --- a/airflow/contrib/hooks/cassandra_hook.py +++ b/airflow/contrib/hooks/cassandra_hook.py @@ -17,10 +17,11 @@ # specific language governing permissions and limitations # under the License. -from cassandra.cluster import Cluster -from cassandra.policies import (RoundRobinPolicy, DCAwareRoundRobinPolicy, - TokenAwarePolicy, WhiteListRoundRobinPolicy) from cassandra.auth import PlainTextAuthProvider +from cassandra.cluster import Cluster +from cassandra.policies import ( + DCAwareRoundRobinPolicy, RoundRobinPolicy, TokenAwarePolicy, WhiteListRoundRobinPolicy, +) from airflow.hooks.base_hook import BaseHook from airflow.utils.log.logging_mixin import LoggingMixin diff --git a/airflow/contrib/hooks/datadog_hook.py b/airflow/contrib/hooks/datadog_hook.py index 50209dd112e066..21fb415c63a41f 100644 --- a/airflow/contrib/hooks/datadog_hook.py +++ b/airflow/contrib/hooks/datadog_hook.py @@ -18,10 +18,11 @@ # under the License. import time -from airflow.hooks.base_hook import BaseHook -from airflow.exceptions import AirflowException -from datadog import initialize, api +from datadog import api, initialize + +from airflow.exceptions import AirflowException +from airflow.hooks.base_hook import BaseHook from airflow.utils.log.logging_mixin import LoggingMixin diff --git a/airflow/contrib/hooks/discord_webhook_hook.py b/airflow/contrib/hooks/discord_webhook_hook.py index 731d9d575a9d6c..d2dbaea8994dd2 100644 --- a/airflow/contrib/hooks/discord_webhook_hook.py +++ b/airflow/contrib/hooks/discord_webhook_hook.py @@ -20,8 +20,8 @@ import json import re -from airflow.hooks.http_hook import HttpHook from airflow.exceptions import AirflowException +from airflow.hooks.http_hook import HttpHook class DiscordWebhookHook(HttpHook): diff --git a/airflow/contrib/hooks/emr_hook.py b/airflow/contrib/hooks/emr_hook.py index 7571d0988f697f..d261ae89d1e348 100644 --- a/airflow/contrib/hooks/emr_hook.py +++ b/airflow/contrib/hooks/emr_hook.py @@ -17,8 +17,8 @@ # specific language governing permissions and limitations # under the License. -from airflow.exceptions import AirflowException from airflow.contrib.hooks.aws_hook import AwsHook +from airflow.exceptions import AirflowException class EmrHook(AwsHook): diff --git a/airflow/contrib/hooks/ftp_hook.py b/airflow/contrib/hooks/ftp_hook.py index ec0002f7d50275..99b2e204d19150 100644 --- a/airflow/contrib/hooks/ftp_hook.py +++ b/airflow/contrib/hooks/ftp_hook.py @@ -21,9 +21,11 @@ import datetime import ftplib import os.path -from airflow.hooks.base_hook import BaseHook + from past.builtins import basestring +from airflow.hooks.base_hook import BaseHook + def mlsd(conn, path="", facts=None): """ diff --git a/airflow/contrib/hooks/grpc_hook.py b/airflow/contrib/hooks/grpc_hook.py index 7ad4e596b7d442..e45af3ccb379c9 100644 --- a/airflow/contrib/hooks/grpc_hook.py +++ b/airflow/contrib/hooks/grpc_hook.py @@ -22,11 +22,12 @@ import grpc from google import auth as google_auth from google.auth import jwt as google_auth_jwt -from google.auth.transport import grpc as google_auth_transport_grpc -from google.auth.transport import requests as google_auth_transport_requests +from google.auth.transport import ( + grpc as google_auth_transport_grpc, requests as google_auth_transport_requests, +) -from airflow.hooks.base_hook import BaseHook from airflow.exceptions import AirflowConfigException +from airflow.hooks.base_hook import BaseHook class GrpcHook(BaseHook): diff --git a/airflow/contrib/hooks/imap_hook.py b/airflow/contrib/hooks/imap_hook.py index 79f08cd5649cb0..969f27beb30016 100644 --- a/airflow/contrib/hooks/imap_hook.py +++ b/airflow/contrib/hooks/imap_hook.py @@ -22,7 +22,7 @@ import os.path import re -from airflow import LoggingMixin, AirflowException +from airflow import AirflowException, LoggingMixin from airflow.hooks.base_hook import BaseHook diff --git a/airflow/contrib/hooks/jenkins_hook.py b/airflow/contrib/hooks/jenkins_hook.py index 626fd1fe9e3795..a5a6cd5ac0edaf 100644 --- a/airflow/contrib/hooks/jenkins_hook.py +++ b/airflow/contrib/hooks/jenkins_hook.py @@ -18,10 +18,11 @@ # under the License. # -from airflow.hooks.base_hook import BaseHook +from distutils.util import strtobool import jenkins -from distutils.util import strtobool + +from airflow.hooks.base_hook import BaseHook class JenkinsHook(BaseHook): diff --git a/airflow/contrib/hooks/openfaas_hook.py b/airflow/contrib/hooks/openfaas_hook.py index 00e3667a77c656..e03a4751f158bf 100644 --- a/airflow/contrib/hooks/openfaas_hook.py +++ b/airflow/contrib/hooks/openfaas_hook.py @@ -17,9 +17,10 @@ # specific language governing permissions and limitations # under the License. -from airflow.hooks.base_hook import BaseHook import requests + from airflow import AirflowException +from airflow.hooks.base_hook import BaseHook OK_STATUS_CODE = 202 diff --git a/airflow/contrib/hooks/opsgenie_alert_hook.py b/airflow/contrib/hooks/opsgenie_alert_hook.py index d576c08c239820..0b79966d8124b1 100644 --- a/airflow/contrib/hooks/opsgenie_alert_hook.py +++ b/airflow/contrib/hooks/opsgenie_alert_hook.py @@ -22,8 +22,8 @@ import requests -from airflow.hooks.http_hook import HttpHook from airflow import AirflowException +from airflow.hooks.http_hook import HttpHook class OpsgenieAlertHook(HttpHook): diff --git a/airflow/contrib/hooks/redis_hook.py b/airflow/contrib/hooks/redis_hook.py index a63c839e34c852..dbf7343baf196c 100644 --- a/airflow/contrib/hooks/redis_hook.py +++ b/airflow/contrib/hooks/redis_hook.py @@ -21,6 +21,7 @@ RedisHook module """ from redis import Redis + from airflow.hooks.base_hook import BaseHook diff --git a/airflow/contrib/hooks/sagemaker_hook.py b/airflow/contrib/hooks/sagemaker_hook.py index 117dfa1029a0c2..4d835ef19ff6f6 100644 --- a/airflow/contrib/hooks/sagemaker_hook.py +++ b/airflow/contrib/hooks/sagemaker_hook.py @@ -16,18 +16,18 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import collections +import os import tarfile import tempfile import time -import os -import collections import warnings from botocore.exceptions import ClientError -from airflow.exceptions import AirflowException from airflow.contrib.hooks.aws_hook import AwsHook from airflow.contrib.hooks.aws_logs_hook import AwsLogsHook +from airflow.exceptions import AirflowException from airflow.hooks.S3_hook import S3Hook from airflow.utils import timezone diff --git a/airflow/contrib/hooks/segment_hook.py b/airflow/contrib/hooks/segment_hook.py index a072a9f1eaf999..6bd6433c840447 100644 --- a/airflow/contrib/hooks/segment_hook.py +++ b/airflow/contrib/hooks/segment_hook.py @@ -26,8 +26,9 @@ https://github.com/segmentio/analytics-python """ import analytics -from airflow.hooks.base_hook import BaseHook + from airflow.exceptions import AirflowException +from airflow.hooks.base_hook import BaseHook class SegmentHook(BaseHook): diff --git a/airflow/contrib/hooks/sftp_hook.py b/airflow/contrib/hooks/sftp_hook.py index 6fc01babe5ab26..6e2932e5781eeb 100644 --- a/airflow/contrib/hooks/sftp_hook.py +++ b/airflow/contrib/hooks/sftp_hook.py @@ -17,9 +17,11 @@ # specific language governing permissions and limitations # under the License. +import datetime import stat + import pysftp -import datetime + from airflow.contrib.hooks.ssh_hook import SSHHook diff --git a/airflow/contrib/hooks/slack_webhook_hook.py b/airflow/contrib/hooks/slack_webhook_hook.py index 1b3aaa46358c40..55094b9d2881f6 100644 --- a/airflow/contrib/hooks/slack_webhook_hook.py +++ b/airflow/contrib/hooks/slack_webhook_hook.py @@ -19,8 +19,8 @@ # import json -from airflow.hooks.http_hook import HttpHook from airflow.exceptions import AirflowException +from airflow.hooks.http_hook import HttpHook class SlackWebhookHook(HttpHook): diff --git a/airflow/contrib/hooks/spark_jdbc_hook.py b/airflow/contrib/hooks/spark_jdbc_hook.py index c188b1e863de27..138d692d1bbeee 100644 --- a/airflow/contrib/hooks/spark_jdbc_hook.py +++ b/airflow/contrib/hooks/spark_jdbc_hook.py @@ -18,6 +18,7 @@ # under the License. # import os + from airflow.contrib.hooks.spark_submit_hook import SparkSubmitHook from airflow.exceptions import AirflowException diff --git a/airflow/contrib/hooks/spark_jdbc_script.py b/airflow/contrib/hooks/spark_jdbc_script.py index 8e27f363f8945c..8d0014529f9624 100644 --- a/airflow/contrib/hooks/spark_jdbc_script.py +++ b/airflow/contrib/hooks/spark_jdbc_script.py @@ -18,6 +18,7 @@ # under the License. # import argparse + from pyspark.sql import SparkSession diff --git a/airflow/contrib/hooks/spark_sql_hook.py b/airflow/contrib/hooks/spark_sql_hook.py index 25c0bc53c55764..a107568a6ac104 100644 --- a/airflow/contrib/hooks/spark_sql_hook.py +++ b/airflow/contrib/hooks/spark_sql_hook.py @@ -19,8 +19,8 @@ # import subprocess -from airflow.hooks.base_hook import BaseHook from airflow.exceptions import AirflowException +from airflow.hooks.base_hook import BaseHook class SparkSqlHook(BaseHook): diff --git a/airflow/contrib/hooks/sqoop_hook.py b/airflow/contrib/hooks/sqoop_hook.py index aa5ec5222afd0d..e8e2feb3e62e83 100644 --- a/airflow/contrib/hooks/sqoop_hook.py +++ b/airflow/contrib/hooks/sqoop_hook.py @@ -22,10 +22,10 @@ This module contains a sqoop 1.x hook """ import subprocess +from copy import deepcopy from airflow.exceptions import AirflowException from airflow.hooks.base_hook import BaseHook -from copy import deepcopy class SqoopHook(BaseHook): diff --git a/airflow/contrib/operators/aws_athena_operator.py b/airflow/contrib/operators/aws_athena_operator.py index 95f87531f341c2..38b88ca99e4468 100644 --- a/airflow/contrib/operators/aws_athena_operator.py +++ b/airflow/contrib/operators/aws_athena_operator.py @@ -20,9 +20,9 @@ from uuid import uuid4 +from airflow.contrib.hooks.aws_athena_hook import AWSAthenaHook from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults -from airflow.contrib.hooks.aws_athena_hook import AWSAthenaHook class AWSAthenaOperator(BaseOperator): diff --git a/airflow/contrib/operators/aws_sqs_publish_operator.py b/airflow/contrib/operators/aws_sqs_publish_operator.py index 4e9476fd2aed2f..bf38ab7a6ba215 100644 --- a/airflow/contrib/operators/aws_sqs_publish_operator.py +++ b/airflow/contrib/operators/aws_sqs_publish_operator.py @@ -18,9 +18,9 @@ # under the License. """Publish message to SQS queue""" +from airflow.contrib.hooks.aws_sqs_hook import SQSHook from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults -from airflow.contrib.hooks.aws_sqs_hook import SQSHook class SQSPublishOperator(BaseOperator): diff --git a/airflow/contrib/operators/azure_container_instances_operator.py b/airflow/contrib/operators/azure_container_instances_operator.py index 309a5d08707c49..e6c2c0bf8fe566 100644 --- a/airflow/contrib/operators/azure_container_instances_operator.py +++ b/airflow/contrib/operators/azure_container_instances_operator.py @@ -17,28 +17,23 @@ # specific language governing permissions and limitations # under the License. +import re from collections import namedtuple -from typing import Sequence, Dict from time import sleep -import re +from typing import Dict, Sequence + +from azure.mgmt.containerinstance.models import ( + Container, ContainerGroup, EnvironmentVariable, ResourceRequests, ResourceRequirements, VolumeMount, +) +from msrestazure.azure_exceptions import CloudError from airflow.contrib.hooks.azure_container_instance_hook import AzureContainerInstanceHook from airflow.contrib.hooks.azure_container_registry_hook import AzureContainerRegistryHook from airflow.contrib.hooks.azure_container_volume_hook import AzureContainerVolumeHook - from airflow.exceptions import AirflowException, AirflowTaskTimeout from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults -from azure.mgmt.containerinstance.models import (EnvironmentVariable, - VolumeMount, - ResourceRequests, - ResourceRequirements, - Container, - ContainerGroup) -from msrestazure.azure_exceptions import CloudError - - Volume = namedtuple( 'Volume', ['conn_id', 'account_name', 'share_name', 'mount_path', 'read_only'], diff --git a/airflow/contrib/operators/databricks_operator.py b/airflow/contrib/operators/databricks_operator.py index 868e32a45e6f96..7dd400800acf62 100644 --- a/airflow/contrib/operators/databricks_operator.py +++ b/airflow/contrib/operators/databricks_operator.py @@ -18,15 +18,15 @@ # under the License. # -import six import time -from airflow.exceptions import AirflowException +import six + from airflow.contrib.hooks.databricks_hook import DatabricksHook +from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults - XCOM_RUN_ID_KEY = 'run_id' XCOM_RUN_PAGE_URL_KEY = 'run_page_url' diff --git a/airflow/contrib/operators/jira_operator.py b/airflow/contrib/operators/jira_operator.py index 01d78b8645ab6c..e2ad7168dc869b 100644 --- a/airflow/contrib/operators/jira_operator.py +++ b/airflow/contrib/operators/jira_operator.py @@ -18,8 +18,7 @@ # under the License. -from airflow.contrib.hooks.jira_hook import JIRAError -from airflow.contrib.hooks.jira_hook import JiraHook +from airflow.contrib.hooks.jira_hook import JIRAError, JiraHook from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults diff --git a/airflow/contrib/operators/mongo_to_s3.py b/airflow/contrib/operators/mongo_to_s3.py index a3fe63f387c1ad..676bc699ca541c 100644 --- a/airflow/contrib/operators/mongo_to_s3.py +++ b/airflow/contrib/operators/mongo_to_s3.py @@ -18,11 +18,12 @@ # under the License. import json +from bson import json_util + from airflow.contrib.hooks.mongo_hook import MongoHook from airflow.hooks.S3_hook import S3Hook from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults -from bson import json_util class MongoToS3Operator(BaseOperator): diff --git a/airflow/contrib/operators/oracle_to_azure_data_lake_transfer.py b/airflow/contrib/operators/oracle_to_azure_data_lake_transfer.py index 967f84828d9fd8..c604f45f28e3a6 100644 --- a/airflow/contrib/operators/oracle_to_azure_data_lake_transfer.py +++ b/airflow/contrib/operators/oracle_to_azure_data_lake_transfer.py @@ -17,15 +17,16 @@ # specific language governing permissions and limitations # under the License. -from airflow.hooks.oracle_hook import OracleHook +import os + +import unicodecsv as csv + from airflow.contrib.hooks.azure_data_lake_hook import AzureDataLakeHook +from airflow.hooks.oracle_hook import OracleHook from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults from airflow.utils.file import TemporaryDirectory -import unicodecsv as csv -import os - class OracleToAzureDataLakeTransfer(BaseOperator): """ diff --git a/airflow/contrib/operators/qubole_check_operator.py b/airflow/contrib/operators/qubole_check_operator.py index 0286763198f154..181468ae81e9e5 100644 --- a/airflow/contrib/operators/qubole_check_operator.py +++ b/airflow/contrib/operators/qubole_check_operator.py @@ -17,11 +17,11 @@ # specific language governing permissions and limitations # under the License. # -from airflow.contrib.operators.qubole_operator import QuboleOperator -from airflow.utils.decorators import apply_defaults from airflow.contrib.hooks.qubole_check_hook import QuboleCheckHook -from airflow.operators.check_operator import CheckOperator, ValueCheckOperator +from airflow.contrib.operators.qubole_operator import QuboleOperator from airflow.exceptions import AirflowException +from airflow.operators.check_operator import CheckOperator, ValueCheckOperator +from airflow.utils.decorators import apply_defaults class QuboleCheckOperator(CheckOperator, QuboleOperator): diff --git a/airflow/contrib/operators/qubole_operator.py b/airflow/contrib/operators/qubole_operator.py index 6497646a85c0b9..a86ca8a3aed80e 100644 --- a/airflow/contrib/operators/qubole_operator.py +++ b/airflow/contrib/operators/qubole_operator.py @@ -19,10 +19,12 @@ """Qubole operator""" from typing import Iterable + +from airflow.contrib.hooks.qubole_hook import ( + COMMAND_ARGS, HYPHEN_ARGS, POSITIONAL_ARGS, QuboleHook, flatten_list, +) from airflow.models.baseoperator import BaseOperator, BaseOperatorLink from airflow.utils.decorators import apply_defaults -from airflow.contrib.hooks.qubole_hook import QuboleHook, COMMAND_ARGS, HYPHEN_ARGS, \ - flatten_list, POSITIONAL_ARGS class QDSLink(BaseOperatorLink): diff --git a/airflow/contrib/operators/redis_publish_operator.py b/airflow/contrib/operators/redis_publish_operator.py index 995a45a91be27c..bb78692879e501 100644 --- a/airflow/contrib/operators/redis_publish_operator.py +++ b/airflow/contrib/operators/redis_publish_operator.py @@ -17,9 +17,9 @@ # specific language governing permissions and limitations # under the License. -from airflow.utils.decorators import apply_defaults from airflow.contrib.hooks.redis_hook import RedisHook from airflow.models import BaseOperator +from airflow.utils.decorators import apply_defaults class RedisPublishOperator(BaseOperator): diff --git a/airflow/contrib/operators/sagemaker_base_operator.py b/airflow/contrib/operators/sagemaker_base_operator.py index d6717fd6b41fcd..344aa9931cfb49 100644 --- a/airflow/contrib/operators/sagemaker_base_operator.py +++ b/airflow/contrib/operators/sagemaker_base_operator.py @@ -18,7 +18,6 @@ # under the License. import json - from typing import Iterable from airflow.contrib.hooks.sagemaker_hook import SageMakerHook diff --git a/airflow/contrib/operators/sagemaker_endpoint_config_operator.py b/airflow/contrib/operators/sagemaker_endpoint_config_operator.py index e4f8ce7c9103f4..75702b6ec726c8 100644 --- a/airflow/contrib/operators/sagemaker_endpoint_config_operator.py +++ b/airflow/contrib/operators/sagemaker_endpoint_config_operator.py @@ -18,8 +18,8 @@ # under the License. from airflow.contrib.operators.sagemaker_base_operator import SageMakerBaseOperator -from airflow.utils.decorators import apply_defaults from airflow.exceptions import AirflowException +from airflow.utils.decorators import apply_defaults class SageMakerEndpointConfigOperator(SageMakerBaseOperator): diff --git a/airflow/contrib/operators/sagemaker_endpoint_operator.py b/airflow/contrib/operators/sagemaker_endpoint_operator.py index 45af3381cb9cdd..5e3f7aff034316 100644 --- a/airflow/contrib/operators/sagemaker_endpoint_operator.py +++ b/airflow/contrib/operators/sagemaker_endpoint_operator.py @@ -19,8 +19,8 @@ from airflow.contrib.hooks.aws_hook import AwsHook from airflow.contrib.operators.sagemaker_base_operator import SageMakerBaseOperator -from airflow.utils.decorators import apply_defaults from airflow.exceptions import AirflowException +from airflow.utils.decorators import apply_defaults class SageMakerEndpointOperator(SageMakerBaseOperator): diff --git a/airflow/contrib/operators/sagemaker_model_operator.py b/airflow/contrib/operators/sagemaker_model_operator.py index 5bfc1feb3149b3..e9e87b8db0b023 100644 --- a/airflow/contrib/operators/sagemaker_model_operator.py +++ b/airflow/contrib/operators/sagemaker_model_operator.py @@ -19,8 +19,8 @@ from airflow.contrib.hooks.aws_hook import AwsHook from airflow.contrib.operators.sagemaker_base_operator import SageMakerBaseOperator -from airflow.utils.decorators import apply_defaults from airflow.exceptions import AirflowException +from airflow.utils.decorators import apply_defaults class SageMakerModelOperator(SageMakerBaseOperator): diff --git a/airflow/contrib/operators/sagemaker_training_operator.py b/airflow/contrib/operators/sagemaker_training_operator.py index f10e13e9736c5c..1e360e485b658c 100644 --- a/airflow/contrib/operators/sagemaker_training_operator.py +++ b/airflow/contrib/operators/sagemaker_training_operator.py @@ -19,8 +19,8 @@ from airflow.contrib.hooks.aws_hook import AwsHook from airflow.contrib.operators.sagemaker_base_operator import SageMakerBaseOperator -from airflow.utils.decorators import apply_defaults from airflow.exceptions import AirflowException +from airflow.utils.decorators import apply_defaults class SageMakerTrainingOperator(SageMakerBaseOperator): diff --git a/airflow/contrib/operators/sagemaker_transform_operator.py b/airflow/contrib/operators/sagemaker_transform_operator.py index 9dcf9ab32764eb..9ac27a537ac6f0 100644 --- a/airflow/contrib/operators/sagemaker_transform_operator.py +++ b/airflow/contrib/operators/sagemaker_transform_operator.py @@ -19,8 +19,8 @@ from airflow.contrib.hooks.aws_hook import AwsHook from airflow.contrib.operators.sagemaker_base_operator import SageMakerBaseOperator -from airflow.utils.decorators import apply_defaults from airflow.exceptions import AirflowException +from airflow.utils.decorators import apply_defaults class SageMakerTransformOperator(SageMakerBaseOperator): diff --git a/airflow/contrib/operators/sagemaker_tuning_operator.py b/airflow/contrib/operators/sagemaker_tuning_operator.py index 2a92bf1b81cf74..e7eaec9dff7f4c 100644 --- a/airflow/contrib/operators/sagemaker_tuning_operator.py +++ b/airflow/contrib/operators/sagemaker_tuning_operator.py @@ -19,8 +19,8 @@ from airflow.contrib.hooks.aws_hook import AwsHook from airflow.contrib.operators.sagemaker_base_operator import SageMakerBaseOperator -from airflow.utils.decorators import apply_defaults from airflow.exceptions import AirflowException +from airflow.utils.decorators import apply_defaults class SageMakerTuningOperator(SageMakerBaseOperator): diff --git a/airflow/contrib/operators/slack_webhook_operator.py b/airflow/contrib/operators/slack_webhook_operator.py index 787872bb2f66e3..00533a127d4193 100644 --- a/airflow/contrib/operators/slack_webhook_operator.py +++ b/airflow/contrib/operators/slack_webhook_operator.py @@ -17,9 +17,9 @@ # specific language governing permissions and limitations # under the License. # +from airflow.contrib.hooks.slack_webhook_hook import SlackWebhookHook from airflow.operators.http_operator import SimpleHttpOperator from airflow.utils.decorators import apply_defaults -from airflow.contrib.hooks.slack_webhook_hook import SlackWebhookHook class SlackWebhookOperator(SimpleHttpOperator): diff --git a/airflow/contrib/operators/spark_jdbc_operator.py b/airflow/contrib/operators/spark_jdbc_operator.py index b1c2a138b9ea2e..725dfc21274f06 100644 --- a/airflow/contrib/operators/spark_jdbc_operator.py +++ b/airflow/contrib/operators/spark_jdbc_operator.py @@ -17,8 +17,8 @@ # specific language governing permissions and limitations # under the License. # -from airflow.contrib.operators.spark_submit_operator import SparkSubmitOperator from airflow.contrib.hooks.spark_jdbc_hook import SparkJDBCHook +from airflow.contrib.operators.spark_submit_operator import SparkSubmitOperator from airflow.utils.decorators import apply_defaults diff --git a/airflow/contrib/operators/spark_sql_operator.py b/airflow/contrib/operators/spark_sql_operator.py index 825a367f6c6d24..fec0b18e1ec134 100644 --- a/airflow/contrib/operators/spark_sql_operator.py +++ b/airflow/contrib/operators/spark_sql_operator.py @@ -17,9 +17,9 @@ # specific language governing permissions and limitations # under the License. # +from airflow.contrib.hooks.spark_sql_hook import SparkSqlHook from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults -from airflow.contrib.hooks.spark_sql_hook import SparkSqlHook class SparkSqlOperator(BaseOperator): diff --git a/airflow/contrib/operators/vertica_to_hive.py b/airflow/contrib/operators/vertica_to_hive.py index 00fdb649c487a9..0aec7e2dae5c72 100644 --- a/airflow/contrib/operators/vertica_to_hive.py +++ b/airflow/contrib/operators/vertica_to_hive.py @@ -19,11 +19,12 @@ from builtins import chr from collections import OrderedDict -import unicodecsv as csv from tempfile import NamedTemporaryFile -from airflow.hooks.hive_hooks import HiveCliHook +import unicodecsv as csv + from airflow.contrib.hooks.vertica_hook import VerticaHook +from airflow.hooks.hive_hooks import HiveCliHook from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults diff --git a/airflow/contrib/operators/vertica_to_mysql.py b/airflow/contrib/operators/vertica_to_mysql.py index 99b55e61bc6e0f..f4fd617b827490 100644 --- a/airflow/contrib/operators/vertica_to_mysql.py +++ b/airflow/contrib/operators/vertica_to_mysql.py @@ -17,18 +17,17 @@ # specific language governing permissions and limitations # under the License. +from contextlib import closing +from tempfile import NamedTemporaryFile + import MySQLdb +import unicodecsv as csv from airflow.contrib.hooks.vertica_hook import VerticaHook from airflow.hooks.mysql_hook import MySqlHook from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults -from contextlib import closing - -import unicodecsv as csv -from tempfile import NamedTemporaryFile - class VerticaToMySqlTransfer(BaseOperator): """ diff --git a/airflow/contrib/operators/winrm_operator.py b/airflow/contrib/operators/winrm_operator.py index e773ab07f1b8b6..074009ea0ce6a6 100644 --- a/airflow/contrib/operators/winrm_operator.py +++ b/airflow/contrib/operators/winrm_operator.py @@ -17,8 +17,8 @@ # specific language governing permissions and limitations # under the License. -from base64 import b64encode import logging +from base64 import b64encode from winrm.exceptions import WinRMOperationTimeoutError diff --git a/airflow/contrib/sensors/aws_athena_sensor.py b/airflow/contrib/sensors/aws_athena_sensor.py index af758b93c1335c..f3222f3b0a95fc 100644 --- a/airflow/contrib/sensors/aws_athena_sensor.py +++ b/airflow/contrib/sensors/aws_athena_sensor.py @@ -18,10 +18,10 @@ # under the License. -from airflow.exceptions import AirflowException -from airflow.utils.decorators import apply_defaults from airflow.contrib.hooks.aws_athena_hook import AWSAthenaHook +from airflow.exceptions import AirflowException from airflow.sensors.base_sensor_operator import BaseSensorOperator +from airflow.utils.decorators import apply_defaults class AthenaSensor(BaseSensorOperator): diff --git a/airflow/contrib/sensors/aws_sqs_sensor.py b/airflow/contrib/sensors/aws_sqs_sensor.py index f58ae6d2c2392f..a8e9b4c4717a1c 100644 --- a/airflow/contrib/sensors/aws_sqs_sensor.py +++ b/airflow/contrib/sensors/aws_sqs_sensor.py @@ -17,10 +17,10 @@ # specific language governing permissions and limitations # under the License. -from airflow.sensors.base_sensor_operator import BaseSensorOperator -from airflow.utils.decorators import apply_defaults from airflow.contrib.hooks.aws_sqs_hook import SQSHook from airflow.exceptions import AirflowException +from airflow.sensors.base_sensor_operator import BaseSensorOperator +from airflow.utils.decorators import apply_defaults class SQSSensor(BaseSensorOperator): diff --git a/airflow/contrib/sensors/bash_sensor.py b/airflow/contrib/sensors/bash_sensor.py index 8afd6bc097e9f1..6883ed71c299ca 100644 --- a/airflow/contrib/sensors/bash_sensor.py +++ b/airflow/contrib/sensors/bash_sensor.py @@ -17,12 +17,13 @@ # specific language governing permissions and limitations # under the License. -from builtins import bytes import os -from subprocess import Popen, STDOUT, PIPE -from tempfile import gettempdir, NamedTemporaryFile -from airflow.utils.decorators import apply_defaults +from builtins import bytes +from subprocess import PIPE, STDOUT, Popen +from tempfile import NamedTemporaryFile, gettempdir + from airflow.sensors.base_sensor_operator import BaseSensorOperator +from airflow.utils.decorators import apply_defaults from airflow.utils.file import TemporaryDirectory diff --git a/airflow/contrib/sensors/jira_sensor.py b/airflow/contrib/sensors/jira_sensor.py index 0583c2221dc936..0de4be740458ee 100644 --- a/airflow/contrib/sensors/jira_sensor.py +++ b/airflow/contrib/sensors/jira_sensor.py @@ -18,8 +18,7 @@ # under the License. from jira.resources import Resource -from airflow.contrib.operators.jira_operator import JIRAError -from airflow.contrib.operators.jira_operator import JiraOperator +from airflow.contrib.operators.jira_operator import JIRAError, JiraOperator from airflow.sensors.base_sensor_operator import BaseSensorOperator from airflow.utils.decorators import apply_defaults diff --git a/airflow/contrib/sensors/redis_pub_sub_sensor.py b/airflow/contrib/sensors/redis_pub_sub_sensor.py index 474a8d21d61ef8..612fb69aac54a4 100644 --- a/airflow/contrib/sensors/redis_pub_sub_sensor.py +++ b/airflow/contrib/sensors/redis_pub_sub_sensor.py @@ -17,9 +17,9 @@ # specific language governing permissions and limitations # under the License. +from airflow.contrib.hooks.redis_hook import RedisHook from airflow.sensors.base_sensor_operator import BaseSensorOperator from airflow.utils.decorators import apply_defaults -from airflow.contrib.hooks.redis_hook import RedisHook class RedisPubSubSensor(BaseSensorOperator): diff --git a/airflow/contrib/sensors/sagemaker_base_sensor.py b/airflow/contrib/sensors/sagemaker_base_sensor.py index 10dd6b2357a66d..bbed49f81ea81f 100644 --- a/airflow/contrib/sensors/sagemaker_base_sensor.py +++ b/airflow/contrib/sensors/sagemaker_base_sensor.py @@ -16,9 +16,9 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from airflow.exceptions import AirflowException from airflow.sensors.base_sensor_operator import BaseSensorOperator from airflow.utils.decorators import apply_defaults -from airflow.exceptions import AirflowException class SageMakerBaseSensor(BaseSensorOperator): diff --git a/airflow/contrib/sensors/sagemaker_training_sensor.py b/airflow/contrib/sensors/sagemaker_training_sensor.py index d550a6c434bfb4..fc2cea32926572 100644 --- a/airflow/contrib/sensors/sagemaker_training_sensor.py +++ b/airflow/contrib/sensors/sagemaker_training_sensor.py @@ -19,7 +19,7 @@ import time -from airflow.contrib.hooks.sagemaker_hook import SageMakerHook, LogState +from airflow.contrib.hooks.sagemaker_hook import LogState, SageMakerHook from airflow.contrib.sensors.sagemaker_base_sensor import SageMakerBaseSensor from airflow.utils.decorators import apply_defaults diff --git a/airflow/contrib/sensors/sftp_sensor.py b/airflow/contrib/sensors/sftp_sensor.py index 0b61c801514cc1..72f3c2ef32da9e 100644 --- a/airflow/contrib/sensors/sftp_sensor.py +++ b/airflow/contrib/sensors/sftp_sensor.py @@ -18,6 +18,7 @@ # under the License. from paramiko import SFTP_NO_SUCH_FILE + from airflow.contrib.hooks.sftp_hook import SFTPHook from airflow.operators.sensors import BaseSensorOperator from airflow.utils.decorators import apply_defaults diff --git a/airflow/contrib/task_runner/cgroup_task_runner.py b/airflow/contrib/task_runner/cgroup_task_runner.py index 5e72d12f772d8b..8f7fde6b5466e3 100644 --- a/airflow/contrib/task_runner/cgroup_task_runner.py +++ b/airflow/contrib/task_runner/cgroup_task_runner.py @@ -22,8 +22,8 @@ import os import uuid -from cgroupspy import trees import psutil +from cgroupspy import trees from airflow.task.task_runner.base_task_runner import BaseTaskRunner from airflow.utils.helpers import reap_process_group diff --git a/airflow/executors/__init__.py b/airflow/executors/__init__.py index 48020ee78a562e..6f7a5ccd01b542 100644 --- a/airflow/executors/__init__.py +++ b/airflow/executors/__init__.py @@ -18,12 +18,13 @@ # under the License. import sys -from airflow.utils.log.logging_mixin import LoggingMixin + from airflow.configuration import conf from airflow.exceptions import AirflowException -from airflow.executors.base_executor import BaseExecutor # noqa +from airflow.executors.base_executor import BaseExecutor # noqa from airflow.executors.local_executor import LocalExecutor from airflow.executors.sequential_executor import SequentialExecutor +from airflow.utils.log.logging_mixin import LoggingMixin DEFAULT_EXECUTOR = None diff --git a/airflow/executors/celery_executor.py b/airflow/executors/celery_executor.py index fa763f2a7d0352..fd1fe80a6b8a23 100644 --- a/airflow/executors/celery_executor.py +++ b/airflow/executors/celery_executor.py @@ -24,11 +24,10 @@ import traceback from multiprocessing import Pool, cpu_count -from celery import Celery -from celery import states as celery_states +from celery import Celery, states as celery_states -from airflow.configuration import conf from airflow.config_templates.default_celery import DEFAULT_CELERY_CONFIG +from airflow.configuration import conf from airflow.exceptions import AirflowException from airflow.executors.base_executor import BaseExecutor from airflow.utils.log.logging_mixin import LoggingMixin diff --git a/airflow/executors/dask_executor.py b/airflow/executors/dask_executor.py index d322f34bdc8294..4bfb5a30df8468 100644 --- a/airflow/executors/dask_executor.py +++ b/airflow/executors/dask_executor.py @@ -17,10 +17,11 @@ # specific language governing permissions and limitations # under the License. -import distributed import subprocess import warnings +import distributed + from airflow.configuration import conf from airflow.executors.base_executor import BaseExecutor diff --git a/airflow/hooks/base_hook.py b/airflow/hooks/base_hook.py index 4f96acd9ba4922..52a6d56c1086b2 100644 --- a/airflow/hooks/base_hook.py +++ b/airflow/hooks/base_hook.py @@ -17,17 +17,14 @@ # specific language governing permissions and limitations # under the License. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, division, print_function, unicode_literals import os import random from typing import Iterable -from airflow.models import Connection from airflow.exceptions import AirflowException +from airflow.models import Connection from airflow.utils.db import provide_session from airflow.utils.log.logging_mixin import LoggingMixin diff --git a/airflow/hooks/hive_hooks.py b/airflow/hooks/hive_hooks.py index f93ba2541d94e2..9e0d46af781b63 100644 --- a/airflow/hooks/hive_hooks.py +++ b/airflow/hooks/hive_hooks.py @@ -22,16 +22,15 @@ import contextlib import os import re +import socket import subprocess import time -import socket from collections import OrderedDict from tempfile import NamedTemporaryFile import six import unicodecsv as csv -from past.builtins import basestring -from past.builtins import unicode +from past.builtins import basestring, unicode from six.moves import zip import airflow.security.utils as utils diff --git a/airflow/hooks/http_hook.py b/airflow/hooks/http_hook.py index 4a9148e8eb0ee6..e5ec00f45775e4 100644 --- a/airflow/hooks/http_hook.py +++ b/airflow/hooks/http_hook.py @@ -22,8 +22,8 @@ import requests import tenacity -from airflow.hooks.base_hook import BaseHook from airflow.exceptions import AirflowException +from airflow.hooks.base_hook import BaseHook class HttpHook(BaseHook): diff --git a/airflow/hooks/postgres_hook.py b/airflow/hooks/postgres_hook.py index aa43bd0f228362..3cda24928364d2 100644 --- a/airflow/hooks/postgres_hook.py +++ b/airflow/hooks/postgres_hook.py @@ -18,9 +18,10 @@ # under the License. import os +from contextlib import closing + import psycopg2 import psycopg2.extensions -from contextlib import closing from airflow.hooks.dbapi_hook import DbApiHook diff --git a/airflow/hooks/samba_hook.py b/airflow/hooks/samba_hook.py index 0820500bbcedaf..03659c61afa1cf 100644 --- a/airflow/hooks/samba_hook.py +++ b/airflow/hooks/samba_hook.py @@ -17,9 +17,10 @@ # specific language governing permissions and limitations # under the License. -from smbclient import SambaClient import os +from smbclient import SambaClient + from airflow.hooks.base_hook import BaseHook diff --git a/airflow/hooks/slack_hook.py b/airflow/hooks/slack_hook.py index 953ce319b1eeee..4c5f31914541af 100644 --- a/airflow/hooks/slack_hook.py +++ b/airflow/hooks/slack_hook.py @@ -18,8 +18,9 @@ # under the License. from slackclient import SlackClient -from airflow.hooks.base_hook import BaseHook + from airflow.exceptions import AirflowException +from airflow.hooks.base_hook import BaseHook class SlackHook(BaseHook): diff --git a/airflow/hooks/zendesk_hook.py b/airflow/hooks/zendesk_hook.py index a6f758345018b2..e8df2ef9bf42d7 100644 --- a/airflow/hooks/zendesk_hook.py +++ b/airflow/hooks/zendesk_hook.py @@ -18,7 +18,9 @@ # under the License. import time -from zdesk import Zendesk, RateLimitError, ZendeskError + +from zdesk import RateLimitError, Zendesk, ZendeskError + from airflow.hooks.base_hook import BaseHook diff --git a/airflow/jobs/__init__.py b/airflow/jobs/__init__.py index 00a149e650dc62..0006226233bdfd 100644 --- a/airflow/jobs/__init__.py +++ b/airflow/jobs/__init__.py @@ -18,7 +18,7 @@ # under the License. # -from airflow.jobs.base_job import BaseJob # noqa: F401 from airflow.jobs.backfill_job import BackfillJob # noqa: F401 -from airflow.jobs.scheduler_job import DagFileProcessor, SchedulerJob # noqa: F401 +from airflow.jobs.base_job import BaseJob # noqa: F401 from airflow.jobs.local_task_job import LocalTaskJob # noqa: F401 +from airflow.jobs.scheduler_job import DagFileProcessor, SchedulerJob # noqa: F401 diff --git a/airflow/lineage/__init__.py b/airflow/lineage/__init__.py index f444139f009794..bb5024e4151df4 100644 --- a/airflow/lineage/__init__.py +++ b/airflow/lineage/__init__.py @@ -17,14 +17,13 @@ # specific language governing permissions and limitations # under the License. from functools import wraps +from itertools import chain from airflow.configuration import conf from airflow.lineage.datasets import DataSet from airflow.utils.log.logging_mixin import LoggingMixin from airflow.utils.module_loading import import_string -from itertools import chain - PIPELINE_OUTLETS = "pipeline_outlets" PIPELINE_INLETS = "pipeline_inlets" diff --git a/airflow/lineage/backend/atlas/__init__.py b/airflow/lineage/backend/atlas/__init__.py index 8e0dfc5dc3dfc1..26b91e0b0e5e7e 100644 --- a/airflow/lineage/backend/atlas/__init__.py +++ b/airflow/lineage/backend/atlas/__init__.py @@ -17,15 +17,15 @@ # specific language governing permissions and limitations # under the License. # +from atlasclient.client import Atlas +from atlasclient.exceptions import HttpError + from airflow.configuration import conf from airflow.lineage import datasets from airflow.lineage.backend import LineageBackend from airflow.lineage.backend.atlas.typedefs import operator_typedef from airflow.utils.timezone import convert_to_utc -from atlasclient.client import Atlas -from atlasclient.exceptions import HttpError - SERIALIZED_DATE_FORMAT_STR = "%Y-%m-%dT%H:%M:%S.%fZ" _username = conf.get("atlas", "username") diff --git a/airflow/migrations/env.py b/airflow/migrations/env.py index 5340f0ecf96943..0340b67ab69d07 100644 --- a/airflow/migrations/env.py +++ b/airflow/migrations/env.py @@ -18,11 +18,12 @@ # under the License. from __future__ import with_statement -from alembic import context + from logging.config import fileConfig -from airflow import settings -from airflow import models +from alembic import context + +from airflow import models, settings # this is the Alembic Config object, which provides # access to the values within the .ini file in use. diff --git a/airflow/migrations/versions/004c1210f153_increase_queue_name_size_limit.py b/airflow/migrations/versions/004c1210f153_increase_queue_name_size_limit.py index 4182a2d4aa969d..76245097b4946a 100644 --- a/airflow/migrations/versions/004c1210f153_increase_queue_name_size_limit.py +++ b/airflow/migrations/versions/004c1210f153_increase_queue_name_size_limit.py @@ -24,8 +24,8 @@ """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = '004c1210f153' diff --git a/airflow/migrations/versions/0a2a5b66e19d_add_task_reschedule_table.py b/airflow/migrations/versions/0a2a5b66e19d_add_task_reschedule_table.py index e7767cc8dc3efe..9e95568a132756 100644 --- a/airflow/migrations/versions/0a2a5b66e19d_add_task_reschedule_table.py +++ b/airflow/migrations/versions/0a2a5b66e19d_add_task_reschedule_table.py @@ -22,8 +22,8 @@ Create Date: 2018-06-17 22:50:00.053620 """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import mysql # revision identifiers, used by Alembic. diff --git a/airflow/migrations/versions/0e2a74e0fc9f_add_time_zone_awareness.py b/airflow/migrations/versions/0e2a74e0fc9f_add_time_zone_awareness.py index b52f3459453fab..16fabccb45d638 100644 --- a/airflow/migrations/versions/0e2a74e0fc9f_add_time_zone_awareness.py +++ b/airflow/migrations/versions/0e2a74e0fc9f_add_time_zone_awareness.py @@ -23,9 +23,9 @@ """ +import sqlalchemy as sa from alembic import op from sqlalchemy.dialects import mysql -import sqlalchemy as sa # revision identifiers, used by Alembic. revision = "0e2a74e0fc9f" diff --git a/airflow/migrations/versions/1507a7289a2f_create_is_encrypted.py b/airflow/migrations/versions/1507a7289a2f_create_is_encrypted.py index fe84254c38caa3..95f49a7d382f88 100644 --- a/airflow/migrations/versions/1507a7289a2f_create_is_encrypted.py +++ b/airflow/migrations/versions/1507a7289a2f_create_is_encrypted.py @@ -24,8 +24,8 @@ Create Date: 2015-08-18 18:57:51.927315 """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.engine.reflection import Inspector # revision identifiers, used by Alembic. diff --git a/airflow/migrations/versions/1968acfc09e3_add_is_encrypted_column_to_variable_.py b/airflow/migrations/versions/1968acfc09e3_add_is_encrypted_column_to_variable_.py index 16ab3495634281..cd9f72ffd4e746 100644 --- a/airflow/migrations/versions/1968acfc09e3_add_is_encrypted_column_to_variable_.py +++ b/airflow/migrations/versions/1968acfc09e3_add_is_encrypted_column_to_variable_.py @@ -24,8 +24,8 @@ Create Date: 2016-02-02 17:20:55.692295 """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = '1968acfc09e3' diff --git a/airflow/migrations/versions/1b38cef5b76e_add_dagrun.py b/airflow/migrations/versions/1b38cef5b76e_add_dagrun.py index 50d53652c4734d..d5d3e0eac45f6b 100644 --- a/airflow/migrations/versions/1b38cef5b76e_add_dagrun.py +++ b/airflow/migrations/versions/1b38cef5b76e_add_dagrun.py @@ -25,8 +25,8 @@ """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = '1b38cef5b76e' diff --git a/airflow/migrations/versions/27c6a30d7c24_add_executor_config_to_task_instance.py b/airflow/migrations/versions/27c6a30d7c24_add_executor_config_to_task_instance.py index 7d6dcb9e1cc24d..c02a1f5242eac0 100644 --- a/airflow/migrations/versions/27c6a30d7c24_add_executor_config_to_task_instance.py +++ b/airflow/migrations/versions/27c6a30d7c24_add_executor_config_to_task_instance.py @@ -24,9 +24,9 @@ """ -from alembic import op -import sqlalchemy as sa import dill +import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = '27c6a30d7c24' diff --git a/airflow/migrations/versions/2e541a1dcfed_task_duration.py b/airflow/migrations/versions/2e541a1dcfed_task_duration.py index 595a5774a6b27e..6fc39f63188548 100644 --- a/airflow/migrations/versions/2e541a1dcfed_task_duration.py +++ b/airflow/migrations/versions/2e541a1dcfed_task_duration.py @@ -25,8 +25,8 @@ """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import mysql # revision identifiers, used by Alembic. diff --git a/airflow/migrations/versions/338e90f54d61_more_logging_into_task_isntance.py b/airflow/migrations/versions/338e90f54d61_more_logging_into_task_isntance.py index 9fe515064c35b7..da59ea025006eb 100644 --- a/airflow/migrations/versions/338e90f54d61_more_logging_into_task_isntance.py +++ b/airflow/migrations/versions/338e90f54d61_more_logging_into_task_isntance.py @@ -24,8 +24,8 @@ Create Date: 2015-08-25 06:09:20.460147 """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = '338e90f54d61' diff --git a/airflow/migrations/versions/33ae817a1ff4_add_kubernetes_resource_checkpointing.py b/airflow/migrations/versions/33ae817a1ff4_add_kubernetes_resource_checkpointing.py index c1e62a2cc2a5f8..dfaa4c51a73a7b 100644 --- a/airflow/migrations/versions/33ae817a1ff4_add_kubernetes_resource_checkpointing.py +++ b/airflow/migrations/versions/33ae817a1ff4_add_kubernetes_resource_checkpointing.py @@ -23,8 +23,8 @@ Create Date: 2017-09-11 15:26:47.598494 """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = '33ae817a1ff4' diff --git a/airflow/migrations/versions/40e67319e3a9_dagrun_config.py b/airflow/migrations/versions/40e67319e3a9_dagrun_config.py index 3da4d5f543038d..703a0f0416cbf2 100644 --- a/airflow/migrations/versions/40e67319e3a9_dagrun_config.py +++ b/airflow/migrations/versions/40e67319e3a9_dagrun_config.py @@ -24,8 +24,8 @@ Create Date: 2015-10-29 08:36:31.726728 """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = '40e67319e3a9' diff --git a/airflow/migrations/versions/41f5f12752f8_add_superuser_field.py b/airflow/migrations/versions/41f5f12752f8_add_superuser_field.py index 6e02582b7e8406..8b3e9b4a41de56 100644 --- a/airflow/migrations/versions/41f5f12752f8_add_superuser_field.py +++ b/airflow/migrations/versions/41f5f12752f8_add_superuser_field.py @@ -23,9 +23,8 @@ """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = '41f5f12752f8' diff --git a/airflow/migrations/versions/4446e08588_dagrun_start_end.py b/airflow/migrations/versions/4446e08588_dagrun_start_end.py index 29932c92060e9b..ef7578e6cdca4f 100644 --- a/airflow/migrations/versions/4446e08588_dagrun_start_end.py +++ b/airflow/migrations/versions/4446e08588_dagrun_start_end.py @@ -25,8 +25,8 @@ """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = '4446e08588' diff --git a/airflow/migrations/versions/4addfa1236f1_add_fractional_seconds_to_mysql_tables.py b/airflow/migrations/versions/4addfa1236f1_add_fractional_seconds_to_mysql_tables.py index 655ff61042d1ed..342b576e9c4bcb 100644 --- a/airflow/migrations/versions/4addfa1236f1_add_fractional_seconds_to_mysql_tables.py +++ b/airflow/migrations/versions/4addfa1236f1_add_fractional_seconds_to_mysql_tables.py @@ -24,9 +24,8 @@ """ -from alembic import op +from alembic import context, op from sqlalchemy.dialects import mysql -from alembic import context # revision identifiers, used by Alembic. revision = '4addfa1236f1' diff --git a/airflow/migrations/versions/502898887f84_adding_extra_to_log.py b/airflow/migrations/versions/502898887f84_adding_extra_to_log.py index 632720a4e27b0c..51f79f6dba5f4d 100644 --- a/airflow/migrations/versions/502898887f84_adding_extra_to_log.py +++ b/airflow/migrations/versions/502898887f84_adding_extra_to_log.py @@ -24,8 +24,8 @@ Create Date: 2015-11-03 22:50:49.794097 """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = '502898887f84' diff --git a/airflow/migrations/versions/561833c1c74b_add_password_column_to_user.py b/airflow/migrations/versions/561833c1c74b_add_password_column_to_user.py index a26a105ac82991..58d93df4a27890 100644 --- a/airflow/migrations/versions/561833c1c74b_add_password_column_to_user.py +++ b/airflow/migrations/versions/561833c1c74b_add_password_column_to_user.py @@ -24,8 +24,8 @@ Create Date: 2015-11-30 06:51:25.872557 """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = '561833c1c74b' diff --git a/airflow/migrations/versions/5e7d17757c7a_add_pid_field_to_taskinstance.py b/airflow/migrations/versions/5e7d17757c7a_add_pid_field_to_taskinstance.py index 77a35db48fad51..062ec93aa6f71b 100644 --- a/airflow/migrations/versions/5e7d17757c7a_add_pid_field_to_taskinstance.py +++ b/airflow/migrations/versions/5e7d17757c7a_add_pid_field_to_taskinstance.py @@ -24,8 +24,8 @@ """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = '5e7d17757c7a' diff --git a/airflow/migrations/versions/64de9cddf6c9_add_task_fails_journal_table.py b/airflow/migrations/versions/64de9cddf6c9_add_task_fails_journal_table.py index 2def57e9048704..e23ff78856dd0a 100644 --- a/airflow/migrations/versions/64de9cddf6c9_add_task_fails_journal_table.py +++ b/airflow/migrations/versions/64de9cddf6c9_add_task_fails_journal_table.py @@ -23,8 +23,8 @@ Create Date: 2016-08-03 14:02:59.203021 """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = '64de9cddf6c9' diff --git a/airflow/migrations/versions/6e96a59344a4_make_taskinstance_pool_not_nullable.py b/airflow/migrations/versions/6e96a59344a4_make_taskinstance_pool_not_nullable.py index 8fc21ad08bed23..55e1a2978edac3 100644 --- a/airflow/migrations/versions/6e96a59344a4_make_taskinstance_pool_not_nullable.py +++ b/airflow/migrations/versions/6e96a59344a4_make_taskinstance_pool_not_nullable.py @@ -24,9 +24,9 @@ """ -from alembic import op import dill import sqlalchemy as sa +from alembic import op from sqlalchemy import Column, Float, Integer, PickleType, String from sqlalchemy.ext.declarative import declarative_base diff --git a/airflow/migrations/versions/74effc47d867_change_datetime_to_datetime2_6_on_mssql_.py b/airflow/migrations/versions/74effc47d867_change_datetime_to_datetime2_6_on_mssql_.py index 30a4365e32262c..774e21ca53c00b 100644 --- a/airflow/migrations/versions/74effc47d867_change_datetime_to_datetime2_6_on_mssql_.py +++ b/airflow/migrations/versions/74effc47d867_change_datetime_to_datetime2_6_on_mssql_.py @@ -25,6 +25,7 @@ """ from collections import defaultdict + from alembic import op from sqlalchemy.dialects import mssql diff --git a/airflow/migrations/versions/856955da8476_fix_sqlite_foreign_key.py b/airflow/migrations/versions/856955da8476_fix_sqlite_foreign_key.py index 91e0cf17831867..c339d8e932199d 100644 --- a/airflow/migrations/versions/856955da8476_fix_sqlite_foreign_key.py +++ b/airflow/migrations/versions/856955da8476_fix_sqlite_foreign_key.py @@ -23,8 +23,8 @@ """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = '856955da8476' diff --git a/airflow/migrations/versions/86770d1215c0_add_kubernetes_scheduler_uniqueness.py b/airflow/migrations/versions/86770d1215c0_add_kubernetes_scheduler_uniqueness.py index 86ad47131e4a1f..520c505f55b2b2 100644 --- a/airflow/migrations/versions/86770d1215c0_add_kubernetes_scheduler_uniqueness.py +++ b/airflow/migrations/versions/86770d1215c0_add_kubernetes_scheduler_uniqueness.py @@ -23,8 +23,8 @@ Create Date: 2018-04-03 15:31:20.814328 """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = '86770d1215c0' diff --git a/airflow/migrations/versions/a56c9515abdc_remove_dag_stat_table.py b/airflow/migrations/versions/a56c9515abdc_remove_dag_stat_table.py index 89dba33c3e3947..a8c95d7cf7dbaa 100644 --- a/airflow/migrations/versions/a56c9515abdc_remove_dag_stat_table.py +++ b/airflow/migrations/versions/a56c9515abdc_remove_dag_stat_table.py @@ -24,8 +24,8 @@ """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = 'a56c9515abdc' diff --git a/airflow/migrations/versions/bba5a7cfc896_add_a_column_to_track_the_encryption_.py b/airflow/migrations/versions/bba5a7cfc896_add_a_column_to_track_the_encryption_.py index 503cd0b6f04068..e2e1a6298433d8 100644 --- a/airflow/migrations/versions/bba5a7cfc896_add_a_column_to_track_the_encryption_.py +++ b/airflow/migrations/versions/bba5a7cfc896_add_a_column_to_track_the_encryption_.py @@ -25,8 +25,8 @@ """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = 'bba5a7cfc896' diff --git a/airflow/migrations/versions/bbc73705a13e_add_notification_sent_column_to_sla_miss.py b/airflow/migrations/versions/bbc73705a13e_add_notification_sent_column_to_sla_miss.py index 9855a6d4daf3c4..6c720344380522 100644 --- a/airflow/migrations/versions/bbc73705a13e_add_notification_sent_column_to_sla_miss.py +++ b/airflow/migrations/versions/bbc73705a13e_add_notification_sent_column_to_sla_miss.py @@ -24,8 +24,8 @@ Create Date: 2016-01-14 18:05:54.871682 """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = 'bbc73705a13e' diff --git a/airflow/migrations/versions/bdaa763e6c56_make_xcom_value_column_a_large_binary.py b/airflow/migrations/versions/bdaa763e6c56_make_xcom_value_column_a_large_binary.py index a1a5270c7bff40..d69d24aed6e916 100644 --- a/airflow/migrations/versions/bdaa763e6c56_make_xcom_value_column_a_large_binary.py +++ b/airflow/migrations/versions/bdaa763e6c56_make_xcom_value_column_a_large_binary.py @@ -23,9 +23,9 @@ Create Date: 2017-08-14 16:06:31.568971 """ -from alembic import op import dill import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = 'bdaa763e6c56' diff --git a/airflow/migrations/versions/c8ffec048a3b_add_fields_to_dag.py b/airflow/migrations/versions/c8ffec048a3b_add_fields_to_dag.py index 70282715edfa6f..ff02efbf9f1a01 100644 --- a/airflow/migrations/versions/c8ffec048a3b_add_fields_to_dag.py +++ b/airflow/migrations/versions/c8ffec048a3b_add_fields_to_dag.py @@ -24,8 +24,8 @@ """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = 'c8ffec048a3b' diff --git a/airflow/migrations/versions/cc1e65623dc7_add_max_tries_column_to_task_instance.py b/airflow/migrations/versions/cc1e65623dc7_add_max_tries_column_to_task_instance.py index 9967586fccba37..ae4192ce0a5d07 100644 --- a/airflow/migrations/versions/cc1e65623dc7_add_max_tries_column_to_task_instance.py +++ b/airflow/migrations/versions/cc1e65623dc7_add_max_tries_column_to_task_instance.py @@ -23,15 +23,15 @@ """ -from alembic import op import sqlalchemy as sa -from airflow import settings -from airflow.models import DagBag - +from alembic import op from sqlalchemy import Column, Integer, String from sqlalchemy.engine.reflection import Inspector from sqlalchemy.ext.declarative import declarative_base +from airflow import settings +from airflow.models import DagBag + # revision identifiers, used by Alembic. revision = 'cc1e65623dc7' down_revision = '127d2bf2dfa7' diff --git a/airflow/migrations/versions/d2ae31099d61_increase_text_size_for_mysql.py b/airflow/migrations/versions/d2ae31099d61_increase_text_size_for_mysql.py index db5afaf023e546..fb198d0557faf7 100644 --- a/airflow/migrations/versions/d2ae31099d61_increase_text_size_for_mysql.py +++ b/airflow/migrations/versions/d2ae31099d61_increase_text_size_for_mysql.py @@ -23,9 +23,8 @@ Create Date: 2017-08-18 17:07:16.686130 """ -from alembic import op +from alembic import context, op from sqlalchemy.dialects import mysql -from alembic import context # revision identifiers, used by Alembic. revision = 'd2ae31099d61' diff --git a/airflow/migrations/versions/dd4ecb8fbee3_add_schedule_interval_to_dag.py b/airflow/migrations/versions/dd4ecb8fbee3_add_schedule_interval_to_dag.py index 7ae562756b63a5..776527e16633b1 100644 --- a/airflow/migrations/versions/dd4ecb8fbee3_add_schedule_interval_to_dag.py +++ b/airflow/migrations/versions/dd4ecb8fbee3_add_schedule_interval_to_dag.py @@ -24,8 +24,8 @@ """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = 'dd4ecb8fbee3' diff --git a/airflow/migrations/versions/e3a246e0dc1_current_schema.py b/airflow/migrations/versions/e3a246e0dc1_current_schema.py index cbf98976458d4f..d2a3cbf99e0aa1 100644 --- a/airflow/migrations/versions/e3a246e0dc1_current_schema.py +++ b/airflow/migrations/versions/e3a246e0dc1_current_schema.py @@ -25,8 +25,8 @@ """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy import func from sqlalchemy.engine.reflection import Inspector diff --git a/airflow/migrations/versions/f2ca10b85618_add_dag_stats_table.py b/airflow/migrations/versions/f2ca10b85618_add_dag_stats_table.py index e14b4b80251091..e3679830bc8ad3 100644 --- a/airflow/migrations/versions/f2ca10b85618_add_dag_stats_table.py +++ b/airflow/migrations/versions/f2ca10b85618_add_dag_stats_table.py @@ -23,8 +23,8 @@ Create Date: 2016-07-20 15:08:28.247537 """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = 'f2ca10b85618' diff --git a/airflow/models/base.py b/airflow/models/base.py index 3dacb8adcedb1e..29fcc5cb76d9fd 100644 --- a/airflow/models/base.py +++ b/airflow/models/base.py @@ -18,6 +18,7 @@ # under the License. from typing import Any + from sqlalchemy import MetaData from sqlalchemy.ext.declarative import declarative_base diff --git a/airflow/models/errors.py b/airflow/models/errors.py index 6a3797ca3df9b4..f25a788c46ba75 100644 --- a/airflow/models/errors.py +++ b/airflow/models/errors.py @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. -from sqlalchemy import Integer, Column, String, Text +from sqlalchemy import Column, Integer, String, Text from airflow.models.base import Base from airflow.utils.sqlalchemy import UtcDateTime diff --git a/airflow/models/kubernetes.py b/airflow/models/kubernetes.py index a18689eefd3167..50c205f0e8358f 100644 --- a/airflow/models/kubernetes.py +++ b/airflow/models/kubernetes.py @@ -19,7 +19,7 @@ import uuid -from sqlalchemy import Column, Boolean, true as sqltrue, String +from sqlalchemy import Boolean, Column, String, true as sqltrue from airflow.models.base import Base from airflow.utils.db import provide_session diff --git a/airflow/models/log.py b/airflow/models/log.py index fa3fed60ee2401..f58eb5227dc023 100644 --- a/airflow/models/log.py +++ b/airflow/models/log.py @@ -17,9 +17,9 @@ # specific language governing permissions and limitations # under the License. -from sqlalchemy import Column, Integer, String, Text, Index +from sqlalchemy import Column, Index, Integer, String, Text -from airflow.models.base import Base, ID_LEN +from airflow.models.base import ID_LEN, Base from airflow.utils import timezone from airflow.utils.sqlalchemy import UtcDateTime diff --git a/airflow/models/pool.py b/airflow/models/pool.py index 4e99b02bd38dff..1cb31182383156 100644 --- a/airflow/models/pool.py +++ b/airflow/models/pool.py @@ -20,9 +20,9 @@ from sqlalchemy import Column, Integer, String, Text from airflow.models.base import Base -from airflow.utils.state import State from airflow.ti_deps.deps.pool_slots_available_dep import STATES_TO_COUNT_AS_RUNNING from airflow.utils.db import provide_session +from airflow.utils.state import State class Pool(Base): diff --git a/airflow/models/slamiss.py b/airflow/models/slamiss.py index 0981be886ea015..44cb0b6c6b725d 100644 --- a/airflow/models/slamiss.py +++ b/airflow/models/slamiss.py @@ -17,9 +17,9 @@ # specific language governing permissions and limitations # under the License. -from sqlalchemy import Boolean, Column, String, Index, Text +from sqlalchemy import Boolean, Column, Index, String, Text -from airflow.models.base import Base, ID_LEN +from airflow.models.base import ID_LEN, Base from airflow.utils.sqlalchemy import UtcDateTime diff --git a/airflow/models/taskfail.py b/airflow/models/taskfail.py index 3fb08b3cee0009..36a979d823fed2 100644 --- a/airflow/models/taskfail.py +++ b/airflow/models/taskfail.py @@ -19,7 +19,7 @@ """Taskfail tracks the failed run durations of each task instance""" from sqlalchemy import Column, Index, Integer, String -from airflow.models.base import Base, ID_LEN +from airflow.models.base import ID_LEN, Base from airflow.utils.sqlalchemy import UtcDateTime diff --git a/airflow/models/taskreschedule.py b/airflow/models/taskreschedule.py index ebca6ee4e4ee6f..b79294c6f95c35 100644 --- a/airflow/models/taskreschedule.py +++ b/airflow/models/taskreschedule.py @@ -19,7 +19,7 @@ """TaskReschedule tracks rescheduled task instances.""" from sqlalchemy import Column, ForeignKeyConstraint, Index, Integer, String, asc -from airflow.models.base import Base, ID_LEN +from airflow.models.base import ID_LEN, Base from airflow.utils.db import provide_session from airflow.utils.sqlalchemy import UtcDateTime diff --git a/airflow/models/variable.py b/airflow/models/variable.py index 0a900c6022ae2b..5dbdbab25d9a50 100644 --- a/airflow/models/variable.py +++ b/airflow/models/variable.py @@ -21,12 +21,12 @@ from builtins import bytes from typing import Any -from sqlalchemy import Column, Integer, String, Text, Boolean +from sqlalchemy import Boolean, Column, Integer, String, Text from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.orm import synonym -from airflow.models.base import Base, ID_LEN -from airflow.models.crypto import get_fernet, InvalidFernetToken +from airflow.models.base import ID_LEN, Base +from airflow.models.crypto import InvalidFernetToken, get_fernet from airflow.utils.db import provide_session from airflow.utils.log.logging_mixin import LoggingMixin diff --git a/airflow/models/xcom.py b/airflow/models/xcom.py index 434cb7151f5dc1..f578f94ebd2710 100644 --- a/airflow/models/xcom.py +++ b/airflow/models/xcom.py @@ -20,18 +20,17 @@ import json import pickle -from sqlalchemy import Column, Integer, String, Index, LargeBinary, and_ +from sqlalchemy import Column, Index, Integer, LargeBinary, String, and_ from sqlalchemy.orm import reconstructor from airflow.configuration import conf -from airflow.models.base import Base, ID_LEN +from airflow.models.base import ID_LEN, Base from airflow.utils import timezone from airflow.utils.db import provide_session from airflow.utils.helpers import as_tuple from airflow.utils.log.logging_mixin import LoggingMixin from airflow.utils.sqlalchemy import UtcDateTime - # MAX XCOM Size is 48KB # https://github.com/apache/airflow/pull/1618#discussion_r68249677 MAX_XCOM_SIZE = 49344 diff --git a/airflow/operators/branch_operator.py b/airflow/operators/branch_operator.py index 249caf10d995ed..af12b8cc16c951 100644 --- a/airflow/operators/branch_operator.py +++ b/airflow/operators/branch_operator.py @@ -18,7 +18,7 @@ # under the License. """Branching operators""" -from typing import Union, Iterable, Dict +from typing import Dict, Iterable, Union from airflow.models import BaseOperator, SkipMixin diff --git a/airflow/operators/papermill_operator.py b/airflow/operators/papermill_operator.py index 7e7bf8bb5a90f8..13da69237baf98 100644 --- a/airflow/operators/papermill_operator.py +++ b/airflow/operators/papermill_operator.py @@ -19,8 +19,8 @@ import papermill as pm -from airflow.models import BaseOperator from airflow.lineage.datasets import DataSet +from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults diff --git a/airflow/operators/presto_check_operator.py b/airflow/operators/presto_check_operator.py index d70dcaa7d25afa..50a8fa3b0730e4 100644 --- a/airflow/operators/presto_check_operator.py +++ b/airflow/operators/presto_check_operator.py @@ -18,8 +18,7 @@ # under the License. from airflow.hooks.presto_hook import PrestoHook -from airflow.operators.check_operator import CheckOperator, \ - ValueCheckOperator, IntervalCheckOperator +from airflow.operators.check_operator import CheckOperator, IntervalCheckOperator, ValueCheckOperator from airflow.utils.decorators import apply_defaults diff --git a/airflow/plugins_manager.py b/airflow/plugins_manager.py index 5d401aebd5378f..e736f6e481a81e 100644 --- a/airflow/plugins_manager.py +++ b/airflow/plugins_manager.py @@ -17,18 +17,16 @@ # specific language governing permissions and limitations # under the License. # -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, division, print_function, unicode_literals -from builtins import object import imp import inspect import os import re +from builtins import object +from typing import Any, List + import pkg_resources -from typing import List, Any from airflow import settings from airflow.models.baseoperator import BaseOperatorLink diff --git a/airflow/sensors/external_task_sensor.py b/airflow/sensors/external_task_sensor.py index ca98cbf50e7367..fcdbd26bf4a97a 100644 --- a/airflow/sensors/external_task_sensor.py +++ b/airflow/sensors/external_task_sensor.py @@ -20,7 +20,7 @@ import os from airflow.exceptions import AirflowException -from airflow.models import TaskInstance, DagBag, DagModel, DagRun +from airflow.models import DagBag, DagModel, DagRun, TaskInstance from airflow.sensors.base_sensor_operator import BaseSensorOperator from airflow.utils.db import provide_session from airflow.utils.decorators import apply_defaults diff --git a/airflow/sensors/s3_prefix_sensor.py b/airflow/sensors/s3_prefix_sensor.py index 16980e7b4ac31e..c8aaf415540467 100644 --- a/airflow/sensors/s3_prefix_sensor.py +++ b/airflow/sensors/s3_prefix_sensor.py @@ -18,7 +18,6 @@ # under the License. from airflow.sensors.base_sensor_operator import BaseSensorOperator - from airflow.utils.decorators import apply_defaults diff --git a/airflow/sentry.py b/airflow/sentry.py index 7d22aec65d3c7b..5cc3ded3b4e0ea 100644 --- a/airflow/sentry.py +++ b/airflow/sentry.py @@ -27,7 +27,6 @@ from airflow.utils.log.logging_mixin import LoggingMixin from airflow.utils.state import State - log = LoggingMixin().log diff --git a/airflow/task/task_runner/__init__.py b/airflow/task/task_runner/__init__.py index 0c422013dbe192..6d0d1d92e7f3c3 100644 --- a/airflow/task/task_runner/__init__.py +++ b/airflow/task/task_runner/__init__.py @@ -18,8 +18,8 @@ # under the License. from airflow.configuration import conf -from airflow.task.task_runner.standard_task_runner import StandardTaskRunner from airflow.exceptions import AirflowException +from airflow.task.task_runner.standard_task_runner import StandardTaskRunner _TASK_RUNNER = conf.get('core', 'TASK_RUNNER') diff --git a/airflow/task/task_runner/base_task_runner.py b/airflow/task/task_runner/base_task_runner.py index efd53e05ed3b60..396fcda42b0923 100644 --- a/airflow/task/task_runner/base_task_runner.py +++ b/airflow/task/task_runner/base_task_runner.py @@ -24,11 +24,9 @@ import subprocess import threading -from airflow.utils.log.logging_mixin import LoggingMixin - from airflow.configuration import conf from airflow.utils.configuration import tmp_configuration_copy - +from airflow.utils.log.logging_mixin import LoggingMixin PYTHONPATH_VAR = 'PYTHONPATH' diff --git a/airflow/utils/cli_action_loggers.py b/airflow/utils/cli_action_loggers.py index 1c96e5f55c89e5..046acf5250b10f 100644 --- a/airflow/utils/cli_action_loggers.py +++ b/airflow/utils/cli_action_loggers.py @@ -24,7 +24,7 @@ from __future__ import absolute_import import logging -from typing import List, Callable +from typing import Callable, List from airflow.utils.db import create_session diff --git a/airflow/utils/compression.py b/airflow/utils/compression.py index 2565299175bb07..07a2790d9d6bf7 100644 --- a/airflow/utils/compression.py +++ b/airflow/utils/compression.py @@ -17,10 +17,10 @@ # specific language governing permissions and limitations # under the License. -from tempfile import NamedTemporaryFile -import shutil -import gzip import bz2 +import gzip +import shutil +from tempfile import NamedTemporaryFile def uncompress_file(input_file_name, file_extension, dest_dir): diff --git a/airflow/utils/configuration.py b/airflow/utils/configuration.py index ca16f871a30932..a8d5d0a5bff9ec 100644 --- a/airflow/utils/configuration.py +++ b/airflow/utils/configuration.py @@ -16,11 +16,10 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from __future__ import unicode_literals -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals -import os import json +import os from tempfile import mkstemp from airflow.configuration import conf diff --git a/airflow/utils/db.py b/airflow/utils/db.py index 0a13069fd9d52c..45feaa4827db50 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -17,15 +17,11 @@ # specific language governing permissions and limitations # under the License. # -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, division, print_function, unicode_literals -from functools import wraps - -import os import contextlib +import os +from functools import wraps from airflow import settings from airflow.configuration import conf diff --git a/airflow/utils/email.py b/airflow/utils/email.py index efd0def8440942..102d94694336b8 100644 --- a/airflow/utils/email.py +++ b/airflow/utils/email.py @@ -17,22 +17,18 @@ # specific language governing permissions and limitations # under the License. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from past.builtins import basestring +from __future__ import absolute_import, division, print_function, unicode_literals import importlib import os import smtplib - -from email.mime.text import MIMEText -from email.mime.multipart import MIMEMultipart from email.mime.application import MIMEApplication +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText from email.utils import formatdate +from past.builtins import basestring + from airflow.configuration import conf from airflow.exceptions import AirflowConfigException from airflow.utils.log.logging_mixin import LoggingMixin diff --git a/airflow/utils/file.py b/airflow/utils/file.py index 918375fe5e5b5f..32b680e9ae0e92 100644 --- a/airflow/utils/file.py +++ b/airflow/utils/file.py @@ -17,15 +17,13 @@ # specific language governing permissions and limitations # under the License. # -from __future__ import absolute_import -from __future__ import unicode_literals +from __future__ import absolute_import, unicode_literals import errno import os import shutil -from tempfile import mkdtemp - from contextlib import contextmanager +from tempfile import mkdtemp @contextmanager diff --git a/airflow/utils/log/es_task_handler.py b/airflow/utils/log/es_task_handler.py index 9982a5a63c49fb..db0cbcb9aad6c1 100644 --- a/airflow/utils/log/es_task_handler.py +++ b/airflow/utils/log/es_task_handler.py @@ -17,19 +17,20 @@ # specific language governing permissions and limitations # under the License. -# Using `from elasticsearch import *` would break elasticsearch mocking used in unit test. -import elasticsearch import logging import sys + +# Using `from elasticsearch import *` would break elasticsearch mocking used in unit test. +import elasticsearch import pendulum from elasticsearch_dsl import Search +from airflow.configuration import conf from airflow.utils import timezone from airflow.utils.helpers import parse_template_string from airflow.utils.log.file_task_handler import FileTaskHandler from airflow.utils.log.json_formatter import JSONFormatter from airflow.utils.log.logging_mixin import LoggingMixin -from airflow.configuration import conf class ElasticsearchTaskHandler(FileTaskHandler, LoggingMixin): diff --git a/airflow/utils/log/file_processor_handler.py b/airflow/utils/log/file_processor_handler.py index 098b018d472753..42d4b8c01d3c3e 100644 --- a/airflow/utils/log/file_processor_handler.py +++ b/airflow/utils/log/file_processor_handler.py @@ -19,10 +19,10 @@ import logging import os +from datetime import datetime from airflow import settings from airflow.utils.helpers import parse_template_string -from datetime import datetime class FileProcessorHandler(logging.Handler): diff --git a/airflow/utils/log/json_formatter.py b/airflow/utils/log/json_formatter.py index a92fbc23c7e68f..b346ae48e78f6e 100644 --- a/airflow/utils/log/json_formatter.py +++ b/airflow/utils/log/json_formatter.py @@ -21,8 +21,8 @@ json_formatter module stores all related to ElasticSearch specific logger classes """ -import logging import json +import logging def merge_dicts(d1, d2): diff --git a/airflow/utils/log/s3_task_handler.py b/airflow/utils/log/s3_task_handler.py index 0f5bf7c90facf0..0f6a530425c3af 100644 --- a/airflow/utils/log/s3_task_handler.py +++ b/airflow/utils/log/s3_task_handler.py @@ -21,8 +21,8 @@ from cached_property import cached_property from airflow.configuration import conf -from airflow.utils.log.logging_mixin import LoggingMixin from airflow.utils.log.file_task_handler import FileTaskHandler +from airflow.utils.log.logging_mixin import LoggingMixin class S3TaskHandler(FileTaskHandler, LoggingMixin): diff --git a/airflow/utils/log/wasb_task_handler.py b/airflow/utils/log/wasb_task_handler.py index c6a22099059f23..30a8420df2f69f 100644 --- a/airflow/utils/log/wasb_task_handler.py +++ b/airflow/utils/log/wasb_task_handler.py @@ -19,12 +19,12 @@ import os import shutil +from azure.common import AzureHttpError from cached_property import cached_property from airflow.configuration import conf -from airflow.utils.log.logging_mixin import LoggingMixin from airflow.utils.log.file_task_handler import FileTaskHandler -from azure.common import AzureHttpError +from airflow.utils.log.logging_mixin import LoggingMixin class WasbTaskHandler(FileTaskHandler, LoggingMixin): diff --git a/airflow/utils/net.py b/airflow/utils/net.py index 206102591344ae..318cfa31b234cd 100644 --- a/airflow/utils/net.py +++ b/airflow/utils/net.py @@ -19,7 +19,8 @@ # import importlib import socket -from airflow.configuration import (conf, AirflowConfigException) + +from airflow.configuration import AirflowConfigException, conf def get_host_ip_address(): diff --git a/airflow/utils/sqlalchemy.py b/airflow/utils/sqlalchemy.py index 111d98ca777698..a8efe27d4ed33d 100644 --- a/airflow/utils/sqlalchemy.py +++ b/airflow/utils/sqlalchemy.py @@ -17,19 +17,16 @@ # specific language governing permissions and limitations # under the License. # -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, division, print_function, unicode_literals import datetime -import os import json -import pendulum +import os +import pendulum from dateutil import relativedelta from sqlalchemy import event, exc -from sqlalchemy.types import Text, DateTime, TypeDecorator +from sqlalchemy.types import DateTime, Text, TypeDecorator from airflow.utils.log.logging_mixin import LoggingMixin diff --git a/airflow/utils/timeout.py b/airflow/utils/timeout.py index 362baa197f33df..8b527b686ea92b 100644 --- a/airflow/utils/timeout.py +++ b/airflow/utils/timeout.py @@ -17,13 +17,10 @@ # specific language governing permissions and limitations # under the License. # -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, division, print_function, unicode_literals -import signal import os +import signal from airflow.exceptions import AirflowTaskTimeout from airflow.utils.log.logging_mixin import LoggingMixin diff --git a/airflow/utils/timezone.py b/airflow/utils/timezone.py index fd3dffa908b4d7..13b0a047d7e3f7 100644 --- a/airflow/utils/timezone.py +++ b/airflow/utils/timezone.py @@ -18,11 +18,11 @@ # under the License. # import datetime as dt + import pendulum from airflow.settings import TIMEZONE - # UTC time zone as a tzinfo instance. utc = pendulum.timezone('UTC') diff --git a/airflow/www/gunicorn_config.py b/airflow/www/gunicorn_config.py index db76b6de0c5af3..4eb1d5fd31eba9 100644 --- a/airflow/www/gunicorn_config.py +++ b/airflow/www/gunicorn_config.py @@ -19,6 +19,7 @@ # under the License. import setproctitle + from airflow import settings diff --git a/airflow/www/validators.py b/airflow/www/validators.py index 3db8b2850e0ad8..bce34a6a5841ca 100644 --- a/airflow/www/validators.py +++ b/airflow/www/validators.py @@ -17,8 +17,7 @@ # specific language governing permissions and limitations # under the License. -from wtforms.validators import EqualTo -from wtforms.validators import ValidationError +from wtforms.validators import EqualTo, ValidationError class GreaterEqualThan(EqualTo): diff --git a/airflow/www_rbac/decorators.py b/airflow/www_rbac/decorators.py index e313977e519f82..de6cb0d6f95a1d 100644 --- a/airflow/www_rbac/decorators.py +++ b/airflow/www_rbac/decorators.py @@ -17,11 +17,13 @@ # specific language governing permissions and limitations # under the License. -import gzip import functools -import pendulum +import gzip from io import BytesIO as IO -from flask import after_this_request, flash, redirect, request, url_for, g + +import pendulum +from flask import after_this_request, flash, g, redirect, request, url_for + from airflow.models import Log from airflow.utils.db import create_session diff --git a/breeze-complete b/breeze-complete index ee593e1444f2ed..82a75adf259c60 100644 --- a/breeze-complete +++ b/breeze-complete @@ -20,7 +20,7 @@ _BREEZE_ALLOWED_PYTHON_VERSIONS=" 2.7 3.5 3.6 " _BREEZE_ALLOWED_ENVS=" docker kubernetes " _BREEZE_ALLOWED_BACKENDS=" sqlite mysql postgres " -_BREEZE_ALLOWED_KUBERNETES_VERSIONS=" v1.13.0 " +_BREEZE_ALLOWED_KUBERNETES_VERSIONS=" v1.13.0 v1.15.3 " _BREEZE_ALLOWED_KUBERNETES_MODES=" persistent_mode git_mode " _BREEZE_ALLOWED_STATIC_CHECKS=" all check-hooks-apply check-merge-conflict check-executables-have-shebangs check-xml detect-private-key doctoc end-of-file-fixer flake8 forbid-tabs insert-license check-apache-license lint-dockerfile mixed-line-ending mypy shellcheck" _BREEZE_DEFAULT_DOCKERHUB_USER="apache" diff --git a/dags/test_dag.py b/dags/test_dag.py index 245764ae1999ce..358a5de1a1ca5c 100644 --- a/dags/test_dag.py +++ b/dags/test_dag.py @@ -21,8 +21,7 @@ """ from datetime import datetime, timedelta -from airflow import DAG -from airflow import utils +from airflow import DAG, utils from airflow.operators.dummy_operator import DummyOperator now = datetime.now() diff --git a/dev/airflow-license b/dev/airflow-license index d52d13a4a24cbe..db64356c4d6092 100755 --- a/dev/airflow-license +++ b/dev/airflow-license @@ -20,9 +20,10 @@ from __future__ import print_function import os import re -import slugify import string +import slugify + # order is important _licenses = {'MIT': ['Permission is hereby granted free of charge', 'The above copyright notice and this permission notice shall'], 'BSD-3': ['Redistributions of source code must retain the above copyright', 'Redistributions in binary form must reproduce the above copyright', 'specific prior written permission'], diff --git a/docs/conf.py b/docs/conf.py index 0ffb9a47c5bc68..af17d2bcda4716 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -280,7 +280,7 @@ # Add any paths that contain custom themes here, relative to this directory. # html_theme_path = [] -import sphinx_rtd_theme # pylint: disable=wrong-import-position,wrong-import-order +import sphinx_rtd_theme # isort:skip pylint: disable=wrong-import-position,wrong-import-order html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] diff --git a/docs/exts/exampleinclude.py b/docs/exts/exampleinclude.py index e9316ec5f5015d..a7061542e9282d 100644 --- a/docs/exts/exampleinclude.py +++ b/docs/exts/exampleinclude.py @@ -30,8 +30,7 @@ from sphinx.directives.code import LiteralIncludeReader from sphinx.locale import _ from sphinx.pycode import ModuleAnalyzer -from sphinx.util import logging -from sphinx.util import parselinenos +from sphinx.util import logging, parselinenos from sphinx.util.docutils import SphinxDirective from sphinx.util.nodes import set_source_info diff --git a/docs/exts/removemarktransform.py b/docs/exts/removemarktransform.py index 13d792257aa90f..d485c581def16f 100644 --- a/docs/exts/removemarktransform.py +++ b/docs/exts/removemarktransform.py @@ -24,7 +24,7 @@ from docutils import nodes # noinspection PyUnresolvedReferences -from pygments.lexers import guess_lexer, Python3Lexer, PythonLexer # pylint: disable=no-name-in-module +from pygments.lexers import Python3Lexer, PythonLexer, guess_lexer # pylint: disable=no-name-in-module from sphinx.transforms import SphinxTransform from sphinx.transforms.post_transforms.code import TrimDoctestFlagsTransform diff --git a/scripts/ci/ci_isort.sh b/scripts/ci/ci_isort.sh new file mode 100755 index 00000000000000..a8ada59956858b --- /dev/null +++ b/scripts/ci/ci_isort.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +export AIRFLOW_CI_SILENT=${AIRFLOW_CI_SILENT:="true"} +export ASSUME_QUIT_TO_ALL_QUESTIONS=${ASSUME_QUIT_TO_ALL_QUESTIONS:="true"} + +# shellcheck source=scripts/ci/_utils.sh +. "${MY_DIR}/_utils.sh" + +basic_sanity_checks + +force_python_3_5 + +script_start + +rebuild_ci_slim_image_if_needed + +run_flake8 "$@" + +script_end diff --git a/scripts/perf/scheduler_ops_metrics.py b/scripts/perf/scheduler_ops_metrics.py index 6000ce2e9a0c01..33d806a30d5899 100644 --- a/scripts/perf/scheduler_ops_metrics.py +++ b/scripts/perf/scheduler_ops_metrics.py @@ -18,9 +18,10 @@ # under the License. import logging -import pandas as pd import sys +import pandas as pd + from airflow import settings from airflow.configuration import conf from airflow.jobs import SchedulerJob diff --git a/setup.cfg b/setup.cfg index ef06e54a73c2f4..7e8f5dc956a43b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,3 +46,13 @@ ignore_errors = True [mypy-airflow.contrib.auth.*] ignore_errors = True + +[isort] +line_length=110 +combine_as_imports = true +default_section = THIRDPARTY +include_trailing_comma = true +known_first_party=airflow,tests +multi_line_output=5 +# Need to be consistent with the exclude config defined in pre-commit-config.yaml +skip=airflow/_vendor,build,.tox,venv diff --git a/tests/api/common/experimental/test_mark_tasks.py b/tests/api/common/experimental/test_mark_tasks.py index f94911a13daa3e..13c4a07f38cd2c 100644 --- a/tests/api/common/experimental/test_mark_tasks.py +++ b/tests/api/common/experimental/test_mark_tasks.py @@ -17,20 +17,21 @@ # specific language governing permissions and limitations # under the License. -import unittest import time +import unittest from datetime import datetime, timedelta from airflow import models -from airflow.configuration import conf from airflow.api.common.experimental.mark_tasks import ( - set_state, _create_dagruns, set_dag_run_state_to_success, set_dag_run_state_to_failed, - set_dag_run_state_to_running) + _create_dagruns, set_dag_run_state_to_failed, set_dag_run_state_to_running, set_dag_run_state_to_success, + set_state, +) +from airflow.configuration import conf +from airflow.models import DagRun from airflow.utils import timezone -from airflow.utils.db import create_session, provide_session from airflow.utils.dates import days_ago +from airflow.utils.db import create_session, provide_session from airflow.utils.state import State -from airflow.models import DagRun from tests.test_utils.db import clear_db_runs DEV_NULL = "/dev/null" diff --git a/tests/api/common/experimental/test_pool.py b/tests/api/common/experimental/test_pool.py index 29c7105571f07b..13874dd97ca92d 100644 --- a/tests/api/common/experimental/test_pool.py +++ b/tests/api/common/experimental/test_pool.py @@ -19,9 +19,9 @@ import six from airflow import models -from airflow.models.pool import Pool from airflow.api.common.experimental import pool as pool_api from airflow.exceptions import AirflowBadRequest, PoolNotFound +from airflow.models.pool import Pool from airflow.utils.db import create_session from tests.test_utils.db import clear_db_pools diff --git a/tests/cli/test_worker_initialisation.py b/tests/cli/test_worker_initialisation.py index 5f0e24060336f4..0415081ac2af8c 100644 --- a/tests/cli/test_worker_initialisation.py +++ b/tests/cli/test_worker_initialisation.py @@ -18,14 +18,16 @@ # under the License. import unittest +from argparse import Namespace + import sqlalchemy + import airflow -from argparse import Namespace +from airflow.bin import cli # noqa from tests.compat import mock, patch from tests.test_utils.config import conf_vars patch('airflow.utils.cli.action_logging', lambda x: x).start() -from airflow.bin import cli # noqa mock_args = Namespace(queues=1, concurrency=1) diff --git a/tests/contrib/hooks/test_aws_glue_catalog_hook.py b/tests/contrib/hooks/test_aws_glue_catalog_hook.py index 57ec7a1a793d33..a4b19354266d1a 100644 --- a/tests/contrib/hooks/test_aws_glue_catalog_hook.py +++ b/tests/contrib/hooks/test_aws_glue_catalog_hook.py @@ -19,6 +19,7 @@ import unittest import boto3 + from airflow.contrib.hooks.aws_glue_catalog_hook import AwsGlueCatalogHook from tests.compat import mock diff --git a/tests/contrib/hooks/test_aws_hook.py b/tests/contrib/hooks/test_aws_hook.py index 736a45baff3286..593ceb7699083b 100644 --- a/tests/contrib/hooks/test_aws_hook.py +++ b/tests/contrib/hooks/test_aws_hook.py @@ -21,8 +21,8 @@ import boto3 -from airflow.models import Connection from airflow.contrib.hooks.aws_hook import AwsHook +from airflow.models import Connection from tests.compat import mock try: diff --git a/tests/contrib/hooks/test_aws_lambda_hook.py b/tests/contrib/hooks/test_aws_lambda_hook.py index ec97098e5119f3..4147fe44e03a28 100644 --- a/tests/contrib/hooks/test_aws_lambda_hook.py +++ b/tests/contrib/hooks/test_aws_lambda_hook.py @@ -18,10 +18,10 @@ # under the License. # -import unittest import io import json import textwrap +import unittest import zipfile from airflow.contrib.hooks.aws_lambda_hook import AwsLambdaHook diff --git a/tests/contrib/hooks/test_azure_container_instance_hook.py b/tests/contrib/hooks/test_azure_container_instance_hook.py index 1c23ea4a852c52..cd2f2a7265fd8e 100644 --- a/tests/contrib/hooks/test_azure_container_instance_hook.py +++ b/tests/contrib/hooks/test_azure_container_instance_hook.py @@ -19,18 +19,16 @@ import json import unittest + +from azure.mgmt.containerinstance.models import ( + Container, ContainerGroup, Logs, ResourceRequests, ResourceRequirements, +) from mock import patch -from airflow.models import Connection from airflow.contrib.hooks.azure_container_instance_hook import AzureContainerInstanceHook +from airflow.models import Connection from airflow.utils import db -from azure.mgmt.containerinstance.models import (Container, - ContainerGroup, - Logs, - ResourceRequests, - ResourceRequirements) - class TestAzureContainerInstanceHook(unittest.TestCase): diff --git a/tests/contrib/hooks/test_azure_container_registry_hook.py b/tests/contrib/hooks/test_azure_container_registry_hook.py index 9d0c2536409fb4..b521603c2f899f 100644 --- a/tests/contrib/hooks/test_azure_container_registry_hook.py +++ b/tests/contrib/hooks/test_azure_container_registry_hook.py @@ -19,8 +19,8 @@ import unittest -from airflow.models import Connection from airflow.contrib.hooks.azure_container_registry_hook import AzureContainerRegistryHook +from airflow.models import Connection from airflow.utils import db diff --git a/tests/contrib/hooks/test_azure_container_volume_hook.py b/tests/contrib/hooks/test_azure_container_volume_hook.py index 484f0b6def476f..a1c7c7ed40fead 100644 --- a/tests/contrib/hooks/test_azure_container_volume_hook.py +++ b/tests/contrib/hooks/test_azure_container_volume_hook.py @@ -19,8 +19,8 @@ import unittest -from airflow.models import Connection from airflow.contrib.hooks.azure_container_volume_hook import AzureContainerVolumeHook +from airflow.models import Connection from airflow.utils import db diff --git a/tests/contrib/hooks/test_azure_cosmos_hook.py b/tests/contrib/hooks/test_azure_cosmos_hook.py index 8acdcd704c2311..2c7ad0908e9558 100644 --- a/tests/contrib/hooks/test_azure_cosmos_hook.py +++ b/tests/contrib/hooks/test_azure_cosmos_hook.py @@ -20,18 +20,16 @@ import json +import logging import unittest import uuid -from airflow.exceptions import AirflowException from airflow.contrib.hooks.azure_cosmos_hook import AzureCosmosDBHook - +from airflow.exceptions import AirflowException from airflow.models import Connection from airflow.utils import db from tests.compat import mock -import logging - class TestAzureCosmosDbHook(unittest.TestCase): diff --git a/tests/contrib/hooks/test_cassandra_hook.py b/tests/contrib/hooks/test_cassandra_hook.py index dc3c1c1d5a8176..9e3153d3ccba62 100644 --- a/tests/contrib/hooks/test_cassandra_hook.py +++ b/tests/contrib/hooks/test_cassandra_hook.py @@ -19,11 +19,12 @@ import unittest -from airflow.contrib.hooks.cassandra_hook import CassandraHook from cassandra.cluster import Cluster from cassandra.policies import ( - TokenAwarePolicy, RoundRobinPolicy, DCAwareRoundRobinPolicy, WhiteListRoundRobinPolicy + DCAwareRoundRobinPolicy, RoundRobinPolicy, TokenAwarePolicy, WhiteListRoundRobinPolicy, ) + +from airflow.contrib.hooks.cassandra_hook import CassandraHook from airflow.models import Connection from airflow.utils import db from tests.compat import mock, patch diff --git a/tests/contrib/hooks/test_databricks_hook.py b/tests/contrib/hooks/test_databricks_hook.py index 98105bf2a17f53..a66f07824a2255 100644 --- a/tests/contrib/hooks/test_databricks_hook.py +++ b/tests/contrib/hooks/test_databricks_hook.py @@ -25,11 +25,7 @@ from requests import exceptions as requests_exceptions from airflow import __version__ -from airflow.contrib.hooks.databricks_hook import ( - DatabricksHook, - RunState, - SUBMIT_RUN_ENDPOINT -) +from airflow.contrib.hooks.databricks_hook import SUBMIT_RUN_ENDPOINT, DatabricksHook, RunState from airflow.exceptions import AirflowException from airflow.models import Connection from airflow.utils import db diff --git a/tests/contrib/hooks/test_dingding_hook.py b/tests/contrib/hooks/test_dingding_hook.py index a246aacf92a746..57e51f78680fe9 100644 --- a/tests/contrib/hooks/test_dingding_hook.py +++ b/tests/contrib/hooks/test_dingding_hook.py @@ -20,10 +20,9 @@ import json import unittest -from airflow.utils import db - from airflow.contrib.hooks.dingding_hook import DingdingHook from airflow.models import Connection +from airflow.utils import db class TestDingdingHook(unittest.TestCase): diff --git a/tests/contrib/hooks/test_discord_webhook_hook.py b/tests/contrib/hooks/test_discord_webhook_hook.py index d0c90010d8238a..cc7673bd1322e2 100644 --- a/tests/contrib/hooks/test_discord_webhook_hook.py +++ b/tests/contrib/hooks/test_discord_webhook_hook.py @@ -21,11 +21,10 @@ import unittest from airflow import AirflowException +from airflow.contrib.hooks.discord_webhook_hook import DiscordWebhookHook from airflow.models import Connection from airflow.utils import db -from airflow.contrib.hooks.discord_webhook_hook import DiscordWebhookHook - class TestDiscordWebhookHook(unittest.TestCase): diff --git a/tests/contrib/hooks/test_emr_hook.py b/tests/contrib/hooks/test_emr_hook.py index b3c77535d93c82..f02fbc31d6cba6 100644 --- a/tests/contrib/hooks/test_emr_hook.py +++ b/tests/contrib/hooks/test_emr_hook.py @@ -19,6 +19,7 @@ # import unittest + import boto3 from airflow.contrib.hooks.emr_hook import EmrHook diff --git a/tests/contrib/hooks/test_gcp_natural_language_hook.py b/tests/contrib/hooks/test_gcp_natural_language_hook.py index 33b03dea96cac3..58e317cfffc0bb 100644 --- a/tests/contrib/hooks/test_gcp_natural_language_hook.py +++ b/tests/contrib/hooks/test_gcp_natural_language_hook.py @@ -18,7 +18,7 @@ # under the License. # import unittest -from typing import Dict, Any +from typing import Any, Dict from google.cloud.language_v1.proto.language_service_pb2 import Document @@ -26,7 +26,6 @@ from tests.compat import mock from tests.contrib.utils.base_gcp_mock import mock_base_gcp_hook_no_default_project_id - API_RESPONSE = {} # type: Dict[Any, Any] DOCUMENT = Document( content="Airflow is a platform to programmatically author, schedule and monitor workflows." diff --git a/tests/contrib/hooks/test_grpc_hook.py b/tests/contrib/hooks/test_grpc_hook.py index 693ad4d5e507ae..4713315b8906b0 100644 --- a/tests/contrib/hooks/test_grpc_hook.py +++ b/tests/contrib/hooks/test_grpc_hook.py @@ -17,16 +17,17 @@ # under the License. import unittest -try: - from StringIO import StringIO -except ImportError: - from io import StringIO -from airflow.exceptions import AirflowConfigException from airflow.contrib.hooks.grpc_hook import GrpcHook +from airflow.exceptions import AirflowConfigException from airflow.models import Connection from tests.compat import mock +try: + from StringIO import StringIO +except ImportError: + from io import StringIO + def get_airflow_connection(auth_type="NO_AUTH", credential_pem_file=None, scopes=None): extra = \ diff --git a/tests/contrib/hooks/test_mongo_hook.py b/tests/contrib/hooks/test_mongo_hook.py index 6dbc8e9b598a76..3d4f3d8adb8887 100644 --- a/tests/contrib/hooks/test_mongo_hook.py +++ b/tests/contrib/hooks/test_mongo_hook.py @@ -17,16 +17,18 @@ # specific language governing permissions and limitations # under the License. import unittest + import pymongo -try: - import mongomock -except ImportError: - mongomock = None from airflow.contrib.hooks.mongo_hook import MongoHook from airflow.models import Connection from airflow.utils import db +try: + import mongomock +except ImportError: + mongomock = None + class MongoHookTest(MongoHook): """ diff --git a/tests/contrib/hooks/test_openfaas_hook.py b/tests/contrib/hooks/test_openfaas_hook.py index d39710ac41ab90..778d8612c42f14 100644 --- a/tests/contrib/hooks/test_openfaas_hook.py +++ b/tests/contrib/hooks/test_openfaas_hook.py @@ -19,11 +19,13 @@ # import unittest + import requests_mock -from airflow.models import Connection + +from airflow import AirflowException from airflow.contrib.hooks.openfaas_hook import OpenFaasHook from airflow.hooks.base_hook import BaseHook -from airflow import AirflowException +from airflow.models import Connection from tests.compat import mock FUNCTION_NAME = "function_name" diff --git a/tests/contrib/hooks/test_opsgenie_alert_hook.py b/tests/contrib/hooks/test_opsgenie_alert_hook.py index 611885f327b51b..6d0306e41fff15 100644 --- a/tests/contrib/hooks/test_opsgenie_alert_hook.py +++ b/tests/contrib/hooks/test_opsgenie_alert_hook.py @@ -17,14 +17,15 @@ # specific language governing permissions and limitations # under the License. # +import json import unittest + import requests_mock -import json +from airflow import AirflowException +from airflow.contrib.hooks.opsgenie_alert_hook import OpsgenieAlertHook from airflow.models import Connection from airflow.utils import db -from airflow.contrib.hooks.opsgenie_alert_hook import OpsgenieAlertHook -from airflow import AirflowException class TestOpsgenieAlertHook(unittest.TestCase): diff --git a/tests/contrib/hooks/test_qubole_check_hook.py b/tests/contrib/hooks/test_qubole_check_hook.py index 150eb45852070a..61790599812bcb 100644 --- a/tests/contrib/hooks/test_qubole_check_hook.py +++ b/tests/contrib/hooks/test_qubole_check_hook.py @@ -18,6 +18,7 @@ # under the License. # import unittest + from airflow.contrib.hooks.qubole_check_hook import parse_first_row diff --git a/tests/contrib/hooks/test_redis_hook.py b/tests/contrib/hooks/test_redis_hook.py index e6696b6b47a7d6..954f2bbce3dfc0 100644 --- a/tests/contrib/hooks/test_redis_hook.py +++ b/tests/contrib/hooks/test_redis_hook.py @@ -19,6 +19,7 @@ import unittest + from airflow.contrib.hooks.redis_hook import RedisHook diff --git a/tests/contrib/hooks/test_redshift_hook.py b/tests/contrib/hooks/test_redshift_hook.py index 08c85727d59680..923c954e846355 100644 --- a/tests/contrib/hooks/test_redshift_hook.py +++ b/tests/contrib/hooks/test_redshift_hook.py @@ -19,10 +19,11 @@ # import unittest + import boto3 -from airflow.contrib.hooks.redshift_hook import RedshiftHook from airflow.contrib.hooks.aws_hook import AwsHook +from airflow.contrib.hooks.redshift_hook import RedshiftHook try: from moto import mock_redshift diff --git a/tests/contrib/hooks/test_sagemaker_hook.py b/tests/contrib/hooks/test_sagemaker_hook.py index da7c69d5dae296..a43c0d1558a78e 100644 --- a/tests/contrib/hooks/test_sagemaker_hook.py +++ b/tests/contrib/hooks/test_sagemaker_hook.py @@ -18,19 +18,20 @@ # under the License. # -import unittest import time +import unittest from datetime import datetime + from tzlocal import get_localzone -from airflow.contrib.hooks.sagemaker_hook import (SageMakerHook, secondary_training_status_changed, - secondary_training_status_message, LogState) from airflow.contrib.hooks.aws_logs_hook import AwsLogsHook -from airflow.hooks.S3_hook import S3Hook +from airflow.contrib.hooks.sagemaker_hook import ( + LogState, SageMakerHook, secondary_training_status_changed, secondary_training_status_message, +) from airflow.exceptions import AirflowException +from airflow.hooks.S3_hook import S3Hook from tests.compat import mock - role = 'arn:aws:iam:role/test-role' path = 'local/data' diff --git a/tests/contrib/hooks/test_spark_jdbc_hook.py b/tests/contrib/hooks/test_spark_jdbc_hook.py index bb0a7a25ace893..3a10996983bf9d 100644 --- a/tests/contrib/hooks/test_spark_jdbc_hook.py +++ b/tests/contrib/hooks/test_spark_jdbc_hook.py @@ -19,11 +19,10 @@ # import unittest +from airflow.contrib.hooks.spark_jdbc_hook import SparkJDBCHook from airflow.models import Connection from airflow.utils import db -from airflow.contrib.hooks.spark_jdbc_hook import SparkJDBCHook - class TestSparkJDBCHook(unittest.TestCase): diff --git a/tests/contrib/operators/test_aws_athena_operator.py b/tests/contrib/operators/test_aws_athena_operator.py index 30aa62fdf5af0f..26ee1deaedf899 100644 --- a/tests/contrib/operators/test_aws_athena_operator.py +++ b/tests/contrib/operators/test_aws_athena_operator.py @@ -20,8 +20,8 @@ import unittest -from airflow.contrib.operators.aws_athena_operator import AWSAthenaOperator from airflow.contrib.hooks.aws_athena_hook import AWSAthenaHook +from airflow.contrib.operators.aws_athena_operator import AWSAthenaOperator from airflow.models import DAG, TaskInstance from airflow.utils import timezone from airflow.utils.timezone import datetime diff --git a/tests/contrib/operators/test_awsbatch_operator.py b/tests/contrib/operators/test_awsbatch_operator.py index 3135a8eef0cff0..8814857c147369 100644 --- a/tests/contrib/operators/test_awsbatch_operator.py +++ b/tests/contrib/operators/test_awsbatch_operator.py @@ -21,8 +21,8 @@ import sys import unittest -from airflow.exceptions import AirflowException from airflow.contrib.operators.awsbatch_operator import AWSBatchOperator +from airflow.exceptions import AirflowException from tests.compat import mock RESPONSE_WITHOUT_FAILURES = { diff --git a/tests/contrib/operators/test_azure_container_instances_operator.py b/tests/contrib/operators/test_azure_container_instances_operator.py index dc18237192ec7a..56f4f6ff77764d 100644 --- a/tests/contrib/operators/test_azure_container_instances_operator.py +++ b/tests/contrib/operators/test_azure_container_instances_operator.py @@ -18,17 +18,15 @@ # under the License. # +import unittest from collections import namedtuple -from airflow.exceptions import AirflowException -from airflow.contrib.operators.azure_container_instances_operator import AzureContainerInstancesOperator -from azure.mgmt.containerinstance.models import (ContainerState, - Event) +from azure.mgmt.containerinstance.models import ContainerState, Event +from airflow.contrib.operators.azure_container_instances_operator import AzureContainerInstancesOperator +from airflow.exceptions import AirflowException from tests.compat import mock -import unittest - def make_mock_cg(container_state, events=[]): """ diff --git a/tests/contrib/operators/test_azure_cosmos_insertdocument_operator.py b/tests/contrib/operators/test_azure_cosmos_insertdocument_operator.py index f776eb56ad0449..7a4e73dc7c0907 100644 --- a/tests/contrib/operators/test_azure_cosmos_insertdocument_operator.py +++ b/tests/contrib/operators/test_azure_cosmos_insertdocument_operator.py @@ -24,7 +24,6 @@ import uuid from airflow.contrib.operators.azure_cosmos_operator import AzureCosmosInsertDocumentOperator - from airflow.models import Connection from airflow.utils import db from tests.compat import mock diff --git a/tests/contrib/operators/test_databricks_operator.py b/tests/contrib/operators/test_databricks_operator.py index 9a7b6ec0416e28..05497b437c70da 100644 --- a/tests/contrib/operators/test_databricks_operator.py +++ b/tests/contrib/operators/test_databricks_operator.py @@ -21,10 +21,11 @@ import unittest from datetime import datetime -from airflow.contrib.hooks.databricks_hook import RunState import airflow.contrib.operators.databricks_operator as databricks_operator -from airflow.contrib.operators.databricks_operator import DatabricksSubmitRunOperator -from airflow.contrib.operators.databricks_operator import DatabricksRunNowOperator +from airflow.contrib.hooks.databricks_hook import RunState +from airflow.contrib.operators.databricks_operator import ( + DatabricksRunNowOperator, DatabricksSubmitRunOperator, +) from airflow.exceptions import AirflowException from airflow.models import DAG from tests.compat import mock diff --git a/tests/contrib/operators/test_discord_webhook_operator.py b/tests/contrib/operators/test_discord_webhook_operator.py index 70077fd5b4d097..2189ec485f36ee 100644 --- a/tests/contrib/operators/test_discord_webhook_operator.py +++ b/tests/contrib/operators/test_discord_webhook_operator.py @@ -20,7 +20,6 @@ import unittest from airflow import DAG - from airflow.contrib.operators.discord_webhook_operator import DiscordWebhookOperator from airflow.utils import timezone diff --git a/tests/contrib/operators/test_ecs_operator.py b/tests/contrib/operators/test_ecs_operator.py index 454e694fc3fae1..8fbe58912f41ae 100644 --- a/tests/contrib/operators/test_ecs_operator.py +++ b/tests/contrib/operators/test_ecs_operator.py @@ -24,8 +24,8 @@ from parameterized import parameterized -from airflow.exceptions import AirflowException from airflow.contrib.operators.ecs_operator import ECSOperator +from airflow.exceptions import AirflowException from tests.compat import mock RESPONSE_WITHOUT_FAILURES = { diff --git a/tests/contrib/operators/test_emr_add_steps_operator.py b/tests/contrib/operators/test_emr_add_steps_operator.py index b2f0666c00ba54..52ce189419eb07 100644 --- a/tests/contrib/operators/test_emr_add_steps_operator.py +++ b/tests/contrib/operators/test_emr_add_steps_operator.py @@ -20,12 +20,11 @@ import unittest from datetime import timedelta -from mock import MagicMock, patch - from airflow import DAG from airflow.contrib.operators.emr_add_steps_operator import EmrAddStepsOperator from airflow.models import TaskInstance from airflow.utils import timezone +from tests.compat import MagicMock, patch DEFAULT_DATE = timezone.datetime(2017, 1, 1) diff --git a/tests/contrib/operators/test_emr_create_job_flow_operator.py b/tests/contrib/operators/test_emr_create_job_flow_operator.py index b403d5362fceaa..3377fee9ba7a16 100644 --- a/tests/contrib/operators/test_emr_create_job_flow_operator.py +++ b/tests/contrib/operators/test_emr_create_job_flow_operator.py @@ -21,12 +21,11 @@ import unittest from datetime import timedelta -from mock import MagicMock, patch - from airflow import DAG from airflow.contrib.operators.emr_create_job_flow_operator import EmrCreateJobFlowOperator from airflow.models import TaskInstance from airflow.utils import timezone +from tests.compat import MagicMock, patch DEFAULT_DATE = timezone.datetime(2017, 1, 1) diff --git a/tests/contrib/operators/test_gcp_translate_speech_operator.py b/tests/contrib/operators/test_gcp_translate_speech_operator.py index a3cfe8b85fad3d..e9a871b2c9e41a 100644 --- a/tests/contrib/operators/test_gcp_translate_speech_operator.py +++ b/tests/contrib/operators/test_gcp_translate_speech_operator.py @@ -18,8 +18,10 @@ # under the License. import unittest -from google.cloud.speech_v1.proto.cloud_speech_pb2 import RecognizeResponse, SpeechRecognitionResult, \ - SpeechRecognitionAlternative + +from google.cloud.speech_v1.proto.cloud_speech_pb2 import ( + RecognizeResponse, SpeechRecognitionAlternative, SpeechRecognitionResult, +) from airflow import AirflowException from airflow.contrib.operators.gcp_translate_speech_operator import GcpTranslateSpeechOperator diff --git a/tests/contrib/operators/test_jenkins_operator.py b/tests/contrib/operators/test_jenkins_operator.py index 634e25a49da6f4..f03c7f2d05a3cb 100644 --- a/tests/contrib/operators/test_jenkins_operator.py +++ b/tests/contrib/operators/test_jenkins_operator.py @@ -21,10 +21,8 @@ import jenkins -from airflow.contrib.operators.jenkins_job_trigger_operator \ - import JenkinsJobTriggerOperator from airflow.contrib.hooks.jenkins_hook import JenkinsHook - +from airflow.contrib.operators.jenkins_job_trigger_operator import JenkinsJobTriggerOperator from airflow.exceptions import AirflowException from tests.compat import mock diff --git a/tests/contrib/operators/test_opsgenie_alert_operator.py b/tests/contrib/operators/test_opsgenie_alert_operator.py index ed1b6f60e09983..36d8e03beccbe4 100644 --- a/tests/contrib/operators/test_opsgenie_alert_operator.py +++ b/tests/contrib/operators/test_opsgenie_alert_operator.py @@ -21,7 +21,6 @@ import unittest from airflow import DAG - from airflow.contrib.operators.opsgenie_alert_operator import OpsgenieAlertOperator from airflow.utils import timezone diff --git a/tests/contrib/operators/test_qubole_check_operator.py b/tests/contrib/operators/test_qubole_check_operator.py index b1692d8c9048cb..6630ca79967e01 100644 --- a/tests/contrib/operators/test_qubole_check_operator.py +++ b/tests/contrib/operators/test_qubole_check_operator.py @@ -19,12 +19,14 @@ # import unittest from datetime import datetime -from airflow.models import DAG -from airflow.exceptions import AirflowException -from airflow.contrib.operators.qubole_check_operator import QuboleValueCheckOperator + +from qds_sdk.commands import HiveCommand + from airflow.contrib.hooks.qubole_check_hook import QuboleCheckHook from airflow.contrib.hooks.qubole_hook import QuboleHook -from qds_sdk.commands import HiveCommand +from airflow.contrib.operators.qubole_check_operator import QuboleValueCheckOperator +from airflow.exceptions import AirflowException +from airflow.models import DAG from tests.compat import mock diff --git a/tests/contrib/operators/test_qubole_operator.py b/tests/contrib/operators/test_qubole_operator.py index fc55f8067e71b4..1a00fb9342b0f1 100644 --- a/tests/contrib/operators/test_qubole_operator.py +++ b/tests/contrib/operators/test_qubole_operator.py @@ -19,15 +19,14 @@ # import unittest -from airflow.utils.timezone import datetime from airflow import settings -from airflow.models import Connection, DAG -from airflow.models.taskinstance import TaskInstance -from airflow.utils import db - from airflow.contrib.hooks.qubole_hook import QuboleHook from airflow.contrib.operators.qubole_operator import QuboleOperator +from airflow.models import DAG, Connection +from airflow.models.taskinstance import TaskInstance +from airflow.utils import db +from airflow.utils.timezone import datetime DAG_ID = "qubole_test_dag" TASK_ID = "test_task" diff --git a/tests/contrib/operators/test_redis_publish_operator.py b/tests/contrib/operators/test_redis_publish_operator.py index 8636b57f30b643..09663675861b6d 100644 --- a/tests/contrib/operators/test_redis_publish_operator.py +++ b/tests/contrib/operators/test_redis_publish_operator.py @@ -19,11 +19,13 @@ import unittest + +from mock import MagicMock + from airflow import DAG -from airflow.contrib.operators.redis_publish_operator import RedisPublishOperator from airflow.contrib.hooks.redis_hook import RedisHook +from airflow.contrib.operators.redis_publish_operator import RedisPublishOperator from airflow.utils import timezone -from mock import MagicMock DEFAULT_DATE = timezone.datetime(2017, 1, 1) diff --git a/tests/contrib/operators/test_s3_to_gcs_operator.py b/tests/contrib/operators/test_s3_to_gcs_operator.py index 3daa6b5fa7028a..a7445e9413f23b 100644 --- a/tests/contrib/operators/test_s3_to_gcs_operator.py +++ b/tests/contrib/operators/test_s3_to_gcs_operator.py @@ -19,8 +19,7 @@ import unittest -from airflow.contrib.operators.s3_to_gcs_operator import \ - S3ToGoogleCloudStorageOperator +from airflow.contrib.operators.s3_to_gcs_operator import S3ToGoogleCloudStorageOperator from tests.compat import mock TASK_ID = 'test-s3-gcs-operator' diff --git a/tests/contrib/operators/test_s3_to_sftp_operator.py b/tests/contrib/operators/test_s3_to_sftp_operator.py index cef922d3178ef3..3d4dab00f4b5c1 100644 --- a/tests/contrib/operators/test_s3_to_sftp_operator.py +++ b/tests/contrib/operators/test_s3_to_sftp_operator.py @@ -19,17 +19,17 @@ import unittest -from airflow.configuration import conf +import boto3 +from moto import mock_s3 + from airflow import models +from airflow.configuration import conf from airflow.contrib.operators.s3_to_sftp_operator import S3ToSFTPOperator from airflow.contrib.operators.ssh_operator import SSHOperator from airflow.models import DAG, TaskInstance from airflow.settings import Session from airflow.utils import timezone from airflow.utils.timezone import datetime -import boto3 -from moto import mock_s3 - TASK_ID = 'test_s3_to_sftp' BUCKET = 'test-s3-bucket' diff --git a/tests/contrib/operators/test_sagemaker_endpoint_config_operator.py b/tests/contrib/operators/test_sagemaker_endpoint_config_operator.py index a294ae04dd6daf..cd1bcbf76fb99a 100644 --- a/tests/contrib/operators/test_sagemaker_endpoint_config_operator.py +++ b/tests/contrib/operators/test_sagemaker_endpoint_config_operator.py @@ -20,8 +20,7 @@ import unittest from airflow.contrib.hooks.sagemaker_hook import SageMakerHook -from airflow.contrib.operators.sagemaker_endpoint_config_operator \ - import SageMakerEndpointConfigOperator +from airflow.contrib.operators.sagemaker_endpoint_config_operator import SageMakerEndpointConfigOperator from airflow.exceptions import AirflowException from tests.compat import mock diff --git a/tests/contrib/operators/test_sagemaker_endpoint_operator.py b/tests/contrib/operators/test_sagemaker_endpoint_operator.py index eb89f06132e25c..cc895941738a50 100644 --- a/tests/contrib/operators/test_sagemaker_endpoint_operator.py +++ b/tests/contrib/operators/test_sagemaker_endpoint_operator.py @@ -20,8 +20,7 @@ import unittest from airflow.contrib.hooks.sagemaker_hook import SageMakerHook -from airflow.contrib.operators.sagemaker_endpoint_operator \ - import SageMakerEndpointOperator +from airflow.contrib.operators.sagemaker_endpoint_operator import SageMakerEndpointOperator from airflow.exceptions import AirflowException from tests.compat import mock diff --git a/tests/contrib/operators/test_sagemaker_model_operator.py b/tests/contrib/operators/test_sagemaker_model_operator.py index bc5fa6537be8c1..252120dda4b2ad 100644 --- a/tests/contrib/operators/test_sagemaker_model_operator.py +++ b/tests/contrib/operators/test_sagemaker_model_operator.py @@ -20,8 +20,7 @@ import unittest from airflow.contrib.hooks.sagemaker_hook import SageMakerHook -from airflow.contrib.operators.sagemaker_model_operator \ - import SageMakerModelOperator +from airflow.contrib.operators.sagemaker_model_operator import SageMakerModelOperator from airflow.exceptions import AirflowException from tests.compat import mock diff --git a/tests/contrib/operators/test_sagemaker_training_operator.py b/tests/contrib/operators/test_sagemaker_training_operator.py index 919b3f1bcac0c2..b385af9c141e0e 100644 --- a/tests/contrib/operators/test_sagemaker_training_operator.py +++ b/tests/contrib/operators/test_sagemaker_training_operator.py @@ -20,8 +20,7 @@ import unittest from airflow.contrib.hooks.sagemaker_hook import SageMakerHook -from airflow.contrib.operators.sagemaker_training_operator \ - import SageMakerTrainingOperator +from airflow.contrib.operators.sagemaker_training_operator import SageMakerTrainingOperator from airflow.exceptions import AirflowException from tests.compat import mock diff --git a/tests/contrib/operators/test_sagemaker_transform_operator.py b/tests/contrib/operators/test_sagemaker_transform_operator.py index 8340c62ac92962..01adcf12b4cc31 100644 --- a/tests/contrib/operators/test_sagemaker_transform_operator.py +++ b/tests/contrib/operators/test_sagemaker_transform_operator.py @@ -20,8 +20,7 @@ import unittest from airflow.contrib.hooks.sagemaker_hook import SageMakerHook -from airflow.contrib.operators.sagemaker_transform_operator \ - import SageMakerTransformOperator +from airflow.contrib.operators.sagemaker_transform_operator import SageMakerTransformOperator from airflow.exceptions import AirflowException from tests.compat import mock diff --git a/tests/contrib/operators/test_sagemaker_tuning_operator.py b/tests/contrib/operators/test_sagemaker_tuning_operator.py index d388d67f5d025b..e5dfe4955fd99c 100644 --- a/tests/contrib/operators/test_sagemaker_tuning_operator.py +++ b/tests/contrib/operators/test_sagemaker_tuning_operator.py @@ -20,8 +20,7 @@ import unittest from airflow.contrib.hooks.sagemaker_hook import SageMakerHook -from airflow.contrib.operators.sagemaker_tuning_operator \ - import SageMakerTuningOperator +from airflow.contrib.operators.sagemaker_tuning_operator import SageMakerTuningOperator from airflow.exceptions import AirflowException from tests.compat import mock diff --git a/tests/contrib/operators/test_sftp_operator.py b/tests/contrib/operators/test_sftp_operator.py index 2ebdd60abcb818..9931b31ba43e16 100644 --- a/tests/contrib/operators/test_sftp_operator.py +++ b/tests/contrib/operators/test_sftp_operator.py @@ -20,10 +20,11 @@ import os import unittest from base64 import b64encode + import six from airflow import models -from airflow.contrib.operators.sftp_operator import SFTPOperator, SFTPOperation +from airflow.contrib.operators.sftp_operator import SFTPOperation, SFTPOperator from airflow.contrib.operators.ssh_operator import SSHOperator from airflow.models import DAG, TaskInstance from airflow.settings import Session diff --git a/tests/contrib/operators/test_sftp_to_s3_operator.py b/tests/contrib/operators/test_sftp_to_s3_operator.py index aefef40246ac1d..f2329ae557cb80 100644 --- a/tests/contrib/operators/test_sftp_to_s3_operator.py +++ b/tests/contrib/operators/test_sftp_to_s3_operator.py @@ -23,14 +23,14 @@ from moto import mock_s3 from airflow import models +from airflow.contrib.hooks.ssh_hook import SSHHook from airflow.contrib.operators.sftp_to_s3_operator import SFTPToS3Operator from airflow.contrib.operators.ssh_operator import SSHOperator +from airflow.hooks.S3_hook import S3Hook from airflow.models import DAG, TaskInstance from airflow.settings import Session from airflow.utils import timezone from airflow.utils.timezone import datetime -from airflow.contrib.hooks.ssh_hook import SSHHook -from airflow.hooks.S3_hook import S3Hook from tests.test_utils.config import conf_vars BUCKET = 'test-bucket' diff --git a/tests/contrib/operators/test_snowflake_operator.py b/tests/contrib/operators/test_snowflake_operator.py index 9aed3d36666027..9f12152ee978e7 100644 --- a/tests/contrib/operators/test_snowflake_operator.py +++ b/tests/contrib/operators/test_snowflake_operator.py @@ -22,12 +22,10 @@ import unittest from airflow import DAG -from airflow.utils import timezone - from airflow.contrib.operators.snowflake_operator import SnowflakeOperator +from airflow.utils import timezone from tests.compat import mock - DEFAULT_DATE = timezone.datetime(2015, 1, 1) DEFAULT_DATE_ISO = DEFAULT_DATE.isoformat() DEFAULT_DATE_DS = DEFAULT_DATE_ISO[:10] diff --git a/tests/contrib/operators/test_spark_jdbc_operator.py b/tests/contrib/operators/test_spark_jdbc_operator.py index 339ef368e4ffdc..42be02b3e2c356 100644 --- a/tests/contrib/operators/test_spark_jdbc_operator.py +++ b/tests/contrib/operators/test_spark_jdbc_operator.py @@ -21,7 +21,6 @@ import unittest from airflow import DAG - from airflow.contrib.operators.spark_jdbc_operator import SparkJDBCOperator from airflow.utils import timezone diff --git a/tests/contrib/operators/test_spark_submit_operator.py b/tests/contrib/operators/test_spark_submit_operator.py index 1a534885d10d93..77e9ebb451260b 100644 --- a/tests/contrib/operators/test_spark_submit_operator.py +++ b/tests/contrib/operators/test_spark_submit_operator.py @@ -19,16 +19,13 @@ # import unittest +from datetime import timedelta from airflow import DAG -from airflow.models import TaskInstance - from airflow.contrib.operators.spark_submit_operator import SparkSubmitOperator +from airflow.models import TaskInstance from airflow.utils import timezone -from datetime import timedelta - - DEFAULT_DATE = timezone.datetime(2017, 1, 1) diff --git a/tests/contrib/sensors/test_athena_sensor.py b/tests/contrib/sensors/test_athena_sensor.py index aa08a839b2e0a7..af25de706841fa 100644 --- a/tests/contrib/sensors/test_athena_sensor.py +++ b/tests/contrib/sensors/test_athena_sensor.py @@ -18,9 +18,10 @@ # under the License. import unittest + from airflow import AirflowException -from airflow.contrib.sensors.aws_athena_sensor import AthenaSensor from airflow.contrib.hooks.aws_athena_hook import AWSAthenaHook +from airflow.contrib.sensors.aws_athena_sensor import AthenaSensor from tests.compat import mock diff --git a/tests/contrib/sensors/test_bash_sensor.py b/tests/contrib/sensors/test_bash_sensor.py index 72fa38469c8fe8..02ab3746d3133a 100644 --- a/tests/contrib/sensors/test_bash_sensor.py +++ b/tests/contrib/sensors/test_bash_sensor.py @@ -18,9 +18,8 @@ # under the License. # -import unittest - import datetime +import unittest from airflow import DAG from airflow.contrib.sensors.bash_sensor import BashSensor diff --git a/tests/contrib/sensors/test_hdfs_sensor.py b/tests/contrib/sensors/test_hdfs_sensor.py index b03b738686ed8b..8169171be7c736 100644 --- a/tests/contrib/sensors/test_hdfs_sensor.py +++ b/tests/contrib/sensors/test_hdfs_sensor.py @@ -17,9 +17,8 @@ # specific language governing permissions and limitations # under the License. import logging -import unittest - import re +import unittest from datetime import timedelta from airflow.contrib.sensors.hdfs_sensor import HdfsSensorFolder, HdfsSensorRegex diff --git a/tests/contrib/sensors/test_mongo_sensor.py b/tests/contrib/sensors/test_mongo_sensor.py index b7cb85634c0166..3ee75cd88b2f2d 100644 --- a/tests/contrib/sensors/test_mongo_sensor.py +++ b/tests/contrib/sensors/test_mongo_sensor.py @@ -26,7 +26,6 @@ from airflow.models import Connection from airflow.utils import db, timezone - DEFAULT_DATE = timezone.datetime(2017, 1, 1) diff --git a/tests/contrib/sensors/test_python_sensor.py b/tests/contrib/sensors/test_python_sensor.py index 1595384be40ba8..a3901e4d003a4c 100644 --- a/tests/contrib/sensors/test_python_sensor.py +++ b/tests/contrib/sensors/test_python_sensor.py @@ -25,7 +25,6 @@ from airflow.exceptions import AirflowSensorTimeout from airflow.utils.timezone import datetime - DEFAULT_DATE = datetime(2015, 1, 1) TEST_DAG_ID = 'python_sensor_dag' diff --git a/tests/contrib/sensors/test_redis_sensor.py b/tests/contrib/sensors/test_redis_sensor.py index 82ab2cbd58f1f0..cd16ceafa8867b 100644 --- a/tests/contrib/sensors/test_redis_sensor.py +++ b/tests/contrib/sensors/test_redis_sensor.py @@ -19,6 +19,7 @@ import unittest + from airflow import DAG from airflow.contrib.hooks.redis_hook import RedisHook from airflow.contrib.sensors.redis_key_sensor import RedisKeySensor diff --git a/tests/contrib/sensors/test_sagemaker_endpoint_sensor.py b/tests/contrib/sensors/test_sagemaker_endpoint_sensor.py index 5c2197a8724b44..970cd01490d5f0 100644 --- a/tests/contrib/sensors/test_sagemaker_endpoint_sensor.py +++ b/tests/contrib/sensors/test_sagemaker_endpoint_sensor.py @@ -19,9 +19,8 @@ import unittest -from airflow.contrib.sensors.sagemaker_endpoint_sensor \ - import SageMakerEndpointSensor from airflow.contrib.hooks.sagemaker_hook import SageMakerHook +from airflow.contrib.sensors.sagemaker_endpoint_sensor import SageMakerEndpointSensor from airflow.exceptions import AirflowException from tests.compat import mock diff --git a/tests/contrib/sensors/test_sagemaker_training_sensor.py b/tests/contrib/sensors/test_sagemaker_training_sensor.py index ab73f15f6d2429..6fece3794968cb 100644 --- a/tests/contrib/sensors/test_sagemaker_training_sensor.py +++ b/tests/contrib/sensors/test_sagemaker_training_sensor.py @@ -20,10 +20,9 @@ import unittest from datetime import datetime -from airflow.contrib.sensors.sagemaker_training_sensor \ - import SageMakerTrainingSensor -from airflow.contrib.hooks.sagemaker_hook import SageMakerHook, LogState from airflow.contrib.hooks.aws_logs_hook import AwsLogsHook +from airflow.contrib.hooks.sagemaker_hook import LogState, SageMakerHook +from airflow.contrib.sensors.sagemaker_training_sensor import SageMakerTrainingSensor from airflow.exceptions import AirflowException from tests.compat import mock diff --git a/tests/contrib/sensors/test_sagemaker_transform_sensor.py b/tests/contrib/sensors/test_sagemaker_transform_sensor.py index 100c83a40be546..b4d81e689d5a48 100644 --- a/tests/contrib/sensors/test_sagemaker_transform_sensor.py +++ b/tests/contrib/sensors/test_sagemaker_transform_sensor.py @@ -19,9 +19,8 @@ import unittest -from airflow.contrib.sensors.sagemaker_transform_sensor \ - import SageMakerTransformSensor from airflow.contrib.hooks.sagemaker_hook import SageMakerHook +from airflow.contrib.sensors.sagemaker_transform_sensor import SageMakerTransformSensor from airflow.exceptions import AirflowException from tests.compat import mock diff --git a/tests/contrib/sensors/test_sagemaker_tuning_sensor.py b/tests/contrib/sensors/test_sagemaker_tuning_sensor.py index df49c0bccfd812..656e2b507859a8 100644 --- a/tests/contrib/sensors/test_sagemaker_tuning_sensor.py +++ b/tests/contrib/sensors/test_sagemaker_tuning_sensor.py @@ -19,9 +19,8 @@ import unittest -from airflow.contrib.sensors.sagemaker_tuning_sensor \ - import SageMakerTuningSensor from airflow.contrib.hooks.sagemaker_hook import SageMakerHook +from airflow.contrib.sensors.sagemaker_tuning_sensor import SageMakerTuningSensor from airflow.exceptions import AirflowException from tests.compat import mock diff --git a/tests/contrib/sensors/test_wasb_sensor.py b/tests/contrib/sensors/test_wasb_sensor.py index fb16ef044d5977..9fa35bd4e4d8aa 100644 --- a/tests/contrib/sensors/test_wasb_sensor.py +++ b/tests/contrib/sensors/test_wasb_sensor.py @@ -18,13 +18,11 @@ # under the License. # -import unittest - import datetime +import unittest from airflow import DAG -from airflow.contrib.sensors.wasb_sensor import WasbBlobSensor -from airflow.contrib.sensors.wasb_sensor import WasbPrefixSensor +from airflow.contrib.sensors.wasb_sensor import WasbBlobSensor, WasbPrefixSensor from tests.compat import mock diff --git a/tests/contrib/sensors/test_weekday_sensor.py b/tests/contrib/sensors/test_weekday_sensor.py index 016f71c56e4f4e..9dcddb0c4d94b2 100644 --- a/tests/contrib/sensors/test_weekday_sensor.py +++ b/tests/contrib/sensors/test_weekday_sensor.py @@ -19,6 +19,7 @@ # import unittest + from airflow import DAG, models from airflow.contrib.sensors.weekday_sensor import DayOfWeekSensor from airflow.contrib.utils.weekday import WeekDay diff --git a/tests/contrib/utils/logging_command_executor.py b/tests/contrib/utils/logging_command_executor.py index 98f870ddb12053..1cf687e9991de7 100644 --- a/tests/contrib/utils/logging_command_executor.py +++ b/tests/contrib/utils/logging_command_executor.py @@ -19,7 +19,7 @@ import os import subprocess -from airflow import LoggingMixin, AirflowException +from airflow import AirflowException, LoggingMixin class LoggingCommandExecutor(LoggingMixin): diff --git a/tests/contrib/utils/test_weekday.py b/tests/contrib/utils/test_weekday.py index 961652a4ea8e6d..790bd19fa100cb 100644 --- a/tests/contrib/utils/test_weekday.py +++ b/tests/contrib/utils/test_weekday.py @@ -17,7 +17,6 @@ # specific language governing permissions and limitations # under the License. import unittest - from enum import Enum from airflow.contrib.utils.weekday import WeekDay diff --git a/tests/dags/test_scheduler_dags.py b/tests/dags/test_scheduler_dags.py index 94e6f8216d226d..ec42a335608b99 100644 --- a/tests/dags/test_scheduler_dags.py +++ b/tests/dags/test_scheduler_dags.py @@ -21,6 +21,7 @@ from airflow.models import DAG from airflow.operators.dummy_operator import DummyOperator + DEFAULT_DATE = datetime(2016, 1, 1) # DAG tests backfill with pooled tasks diff --git a/tests/dags/test_task_view_type_check.py b/tests/dags/test_task_view_type_check.py index a03db8a65cec3f..b78f4de269de23 100644 --- a/tests/dags/test_task_view_type_check.py +++ b/tests/dags/test_task_view_type_check.py @@ -23,7 +23,6 @@ """ import functools import logging - from datetime import datetime from airflow.models import DAG diff --git a/tests/dags_with_system_exit/a_system_exit.py b/tests/dags_with_system_exit/a_system_exit.py index 8d51ddcbb833e4..67aa5ec42170f2 100644 --- a/tests/dags_with_system_exit/a_system_exit.py +++ b/tests/dags_with_system_exit/a_system_exit.py @@ -21,8 +21,8 @@ # Starts with 'a' to get listed first. import sys - from datetime import datetime + from airflow.models import DAG DEFAULT_DATE = datetime(2100, 1, 1) diff --git a/tests/dags_with_system_exit/c_system_exit.py b/tests/dags_with_system_exit/c_system_exit.py index bebaa3e8f39456..39721c186c7879 100644 --- a/tests/dags_with_system_exit/c_system_exit.py +++ b/tests/dags_with_system_exit/c_system_exit.py @@ -21,8 +21,8 @@ # Start with 'z' to get listed last. import sys - from datetime import datetime + from airflow.models import DAG DEFAULT_DATE = datetime(2100, 1, 1) diff --git a/tests/executors/test_executor.py b/tests/executors/test_executor.py index c4ae43e5e4a562..8c02d7072376e2 100644 --- a/tests/executors/test_executor.py +++ b/tests/executors/test_executor.py @@ -20,8 +20,8 @@ from collections import defaultdict from airflow.executors.base_executor import BaseExecutor -from airflow.utils.state import State from airflow.utils.db import create_session +from airflow.utils.state import State class TestExecutor(BaseExecutor): diff --git a/tests/hooks/test_hive_hook.py b/tests/hooks/test_hive_hook.py index 90512c1d518215..79cb17c9c35370 100644 --- a/tests/hooks/test_hive_hook.py +++ b/tests/hooks/test_hive_hook.py @@ -26,7 +26,6 @@ import unittest from collections import OrderedDict -import mock import pandas as pd from hmsclient import HMSClient @@ -38,6 +37,7 @@ from airflow.utils import timezone from airflow.utils.operator_helpers import AIRFLOW_VAR_NAME_FORMAT_MAPPING from airflow.utils.tests import assertEqualIgnoreMultipleSpaces +from tests.compat import mock DEFAULT_DATE = timezone.datetime(2015, 1, 1) DEFAULT_DATE_ISO = DEFAULT_DATE.isoformat() diff --git a/tests/hooks/test_oracle_hook.py b/tests/hooks/test_oracle_hook.py index 927a7b8f110b73..37169eca6b3294 100644 --- a/tests/hooks/test_oracle_hook.py +++ b/tests/hooks/test_oracle_hook.py @@ -21,16 +21,17 @@ import unittest from datetime import datetime -try: - import cx_Oracle -except ImportError: - cx_Oracle = None import numpy from airflow.hooks.oracle_hook import OracleHook from airflow.models import Connection from tests.compat import mock +try: + import cx_Oracle +except ImportError: + cx_Oracle = None + @unittest.skipIf(cx_Oracle is None, 'cx_Oracle package not present') class TestOracleHookConn(unittest.TestCase): diff --git a/tests/hooks/test_pig_hook.py b/tests/hooks/test_pig_hook.py index 8f8dfdc6e53870..f7ae64d425f911 100644 --- a/tests/hooks/test_pig_hook.py +++ b/tests/hooks/test_pig_hook.py @@ -18,6 +18,7 @@ # under the License. import unittest + from airflow.hooks.pig_hook import PigCliHook from tests.compat import mock diff --git a/tests/hooks/test_slack_hook.py b/tests/hooks/test_slack_hook.py index d15c5f251258a0..24bddee0a94833 100644 --- a/tests/hooks/test_slack_hook.py +++ b/tests/hooks/test_slack_hook.py @@ -18,6 +18,7 @@ # under the License. import unittest + from airflow.exceptions import AirflowException from airflow.hooks.slack_hook import SlackHook from tests.compat import mock diff --git a/tests/jobs/test_backfill_job.py b/tests/jobs/test_backfill_job.py index 9c92d8b88dd768..ffdab2de6e1f53 100644 --- a/tests/jobs/test_backfill_job.py +++ b/tests/jobs/test_backfill_job.py @@ -28,10 +28,11 @@ from parameterized import parameterized from airflow import AirflowException, settings -from airflow.configuration import conf from airflow.bin import cli -from airflow.exceptions import AirflowTaskTimeout -from airflow.exceptions import DagConcurrencyLimitReached, NoAvailablePoolSlot, TaskConcurrencyLimitReached +from airflow.configuration import conf +from airflow.exceptions import ( + AirflowTaskTimeout, DagConcurrencyLimitReached, NoAvailablePoolSlot, TaskConcurrencyLimitReached, +) from airflow.jobs import BackfillJob, SchedulerJob from airflow.models import DAG, DagBag, DagRun, Pool, TaskInstance as TI from airflow.operators.dummy_operator import DummyOperator @@ -41,8 +42,7 @@ from airflow.utils.timeout import timeout from tests.compat import Mock, patch from tests.executors.test_executor import TestExecutor -from tests.test_utils.db import clear_db_pools, \ - clear_db_runs, set_default_pool_slots +from tests.test_utils.db import clear_db_pools, clear_db_runs, set_default_pool_slots logger = logging.getLogger(__name__) diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py index 64ae920f38c2cd..534445b0a15a8a 100644 --- a/tests/jobs/test_scheduler_job.py +++ b/tests/jobs/test_scheduler_job.py @@ -18,10 +18,7 @@ # under the License. # -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, division, print_function, unicode_literals import datetime import logging @@ -39,8 +36,7 @@ from airflow.configuration import conf from airflow.executors import BaseExecutor from airflow.jobs import BackfillJob, SchedulerJob -from airflow.models import DAG, DagBag, DagModel, DagRun, Pool, SlaMiss, \ - TaskInstance as TI, errors +from airflow.models import DAG, DagBag, DagModel, DagRun, Pool, SlaMiss, TaskInstance as TI, errors from airflow.operators.bash_operator import BashOperator from airflow.operators.dummy_operator import DummyOperator from airflow.utils import timezone @@ -48,13 +44,12 @@ from airflow.utils.dates import days_ago from airflow.utils.db import create_session, provide_session from airflow.utils.state import State -from tests.compat import MagicMock, Mock, PropertyMock, patch -from tests.compat import mock +from tests.compat import MagicMock, Mock, PropertyMock, mock, patch from tests.core import TEST_DAG_FOLDER from tests.executors.test_executor import TestExecutor -from tests.test_utils.db import clear_db_dags, clear_db_errors, clear_db_pools, \ - clear_db_runs, clear_db_sla_miss, set_default_pool_slots - +from tests.test_utils.db import ( + clear_db_dags, clear_db_errors, clear_db_pools, clear_db_runs, clear_db_sla_miss, set_default_pool_slots, +) DEFAULT_DATE = timezone.datetime(2016, 1, 1) TRY_NUMBER = 1 diff --git a/tests/kubernetes/test_pod_launcher.py b/tests/kubernetes/test_pod_launcher.py index 8943a10978342b..8b50bfa2bf4ff1 100644 --- a/tests/kubernetes/test_pod_launcher.py +++ b/tests/kubernetes/test_pod_launcher.py @@ -15,8 +15,8 @@ # specific language governing permissions and limitations # under the License. import unittest -import mock +import mock from requests.exceptions import BaseHTTPError from airflow import AirflowException diff --git a/tests/lineage/backend/test_atlas.py b/tests/lineage/backend/test_atlas.py index d56e91830664c4..e311db52dd768a 100644 --- a/tests/lineage/backend/test_atlas.py +++ b/tests/lineage/backend/test_atlas.py @@ -18,7 +18,9 @@ # under the License. import unittest -from airflow.configuration import conf, AirflowConfigException +from backports.configparser import DuplicateSectionError + +from airflow.configuration import AirflowConfigException, conf from airflow.lineage.backend.atlas import AtlasBackend from airflow.lineage.datasets import File from airflow.models import DAG, TaskInstance as TI @@ -26,8 +28,6 @@ from airflow.utils import timezone from tests.compat import mock -from backports.configparser import DuplicateSectionError - DEFAULT_DATE = timezone.datetime(2016, 1, 1) diff --git a/tests/macros/test_hive.py b/tests/macros/test_hive.py index c3c4506b0bc611..1fe9b88b2309ea 100644 --- a/tests/macros/test_hive.py +++ b/tests/macros/test_hive.py @@ -17,8 +17,8 @@ # specific language governing permissions and limitations # under the License. -from datetime import datetime import unittest +from datetime import datetime from airflow.macros import hive diff --git a/tests/models/test_cleartasks.py b/tests/models/test_cleartasks.py index c1d0dad598eaa0..527782aa72631b 100644 --- a/tests/models/test_cleartasks.py +++ b/tests/models/test_cleartasks.py @@ -23,7 +23,7 @@ from airflow import settings from airflow.configuration import conf -from airflow.models import DAG, TaskInstance as TI, clear_task_instances, XCom +from airflow.models import DAG, TaskInstance as TI, XCom, clear_task_instances from airflow.operators.dummy_operator import DummyOperator from airflow.utils import timezone from airflow.utils.state import State diff --git a/tests/models/test_dag.py b/tests/models/test_dag.py index c536b3ba78e2b7..4d3154e5bad597 100644 --- a/tests/models/test_dag.py +++ b/tests/models/test_dag.py @@ -24,7 +24,6 @@ import re import unittest from tempfile import NamedTemporaryFile -from tests.compat import mock import pendulum import six @@ -32,7 +31,7 @@ from airflow import models, settings from airflow.configuration import conf -from airflow.exceptions import AirflowException, AirflowDagCycleException +from airflow.exceptions import AirflowDagCycleException, AirflowException from airflow.models import DAG, DagModel, TaskInstance as TI from airflow.operators.dummy_operator import DummyOperator from airflow.operators.subdag_operator import SubDagOperator @@ -40,6 +39,7 @@ from airflow.utils.dag_processing import list_py_file_paths from airflow.utils.state import State from airflow.utils.weight_rule import WeightRule +from tests.compat import mock from tests.models import DEFAULT_DATE diff --git a/tests/models/test_dagrun.py b/tests/models/test_dagrun.py index c90563d75b4604..5f5784fbcbeeba 100644 --- a/tests/models/test_dagrun.py +++ b/tests/models/test_dagrun.py @@ -20,10 +20,9 @@ import datetime import unittest -from airflow import settings, models +from airflow import models, settings from airflow.jobs import BackfillJob -from airflow.models import DAG, DagRun, clear_task_instances -from airflow.models import TaskInstance as TI +from airflow.models import DAG, DagRun, TaskInstance as TI, clear_task_instances from airflow.operators.dummy_operator import DummyOperator from airflow.operators.python_operator import ShortCircuitOperator from airflow.utils import timezone diff --git a/tests/models/test_variable.py b/tests/models/test_variable.py index 7936ccd7a7d96f..6db3bafe5cbe2c 100644 --- a/tests/models/test_variable.py +++ b/tests/models/test_variable.py @@ -22,7 +22,7 @@ from cryptography.fernet import Fernet from airflow import settings -from airflow.models import crypto, Variable +from airflow.models import Variable, crypto from tests.test_utils.config import conf_vars diff --git a/tests/operators/test_bash_operator.py b/tests/operators/test_bash_operator.py index 702796f1b1dc9f..40e7b83628c5f4 100644 --- a/tests/operators/test_bash_operator.py +++ b/tests/operators/test_bash_operator.py @@ -18,7 +18,6 @@ # under the License. import unittest -from tests.compat import mock from datetime import datetime, timedelta from tempfile import NamedTemporaryFile @@ -26,6 +25,7 @@ from airflow.operators.bash_operator import BashOperator from airflow.utils import timezone from airflow.utils.state import State +from tests.compat import mock DEFAULT_DATE = datetime(2016, 1, 1, tzinfo=timezone.utc) END_DATE = datetime(2016, 1, 2, tzinfo=timezone.utc) diff --git a/tests/operators/test_branch_operator.py b/tests/operators/test_branch_operator.py index 7867e195d8384a..0f4b505a5b97da 100644 --- a/tests/operators/test_branch_operator.py +++ b/tests/operators/test_branch_operator.py @@ -17,12 +17,12 @@ # specific language governing permissions and limitations # under the License. -import unittest import datetime +import unittest -from airflow.models import TaskInstance as TI, DAG, DagRun -from airflow.operators.dummy_operator import DummyOperator +from airflow.models import DAG, DagRun, TaskInstance as TI from airflow.operators.branch_operator import BaseBranchOperator +from airflow.operators.dummy_operator import DummyOperator from airflow.utils import timezone from airflow.utils.db import create_session from airflow.utils.state import State diff --git a/tests/operators/test_check_operator.py b/tests/operators/test_check_operator.py index 9ab8df64005640..5824ae104ebb40 100644 --- a/tests/operators/test_check_operator.py +++ b/tests/operators/test_check_operator.py @@ -22,7 +22,7 @@ from airflow.exceptions import AirflowException from airflow.models import DAG -from airflow.operators.check_operator import ValueCheckOperator, CheckOperator, IntervalCheckOperator +from airflow.operators.check_operator import CheckOperator, IntervalCheckOperator, ValueCheckOperator from tests.compat import mock diff --git a/tests/operators/test_docker_operator.py b/tests/operators/test_docker_operator.py index 17db0e4929a5ce..21081772f01575 100644 --- a/tests/operators/test_docker_operator.py +++ b/tests/operators/test_docker_operator.py @@ -17,10 +17,12 @@ # specific language governing permissions and limitations # under the License. -import unittest import logging +import unittest from airflow.exceptions import AirflowException +from tests.compat import mock + try: from airflow.operators.docker_operator import DockerOperator from airflow.hooks.docker_hook import DockerHook @@ -28,8 +30,6 @@ except ImportError: pass -from tests.compat import mock - class DockerOperatorTestCase(unittest.TestCase): @mock.patch('airflow.utils.file.mkdtemp') diff --git a/tests/operators/test_druid_check_operator.py b/tests/operators/test_druid_check_operator.py index c18b56d97115e8..8533f929c7d121 100644 --- a/tests/operators/test_druid_check_operator.py +++ b/tests/operators/test_druid_check_operator.py @@ -18,11 +18,11 @@ # under the License. -from datetime import datetime import unittest +from datetime import datetime -from airflow.models import DAG from airflow.exceptions import AirflowException +from airflow.models import DAG from airflow.operators.druid_check_operator import DruidCheckOperator from tests.compat import mock diff --git a/tests/operators/test_hive_to_druid.py b/tests/operators/test_hive_to_druid.py index b9be8e5723a99b..bbb478a8f57072 100644 --- a/tests/operators/test_hive_to_druid.py +++ b/tests/operators/test_hive_to_druid.py @@ -18,9 +18,10 @@ # under the License. # +import unittest + import requests import requests_mock -import unittest from airflow import DAG from airflow.operators.hive_to_druid import HiveToDruidTransfer diff --git a/tests/operators/test_http_operator.py b/tests/operators/test_http_operator.py index 3656347896b0c0..31f5ef629b24ef 100644 --- a/tests/operators/test_http_operator.py +++ b/tests/operators/test_http_operator.py @@ -21,8 +21,9 @@ import unittest import requests_mock -from airflow.operators.http_operator import SimpleHttpOperator + from airflow.exceptions import AirflowException +from airflow.operators.http_operator import SimpleHttpOperator from tests.compat import mock diff --git a/tests/operators/test_latest_only_operator.py b/tests/operators/test_latest_only_operator.py index 3edff8de8292d8..6d337db9479325 100644 --- a/tests/operators/test_latest_only_operator.py +++ b/tests/operators/test_latest_only_operator.py @@ -26,8 +26,8 @@ from airflow import DAG, settings from airflow.models import TaskInstance -from airflow.operators.latest_only_operator import LatestOnlyOperator from airflow.operators.dummy_operator import DummyOperator +from airflow.operators.latest_only_operator import LatestOnlyOperator from airflow.utils import timezone from airflow.utils.state import State diff --git a/tests/operators/test_s3_to_hive_operator.py b/tests/operators/test_s3_to_hive_operator.py index 8366465a9ebd88..d6f931c9fb126b 100644 --- a/tests/operators/test_s3_to_hive_operator.py +++ b/tests/operators/test_s3_to_hive_operator.py @@ -17,20 +17,20 @@ # specific language governing permissions and limitations # under the License. -import unittest - -from tests.compat import mock +import bz2 +import errno +import filecmp import logging -from itertools import product -from airflow.operators.s3_to_hive_operator import S3ToHiveTransfer +import shutil +import unittest from collections import OrderedDict -from airflow.exceptions import AirflowException -from tempfile import NamedTemporaryFile, mkdtemp from gzip import GzipFile -import bz2 -import shutil -import filecmp -import errno +from itertools import product +from tempfile import NamedTemporaryFile, mkdtemp + +from airflow.exceptions import AirflowException +from airflow.operators.s3_to_hive_operator import S3ToHiveTransfer +from tests.compat import mock try: import boto3 diff --git a/tests/operators/test_slack_operator.py b/tests/operators/test_slack_operator.py index 08434f9622e722..20ef89510a7705 100644 --- a/tests/operators/test_slack_operator.py +++ b/tests/operators/test_slack_operator.py @@ -17,8 +17,9 @@ # specific language governing permissions and limitations # under the License. -import unittest import json +import unittest + from airflow.exceptions import AirflowException from airflow.operators.slack_operator import SlackAPIPostOperator from tests.compat import mock diff --git a/tests/operators/test_virtualenv_operator.py b/tests/operators/test_virtualenv_operator.py index 0b5b62e1abdd05..8b621b429c3526 100644 --- a/tests/operators/test_virtualenv_operator.py +++ b/tests/operators/test_virtualenv_operator.py @@ -20,19 +20,17 @@ from __future__ import print_function, unicode_literals import datetime - -import funcsigs import sys import unittest - from subprocess import CalledProcessError +import funcsigs + from airflow import DAG +from airflow.exceptions import AirflowException from airflow.operators.python_operator import PythonVirtualenvOperator from airflow.utils import timezone -from airflow.exceptions import AirflowException - DEFAULT_DATE = timezone.datetime(2016, 1, 1) END_DATE = timezone.datetime(2016, 1, 2) INTERVAL = datetime.timedelta(hours=12) diff --git a/tests/security/test_kerberos.py b/tests/security/test_kerberos.py index cac73e912859ec..a816ec81727625 100644 --- a/tests/security/test_kerberos.py +++ b/tests/security/test_kerberos.py @@ -21,9 +21,9 @@ import unittest from argparse import Namespace +from airflow import LoggingMixin from airflow.configuration import conf from airflow.security.kerberos import renew_from_kt -from airflow import LoggingMixin from tests.test_utils.config import conf_vars diff --git a/tests/sensors/test_external_task_sensor.py b/tests/sensors/test_external_task_sensor.py index 649f6bf629f40e..e4a9a4f3e2b95e 100644 --- a/tests/sensors/test_external_task_sensor.py +++ b/tests/sensors/test_external_task_sensor.py @@ -17,12 +17,11 @@ # specific language governing permissions and limitations # under the License. import unittest -from datetime import timedelta, time +from datetime import time, timedelta -from airflow import DAG, settings -from airflow import exceptions +from airflow import DAG, exceptions, settings from airflow.exceptions import AirflowException, AirflowSensorTimeout -from airflow.models import TaskInstance, DagBag +from airflow.models import DagBag, TaskInstance from airflow.operators.bash_operator import BashOperator from airflow.operators.dummy_operator import DummyOperator from airflow.sensors.external_task_sensor import ExternalTaskSensor diff --git a/tests/sensors/test_hdfs_sensor.py b/tests/sensors/test_hdfs_sensor.py index a827e04680f5a7..d9cb4a6afa0c92 100644 --- a/tests/sensors/test_hdfs_sensor.py +++ b/tests/sensors/test_hdfs_sensor.py @@ -17,7 +17,6 @@ # specific language governing permissions and limitations # under the License. import unittest - from datetime import timedelta from airflow.exceptions import AirflowSensorTimeout diff --git a/tests/sensors/test_named_hive_partition_sensor.py b/tests/sensors/test_named_hive_partition_sensor.py index ef7ba13b142880..47afe167bf1820 100644 --- a/tests/sensors/test_named_hive_partition_sensor.py +++ b/tests/sensors/test_named_hive_partition_sensor.py @@ -21,9 +21,9 @@ from datetime import timedelta from airflow import DAG, operators +from airflow.hooks.hive_hooks import HiveMetastoreHook from airflow.sensors.named_hive_partition_sensor import NamedHivePartitionSensor from airflow.utils.timezone import datetime -from airflow.hooks.hive_hooks import HiveMetastoreHook DEFAULT_DATE = datetime(2015, 1, 1) DEFAULT_DATE_ISO = DEFAULT_DATE.isoformat() diff --git a/tests/sensors/test_timedelta_sensor.py b/tests/sensors/test_timedelta_sensor.py index f4d7ceb157350c..57f5ef7c7a0d2e 100644 --- a/tests/sensors/test_timedelta_sensor.py +++ b/tests/sensors/test_timedelta_sensor.py @@ -17,10 +17,9 @@ # specific language governing permissions and limitations # under the License. import unittest - from datetime import timedelta -from airflow import models, DAG +from airflow import DAG, models from airflow.sensors.time_delta_sensor import TimeDeltaSensor from airflow.utils.timezone import datetime diff --git a/tests/sensors/test_timeout_sensor.py b/tests/sensors/test_timeout_sensor.py index 0c6cbeb6f8fcc7..091b40f2ccc850 100644 --- a/tests/sensors/test_timeout_sensor.py +++ b/tests/sensors/test_timeout_sensor.py @@ -16,9 +16,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import unittest - import time +import unittest from datetime import timedelta from airflow import DAG diff --git a/tests/test_local_settings.py b/tests/test_local_settings.py index 7e5be54b46d5c2..a0a92127a308fb 100644 --- a/tests/test_local_settings.py +++ b/tests/test_local_settings.py @@ -21,10 +21,9 @@ import sys import tempfile import unittest -from tests.compat import MagicMock, Mock, call, patch from airflow.contrib.kubernetes.pod import Pod - +from tests.compat import MagicMock, Mock, call, patch SETTINGS_FILE_POLICY = """ def policy(task_instance): diff --git a/tests/test_utils/config.py b/tests/test_utils/config.py index f53e8fa0b81133..65570d8dc53f4b 100644 --- a/tests/test_utils/config.py +++ b/tests/test_utils/config.py @@ -23,8 +23,8 @@ from contextlib import contextmanager import os -from airflow.configuration import conf from airflow import settings +from airflow.configuration import conf @contextmanager diff --git a/tests/test_utils/db.py b/tests/test_utils/db.py index c0f4b927c968d5..d967a23eeb9c6b 100644 --- a/tests/test_utils/db.py +++ b/tests/test_utils/db.py @@ -16,9 +16,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from airflow.models import DagModel, DagRun, errors, Pool, SlaMiss, TaskInstance -from airflow.utils.db import add_default_pool_if_not_exists -from airflow.utils.db import create_session +from airflow.models import DagModel, DagRun, Pool, SlaMiss, TaskInstance, errors +from airflow.utils.db import add_default_pool_if_not_exists, create_session def clear_db_runs(): diff --git a/tests/ti_deps/deps/test_not_in_retry_period_dep.py b/tests/ti_deps/deps/test_not_in_retry_period_dep.py index ca0726dc0c3880..a583888a1b3c5a 100644 --- a/tests/ti_deps/deps/test_not_in_retry_period_dep.py +++ b/tests/ti_deps/deps/test_not_in_retry_period_dep.py @@ -19,13 +19,14 @@ import unittest from datetime import timedelta + from freezegun import freeze_time -from mock import Mock from airflow.models import TaskInstance from airflow.ti_deps.deps.not_in_retry_period_dep import NotInRetryPeriodDep from airflow.utils.state import State from airflow.utils.timezone import datetime +from tests.compat import Mock class NotInRetryPeriodDepTest(unittest.TestCase): diff --git a/tests/ti_deps/deps/test_pool_slots_available_dep.py b/tests/ti_deps/deps/test_pool_slots_available_dep.py index 58d9317c841d10..4eb67b0081b8e4 100644 --- a/tests/ti_deps/deps/test_pool_slots_available_dep.py +++ b/tests/ti_deps/deps/test_pool_slots_available_dep.py @@ -21,8 +21,7 @@ from mock import Mock, patch from airflow.models import Pool -from airflow.ti_deps.deps.pool_slots_available_dep import PoolSlotsAvailableDep, \ - STATES_TO_COUNT_AS_RUNNING +from airflow.ti_deps.deps.pool_slots_available_dep import STATES_TO_COUNT_AS_RUNNING, PoolSlotsAvailableDep from airflow.utils.db import create_session from tests.test_utils import db diff --git a/tests/ti_deps/deps/test_trigger_rule_dep.py b/tests/ti_deps/deps/test_trigger_rule_dep.py index d20bab37504a79..a564d3dae4b0a6 100644 --- a/tests/ti_deps/deps/test_trigger_rule_dep.py +++ b/tests/ti_deps/deps/test_trigger_rule_dep.py @@ -21,10 +21,10 @@ from datetime import datetime from airflow.models import BaseOperator, TaskInstance -from airflow.utils.trigger_rule import TriggerRule from airflow.ti_deps.deps.trigger_rule_dep import TriggerRuleDep from airflow.utils.db import create_session from airflow.utils.state import State +from airflow.utils.trigger_rule import TriggerRule class TriggerRuleDepTest(unittest.TestCase): diff --git a/tests/utils/log/elasticmock/__init__.py b/tests/utils/log/elasticmock/__init__.py index 287963368c19c8..3e5aad9bbfe667 100644 --- a/tests/utils/log/elasticmock/__init__.py +++ b/tests/utils/log/elasticmock/__init__.py @@ -43,6 +43,7 @@ from typing import Dict from elasticsearch.client import _normalize_hosts + from tests.compat import patch from .fake_elasticsearch import FakeElasticsearch diff --git a/tests/utils/log/test_file_processor_handler.py b/tests/utils/log/test_file_processor_handler.py index 6180c55a14a664..1ce17da7234e13 100644 --- a/tests/utils/log/test_file_processor_handler.py +++ b/tests/utils/log/test_file_processor_handler.py @@ -17,14 +17,15 @@ # specific language governing permissions and limitations # under the License. -import shutil import os +import shutil import unittest +from datetime import timedelta + from freezegun import freeze_time -from airflow.utils.log.file_processor_handler import FileProcessorHandler from airflow.utils import timezone -from datetime import timedelta +from airflow.utils.log.file_processor_handler import FileProcessorHandler class TestFileProcessorHandler(unittest.TestCase): diff --git a/tests/utils/test_dates.py b/tests/utils/test_dates.py index 85613d079139d9..89eceeefe6786b 100644 --- a/tests/utils/test_dates.py +++ b/tests/utils/test_dates.py @@ -17,12 +17,12 @@ # specific language governing permissions and limitations # under the License. +import unittest from datetime import datetime, timedelta + import pendulum -import unittest -from airflow.utils import dates -from airflow.utils import timezone +from airflow.utils import dates, timezone class Dates(unittest.TestCase): diff --git a/tests/utils/test_db.py b/tests/utils/test_db.py index 718e2ae1066fed..7e2bb10e46f832 100644 --- a/tests/utils/test_db.py +++ b/tests/utils/test_db.py @@ -19,13 +19,13 @@ import unittest -from airflow.models import Base as airflow_base - -from airflow.settings import engine from alembic.autogenerate import compare_metadata from alembic.migration import MigrationContext from sqlalchemy import MetaData +from airflow.models import Base as airflow_base +from airflow.settings import engine + class DbTest(unittest.TestCase): diff --git a/tests/utils/test_decorators.py b/tests/utils/test_decorators.py index d23cdcc3564c86..9d2f89d7eb0e1c 100644 --- a/tests/utils/test_decorators.py +++ b/tests/utils/test_decorators.py @@ -19,8 +19,8 @@ import unittest -from airflow.utils.decorators import apply_defaults from airflow.exceptions import AirflowException +from airflow.utils.decorators import apply_defaults # Essentially similar to airflow.models.BaseOperator diff --git a/tests/utils/test_dot_renderer.py b/tests/utils/test_dot_renderer.py index 87c930087de1ea..4ce13704d31cae 100644 --- a/tests/utils/test_dot_renderer.py +++ b/tests/utils/test_dot_renderer.py @@ -27,7 +27,6 @@ from airflow.operators.python_operator import PythonOperator from airflow.utils import dot_renderer - START_DATE = datetime.datetime.now() diff --git a/tests/utils/test_email.py b/tests/utils/test_email.py index 6dae967c4abb96..a6215ce78d6055 100644 --- a/tests/utils/test_email.py +++ b/tests/utils/test_email.py @@ -18,6 +18,7 @@ # under the License. import unittest + from airflow.utils.email import get_email_address_list EMAILS = ['test1@example.com', 'test2@example.com'] diff --git a/tests/utils/test_helpers.py b/tests/utils/test_helpers.py index 8a3ec470957e6d..2e0e07caf01cfc 100644 --- a/tests/utils/test_helpers.py +++ b/tests/utils/test_helpers.py @@ -29,10 +29,10 @@ import six from airflow import DAG -from airflow.utils import helpers +from airflow.exceptions import AirflowException from airflow.models import TaskInstance from airflow.operators.dummy_operator import DummyOperator -from airflow.exceptions import AirflowException +from airflow.utils import helpers class TestHelpers(unittest.TestCase): diff --git a/tests/utils/test_json.py b/tests/utils/test_json.py index ce0eecef5ac45b..ea9c7a68e3ee94 100644 --- a/tests/utils/test_json.py +++ b/tests/utils/test_json.py @@ -17,9 +17,9 @@ # specific language governing permissions and limitations # under the License. -from datetime import datetime, date import json import unittest +from datetime import date, datetime import numpy as np diff --git a/tests/utils/test_log_handlers.py b/tests/utils/test_log_handlers.py index 72a5793f7e77ba..2773306042b3c5 100644 --- a/tests/utils/test_log_handlers.py +++ b/tests/utils/test_log_handlers.py @@ -21,17 +21,18 @@ import logging.config import os import unittest + import six -from airflow.models import TaskInstance, DAG, DagRun from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG +from airflow.models import DAG, DagRun, TaskInstance from airflow.operators.dummy_operator import DummyOperator from airflow.operators.python_operator import PythonOperator -from airflow.utils.timezone import datetime -from airflow.utils.log.logging_mixin import set_context -from airflow.utils.log.file_task_handler import FileTaskHandler from airflow.utils.db import create_session +from airflow.utils.log.file_task_handler import FileTaskHandler +from airflow.utils.log.logging_mixin import set_context from airflow.utils.state import State +from airflow.utils.timezone import datetime DEFAULT_DATE = datetime(2016, 1, 1) TASK_LOGGER = 'airflow.task' diff --git a/tests/utils/test_operator_helpers.py b/tests/utils/test_operator_helpers.py index a3586012f11082..58fc3709c41140 100644 --- a/tests/utils/test_operator_helpers.py +++ b/tests/utils/test_operator_helpers.py @@ -20,9 +20,8 @@ import unittest from datetime import datetime -import mock - from airflow.utils import operator_helpers +from tests.compat import mock class TestOperatorHelpers(unittest.TestCase): diff --git a/tests/utils/test_tests.py b/tests/utils/test_tests.py index 8ef5b5ce222620..67429a3053726f 100644 --- a/tests/utils/test_tests.py +++ b/tests/utils/test_tests.py @@ -18,6 +18,7 @@ # under the License. import unittest + from airflow.utils.tests import assertEqualIgnoreMultipleSpaces diff --git a/tests/utils/test_timezone.py b/tests/utils/test_timezone.py index e31895e2df5214..cad07aa693ae46 100644 --- a/tests/utils/test_timezone.py +++ b/tests/utils/test_timezone.py @@ -18,9 +18,10 @@ # under the License. import datetime -import pendulum import unittest +import pendulum + from airflow.utils import timezone CET = pendulum.timezone("Europe/Paris") diff --git a/tests/utils/test_trigger_rule.py b/tests/utils/test_trigger_rule.py index 1167ac8c36edb9..8934cc500289cf 100644 --- a/tests/utils/test_trigger_rule.py +++ b/tests/utils/test_trigger_rule.py @@ -18,6 +18,7 @@ # under the License. import unittest + from airflow.utils.trigger_rule import TriggerRule diff --git a/tests/utils/test_weight_rule.py b/tests/utils/test_weight_rule.py index c862263387cc31..947281359aa4c0 100644 --- a/tests/utils/test_weight_rule.py +++ b/tests/utils/test_weight_rule.py @@ -18,6 +18,7 @@ # under the License. import unittest + from airflow.utils.weight_rule import WeightRule