From ff3f50208719eb8e1664ac6796c8eec41ac777c5 Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Sun, 18 Aug 2019 19:26:35 -0700 Subject: [PATCH] [AIRFLOW-5251] add missing typing-extensions dep for py37 --- .travis.yml | 4 +-- airflow/api/auth/backend/default.py | 2 +- .../contrib/operators/awsbatch_operator.py | 2 +- airflow/contrib/operators/ecs_operator.py | 2 +- airflow/hooks/dbapi_hook.py | 2 +- airflow/models/crypto.py | 2 +- airflow/typing.py | 30 +++++++++++++++++++ airflow/utils/helpers.py | 1 - docs/conf.py | 1 + setup.py | 2 +- .../test_cloud_storage_transfer_service.py | 2 +- 11 files changed, 40 insertions(+), 10 deletions(-) create mode 100644 airflow/typing.py diff --git a/.travis.yml b/.travis.yml index 3ea9d97faf868..9b9ea53dc7c26 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,9 +38,9 @@ jobs: env: BACKEND=sqlite ENV=docker python: "3.5" stage: test - - name: "Tests mysql python 3.6" + - name: "Tests mysql python 3.7" env: BACKEND=mysql ENV=docker - python: "3.6" + python: "3.7" stage: test - name: "Tests postgres kubernetes python 3.6 (persistent)" env: BACKEND=postgres ENV=kubernetes KUBERNETES_VERSION=v1.13.0 KUBERNETES_MODE=persistent_mode diff --git a/airflow/api/auth/backend/default.py b/airflow/api/auth/backend/default.py index f80799672da9e..bd1ec3643f987 100644 --- a/airflow/api/auth/backend/default.py +++ b/airflow/api/auth/backend/default.py @@ -19,7 +19,7 @@ """Default authentication backend - everything is allowed""" from typing import Optional from functools import wraps -from typing_extensions import Protocol +from airflow.typing import Protocol class ClientAuthProtocol(Protocol): diff --git a/airflow/contrib/operators/awsbatch_operator.py b/airflow/contrib/operators/awsbatch_operator.py index dca15e1febc69..70dea3fea5b6f 100644 --- a/airflow/contrib/operators/awsbatch_operator.py +++ b/airflow/contrib/operators/awsbatch_operator.py @@ -18,7 +18,7 @@ # under the License. # from typing import Optional -from typing_extensions import Protocol +from airflow.typing import Protocol import sys from math import pow diff --git a/airflow/contrib/operators/ecs_operator.py b/airflow/contrib/operators/ecs_operator.py index d7ede094b3af0..102f50e886f7d 100644 --- a/airflow/contrib/operators/ecs_operator.py +++ b/airflow/contrib/operators/ecs_operator.py @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. from typing import Optional -from typing_extensions import Protocol +from airflow.typing import Protocol import sys import re diff --git a/airflow/hooks/dbapi_hook.py b/airflow/hooks/dbapi_hook.py index f047968fcee2c..af891928c9b18 100644 --- a/airflow/hooks/dbapi_hook.py +++ b/airflow/hooks/dbapi_hook.py @@ -18,7 +18,7 @@ # under the License. from typing import Optional -from typing_extensions import Protocol +from airflow.typing import Protocol from datetime import datetime from contextlib import closing diff --git a/airflow/models/crypto.py b/airflow/models/crypto.py index 20a68863cfdfb..327ef4f8371d0 100644 --- a/airflow/models/crypto.py +++ b/airflow/models/crypto.py @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. -from typing_extensions import Protocol +from airflow.typing import Protocol from typing import Optional from airflow import configuration from airflow.exceptions import AirflowException diff --git a/airflow/typing.py b/airflow/typing.py new file mode 100644 index 0000000000000..f4ed7bd95a822 --- /dev/null +++ b/airflow/typing.py @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +This module provides helper code to make type annotation within Airflow +codebase easier. +""" + +try: + # Protocol is only added to typing module starting from python 3.8 + # we can safely remove this shim import after Airflow drops support for + # <3.8 + from typing import Protocol # noqa # pylint: disable=unused-import +except ImportError: + from typing_extensions import Protocol # type: ignore # noqa diff --git a/airflow/utils/helpers.py b/airflow/utils/helpers.py index 56179b76f7553..fdf24433524b5 100644 --- a/airflow/utils/helpers.py +++ b/airflow/utils/helpers.py @@ -16,7 +16,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - import errno import psutil diff --git a/docs/conf.py b/docs/conf.py index af310d3f78ea1..cf7b8a11eca42 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -188,6 +188,7 @@ '_api/airflow/index.rst', '_api/airflow/jobs', '_api/airflow/lineage', + '_api/airflow/typing', '_api/airflow/logging_config', '_api/airflow/macros', '_api/airflow/migrations', diff --git a/setup.py b/setup.py index 0049021683137..59b6410d89f1d 100644 --- a/setup.py +++ b/setup.py @@ -361,7 +361,7 @@ def do_setup(): 'tzlocal>=1.4,<2.0.0', 'unicodecsv>=0.14.1', 'zope.deprecation>=4.0, <5.0', - 'typing-extensions>=3.7.4;python_version<"3.7"', + 'typing-extensions>=3.7.4;python_version<"3.8"', ], setup_requires=[ 'docutils>=0.14, <1.0', diff --git a/tests/gcp/hooks/test_cloud_storage_transfer_service.py b/tests/gcp/hooks/test_cloud_storage_transfer_service.py index e7f73a24f5d70..48917cb8c4dbf 100644 --- a/tests/gcp/hooks/test_cloud_storage_transfer_service.py +++ b/tests/gcp/hooks/test_cloud_storage_transfer_service.py @@ -233,7 +233,7 @@ def test_wait_for_transfer_job(self, get_conn, mock_sleep): {FILTER_PROJECT_ID: TEST_PROJECT_ID, FILTER_JOB_NAMES: ["transferJobs/test-job"]}, ) - mock_sleep.assert_called_once_with(TIME_TO_SLEEP_IN_SECONDS) + mock_sleep.assert_called_with(TIME_TO_SLEEP_IN_SECONDS) @mock.patch('time.sleep') @mock.patch('airflow.gcp.hooks.cloud_storage_transfer_service.GCPTransferServiceHook.get_conn')