From efa90d87e4ba891c61bb4657232ed54f500121c8 Mon Sep 17 00:00:00 2001 From: Luke Sneeringer Date: Thu, 16 Mar 2017 09:54:46 -0700 Subject: [PATCH 01/10] Non-functional datastore fix for #3152. --- datastore/google/cloud/datastore/helpers.py | 2 +- datastore/tests/unit/test_helpers.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/datastore/google/cloud/datastore/helpers.py b/datastore/google/cloud/datastore/helpers.py index ee4537317030..91b08452812f 100644 --- a/datastore/google/cloud/datastore/helpers.py +++ b/datastore/google/cloud/datastore/helpers.py @@ -133,7 +133,7 @@ def entity_from_protobuf(pb): # Check if ``value_pb`` was excluded from index. Lists need to be # special-cased and we require all ``exclude_from_indexes`` values # in a list agree. - if is_list: + if is_list and len(value) > 0: exclude_values = set(value_pb.exclude_from_indexes for value_pb in value_pb.array_value.values) if len(exclude_values) != 1: diff --git a/datastore/tests/unit/test_helpers.py b/datastore/tests/unit/test_helpers.py index 18ff98e64781..8529f214640c 100644 --- a/datastore/tests/unit/test_helpers.py +++ b/datastore/tests/unit/test_helpers.py @@ -135,6 +135,27 @@ def test_mismatched_value_indexed(self): with self.assertRaises(ValueError): self._call_fut(entity_pb) + def test_index_mismatch_ignores_empty_list(self): + from google.cloud.proto.datastore.v1 import entity_pb2 + from google.cloud.datastore.helpers import _new_value_pb + + _PROJECT = 'PROJECT' + _KIND = 'KIND' + _ID = 1234 + entity_pb = entity_pb2.Entity() + entity_pb.key.partition_id.project_id = _PROJECT + entity_pb.key.path.add(kind=_KIND, id=_ID) + + array_val_pb = _new_value_pb(entity_pb, 'baz') + array_pb = array_val_pb.array_value.values + + # unindexed_value_pb1 = array_pb.add() + # unindexed_value_pb1.integer_value = 10 + + entity = self._call_fut(entity_pb) + entity_dict = dict(entity) + self.assertIsInstance(entity_dict['baz'], list) + def test_entity_no_key(self): from google.cloud.proto.datastore.v1 import entity_pb2 From b96eaca9a7c4e3d2e8db29fbe080c01f3495e811 Mon Sep 17 00:00:00 2001 From: Luke Sneeringer Date: Tue, 8 Aug 2017 09:38:08 -0700 Subject: [PATCH 02/10] Make CONTRIBUTING.rst be up to date. (#3750) --- CONTRIBUTING.rst | 229 +++++++++++++---------------------------------- 1 file changed, 60 insertions(+), 169 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 95a4dd13cfdb..25c449a2bad5 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -21,7 +21,7 @@ In order to add a feature to ``google-cloud-python``: documentation (in ``docs/``). - The feature must work fully on the following CPython versions: 2.7, - 3.4, and 3.5 on both UNIX and Windows. + 3.4, 3.5, and 3.6 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -57,7 +57,7 @@ You'll have to create a development environment to hack on Now your local repo is set up such that you will push changes to your GitHub repo, from which you can submit a pull request. -To work on the codebase and run the tests, we recommend using ``tox``, +To work on the codebase and run the tests, we recommend using ``nox``, but you can also use a ``virtualenv`` of your own creation. .. _repo: https://github.com/GoogleCloudPlatform/google-cloud-python @@ -68,11 +68,15 @@ Using a custom ``virtualenv`` - To create a virtualenv in which to install ``google-cloud-python``:: $ cd ${HOME}/hack-on-google-cloud-python - $ virtualenv --python python2.7 ${ENV_NAME} + $ virtualenv --python python3.6 ${ENV_NAME} You can choose which Python version you want to use by passing a ``--python`` - flag to ``virtualenv``. For example, ``virtualenv --python python2.7`` - chooses the Python 2.7 interpreter to be installed. + flag to ``virtualenv``. For example, ``virtualenv --python python3.6`` + chooses the Python 3.6 interpreter to be installed. + + .. note:: + We recommend developing in Python 3, and using the test suite to + ensure compatibility with Python 2. - From here on in within these instructions, the ``${HOME}/hack-on-google-cloud-python/${ENV_NAME}`` virtual environment you @@ -91,43 +95,32 @@ Using a custom ``virtualenv`` Unfortunately using ``setup.py develop`` is not possible with this project, because it uses `namespace packages`_. -Using ``tox`` +Using ``nox`` ============= -- To test your changes, run unit tests with ``tox``:: - - $ tox -e py27 - $ tox -e py34 - $ ... - -- If you'd like to poke around your code in an interpreter, let - ``tox`` install the environment of your choice:: - - $ # Install only; without running tests - $ tox -e ${ENV} --recreate --notest +We use `nox`_ to instrument our tests. - After doing this, you can activate the virtual environment and - use the interpreter from that environment:: +- To test your changes, run unit tests with ``nox``:: - $ source .tox/${ENV}/bin/activate - (ENV) $ .tox/${ENV}/bin/python + $ nox -f datastore/nox.py -s "unit_tests(python_version='2.7')" + $ nox -f datastore/nox.py -s "unit_tests(python_version='3.4')" + $ ... - Unfortunately, your changes to the source tree won't be picked up - by the ``tox`` environment, so if you make changes, you'll need - to again ``--recreate`` the environment. + .. note:: -- To run unit tests on a restricted set of packages:: + The unit tests and system tests are contained in the individual + ``nox.py`` files in each directory; substitute ``datastore`` in the + example above with the package of your choice. - $ tox -e py27 -- core datastore Alternatively, you can just navigate directly to the package you are currently developing and run tests there:: $ export GIT_ROOT=$(pwd) - $ cd ${GIT_ROOT}/core/ - $ tox -e py27 $ cd ${GIT_ROOT}/datastore/ - $ tox -e py27 + $ nox -s "unit_tests(python_version='3.6')" + +.. nox: https://pypi.org/project/nox-automation/ Note on Editable Installs / Develop Mode ======================================== @@ -162,13 +155,13 @@ On Debian/Ubuntu:: Coding Style ************ -- PEP8 compliance, with exceptions defined in ``tox.ini``. - If you have ``tox`` installed, you can test that you have not introduced +- PEP8 compliance, with exceptions defined in the linter configuration. + If you have ``nox`` installed, you can test that you have not introduced any non-compliant code via:: - $ tox -e lint + $ nox -s lint -- In order to make ``tox -e lint`` run faster, you can set some environment +- In order to make ``nox -s lint`` run faster, you can set some environment variables:: export GOOGLE_CLOUD_TESTING_REMOTE="upstream" @@ -185,49 +178,20 @@ Exceptions to PEP8: "Function-Under-Test"), which is PEP8-incompliant, but more readable. Some also use a local variable, ``MUT`` (short for "Module-Under-Test"). -************* -Running Tests -************* - -- To run all tests for ``google-cloud-python`` on a single Python version, run - ``py.test`` from your development virtualenv (See - `Using a Development Checkout`_ above). - -.. _Using a Development Checkout: #using-a-development-checkout - -- To run the full set of ``google-cloud-python`` tests on all platforms, install - ``tox`` (https://tox.readthedocs.io/en/latest/) into a system Python. The - ``tox`` console script will be installed into the scripts location for that - Python. While ``cd``'-ed to the ``google-cloud-python`` checkout root - directory (it contains ``tox.ini``), invoke the ``tox`` console script. - This will read the ``tox.ini`` file and execute the tests on multiple - Python versions and platforms; while it runs, it creates a ``virtualenv`` for - each version/platform combination. For example:: - - $ sudo --set-home /usr/bin/pip install tox - $ cd ${HOME}/hack-on-google-cloud-python/ - $ /usr/bin/tox - -.. _Using a Development Checkout: #using-a-development-checkout - ******************** Running System Tests ******************** -- To run system tests you can execute:: - - $ tox -e system-tests - $ tox -e system-tests3 - - or run only system tests for a particular package via:: +- To run system tests for a given package, you can execute:: - $ python system_tests/run_system_test.py --package {package} - $ python3 system_tests/run_system_test.py --package {package} + $ nox -f datastore/nox.py -s "system_tests(python_version='3.6')" + $ nox -f datastore/nox.py -s "system_tests(python_version='2.7')" - To run a subset of the system tests:: + .. note:: - $ tox -e system-tests -- datastore storage - $ python system_tests/attempt_system_tests.py datastore storage + System tests are only configured to run under Python 2.7 and + Python 3.6. For expediency, we do not run them in older versions + of Python 3. This alone will not run the tests. You'll need to change some local auth settings and change some configuration in your project to @@ -270,90 +234,21 @@ Running System Tests - For datastore query tests, you'll need stored data in your dataset. To populate this data, run:: - $ python system_tests/populate_datastore.py + $ python datastore/tests/system/utils/populate_datastore.py - If you make a mistake during development (i.e. a failing test that prevents clean-up) you can clear all system test data from your datastore instance via:: - $ python system_tests/clear_datastore.py - -System Test Emulators -===================== - -- System tests can also be run against local `emulators`_ that mock - the production services. To run the system tests with the - ``datastore`` emulator:: - - $ tox -e datastore-emulator - $ GOOGLE_CLOUD_DISABLE_GRPC=true tox -e datastore-emulator - - This also requires that the ``gcloud`` command line tool is - installed. If you'd like to run them directly (outside of a - ``tox`` environment), first start the emulator and - take note of the process ID:: - - $ gcloud beta emulators datastore start --no-legacy 2>&1 > log.txt & - [1] 33333 + $ python datastore/tests/system/utils/clear_datastore.py - then determine the environment variables needed to interact with - the emulator:: - - $ gcloud beta emulators datastore env-init - export DATASTORE_LOCAL_HOST=localhost:8417 - export DATASTORE_HOST=http://localhost:8417 - export DATASTORE_DATASET=google-cloud-settings-app-id - export DATASTORE_PROJECT_ID=google-cloud-settings-app-id - - using these environment variables run the emulator:: - - $ DATASTORE_HOST=http://localhost:8471 \ - > DATASTORE_DATASET=google-cloud-settings-app-id \ - > GOOGLE_CLOUD_NO_PRINT=true \ - > python system_tests/run_system_test.py \ - > --package=datastore --ignore-requirements - - and after completion stop the emulator and any child - processes it spawned:: - - $ kill -- -33333 - -.. _emulators: https://cloud.google.com/sdk/gcloud/reference/beta/emulators/ - -- To run the system tests with the ``pubsub`` emulator:: - - $ tox -e pubsub-emulator - $ GOOGLE_CLOUD_DISABLE_GRPC=true tox -e pubsub-emulator - - If you'd like to run them directly (outside of a ``tox`` environment), first - start the emulator and take note of the process ID:: - - $ gcloud beta emulators pubsub start 2>&1 > log.txt & - [1] 44444 - - then determine the environment variables needed to interact with - the emulator:: - - $ gcloud beta emulators pubsub env-init - export PUBSUB_EMULATOR_HOST=localhost:8897 - - using these environment variables run the emulator:: - - $ PUBSUB_EMULATOR_HOST=localhost:8897 \ - > python system_tests/run_system_test.py \ - > --package=pubsub - - and after completion stop the emulator and any child - processes it spawned:: - - $ kill -- -44444 ************* Test Coverage ************* - The codebase *must* have 100% test statement coverage after each commit. - You can test coverage via ``tox -e cover``. + You can test coverage via ``nox -s cover``. ****************************************************** Documentation Coverage and Building HTML Documentation @@ -386,10 +281,10 @@ using to develop ``google-cloud-python``): #. Open the ``docs/_build/html/index.html`` file to see the resulting HTML rendering. -As an alternative to 1. and 2. above, if you have ``tox`` installed, you +As an alternative to 1. and 2. above, if you have ``nox`` installed, you can build the docs via:: - $ tox -e docs + $ nox -s docs ******************************************** Note About ``README`` as it pertains to PyPI @@ -404,27 +299,15 @@ may cause problems creating links or rendering the description. .. _description on PyPI: https://pypi.org/project/google-cloud/ -******************************************** -Travis Configuration and Build Optimizations -******************************************** +********************** +CircleCI Configuration +********************** -All build scripts in the ``.travis.yml`` configuration file which have -Python dependencies are specified in the ``tox.ini`` configuration. -They are executed in the Travis build via ``tox -e ${ENV}`` where +All build scripts in the ``.circleci/config.yml`` configuration file which have +Python dependencies are specified in the ``nox.py`` configuration. +They are executed in the Travis build via ``nox -s ${ENV}`` where ``${ENV}`` is the environment being tested. -If new ``tox`` environments are added to be run in a Travis build, they -should be listed in ``[tox].envlist`` as a default environment. - -We speed up builds by using the Travis `caching feature`_. - -.. _caching feature: https://docs.travis-ci.com/user/caching/#pip-cache - -We intentionally **do not** cache the ``.tox/`` directory. Instead, we -allow the ``tox`` environments to be re-built for every build. This -way, we'll always get the latest versions of our dependencies and any -caching or wheel optimization to be done will be handled automatically -by ``pip``. ************************* Supported Python Versions @@ -435,14 +318,16 @@ We support: - `Python 2.7`_ - `Python 3.4`_ - `Python 3.5`_ +- `Python 3.6`_ .. _Python 2.7: https://docs.python.org/2.7/ .. _Python 3.4: https://docs.python.org/3.4/ .. _Python 3.5: https://docs.python.org/3.5/ +.. _Python 3.6: https://docs.python.org/3.6/ -Supported versions can be found in our ``tox.ini`` `config`_. +Supported versions can be found in our ``nox.py`` `config`_. -.. _config: https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/tox.ini +.. _config: https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/nox.py We explicitly decided not to support `Python 2.5`_ due to `decreased usage`_ and lack of continuous integration `support`_. @@ -475,17 +360,23 @@ This library follows `Semantic Versioning`_. .. _Semantic Versioning: http://semver.org/ -It is currently in major version zero (``0.y.z``), which means that anything -may change at any time and the public API should not be considered +Some packages are currently in major version zero (``0.y.z``), which means that +anything may change at any time and the public API should not be considered stable. ****************************** Contributor License Agreements ****************************** -Before we can accept your pull requests you'll need to sign a Contributor License Agreement (CLA): +Before we can accept your pull requests you'll need to sign a Contributor +License Agreement (CLA): -- **If you are an individual writing original source code** and **you own the intellectual property**, then you'll need to sign an `individual CLA `__. -- **If you work for a company that wants to allow you to contribute your work**, then you'll need to sign a `corporate CLA `__. +- **If you are an individual writing original source code** and **you own the + intellectual property**, then you'll need to sign an + `individual CLA `__. +- **If you work for a company that wants to allow you to contribute your work**, + then you'll need to sign a + `corporate CLA `__. -You can sign these electronically (just scroll to the bottom). After that, we'll be able to accept your pull requests. +You can sign these electronically (just scroll to the bottom). After that, +we'll be able to accept your pull requests. From fe757be48c29ad325a7f89c04f97e022a0f1fbea Mon Sep 17 00:00:00 2001 From: Luke Sneeringer Date: Tue, 8 Aug 2017 10:05:34 -0700 Subject: [PATCH 03/10] session.run_in_transaction returns the callback's return value. (#3753) --- spanner/google/cloud/spanner/session.py | 10 +++++----- spanner/tests/unit/test_database.py | 4 ++-- spanner/tests/unit/test_session.py | 24 +++++++++--------------- 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/spanner/google/cloud/spanner/session.py b/spanner/google/cloud/spanner/session.py index 953ab62993cc..04fcacea38ee 100644 --- a/spanner/google/cloud/spanner/session.py +++ b/spanner/google/cloud/spanner/session.py @@ -268,8 +268,9 @@ def run_in_transaction(self, func, *args, **kw): If passed, "timeout_secs" will be removed and used to override the default timeout. - :rtype: :class:`datetime.datetime` - :returns: timestamp of committed transaction + :rtype: Any + :returns: The return value of ``func``. + :raises Exception: reraises any non-ABORT execptions raised by ``func``. """ @@ -284,7 +285,7 @@ def run_in_transaction(self, func, *args, **kw): if txn._transaction_id is None: txn.begin() try: - func(txn, *args, **kw) + return_value = func(txn, *args, **kw) except GaxError as exc: _delay_until_retry(exc, deadline) del self._transaction @@ -299,8 +300,7 @@ def run_in_transaction(self, func, *args, **kw): _delay_until_retry(exc, deadline) del self._transaction else: - committed = txn.committed - return committed + return return_value # pylint: disable=misplaced-bare-raise diff --git a/spanner/tests/unit/test_database.py b/spanner/tests/unit/test_database.py index ec94e0198c77..40e10ec971a9 100644 --- a/spanner/tests/unit/test_database.py +++ b/spanner/tests/unit/test_database.py @@ -22,7 +22,7 @@ from google.cloud.spanner import __version__ -def _make_credentials(): +def _make_credentials(): # pragma: NO COVER import google.auth.credentials class _CredentialsWithScopes( @@ -223,7 +223,7 @@ def __init__(self, scopes=(), source=None): self._scopes = scopes self._source = source - def requires_scopes(self): + def requires_scopes(self): # pragma: NO COVER return True def with_scopes(self, scopes): diff --git a/spanner/tests/unit/test_session.py b/spanner/tests/unit/test_session.py index 100555c8e49f..826369079d29 100644 --- a/spanner/tests/unit/test_session.py +++ b/spanner/tests/unit/test_session.py @@ -513,16 +513,16 @@ def test_run_in_transaction_w_args_w_kwargs_wo_abort(self): def unit_of_work(txn, *args, **kw): called_with.append((txn, args, kw)) txn.insert(TABLE_NAME, COLUMNS, VALUES) + return 42 - committed = session.run_in_transaction( + return_value = session.run_in_transaction( unit_of_work, 'abc', some_arg='def') - self.assertEqual(committed, now) self.assertIsNone(session._transaction) self.assertEqual(len(called_with), 1) txn, args, kw = called_with[0] self.assertIsInstance(txn, Transaction) - self.assertEqual(txn.committed, committed) + self.assertEqual(return_value, 42) self.assertEqual(args, ('abc',)) self.assertEqual(kw, {'some_arg': 'def'}) @@ -561,18 +561,15 @@ def test_run_in_transaction_w_abort_no_retry_metadata(self): def unit_of_work(txn, *args, **kw): called_with.append((txn, args, kw)) txn.insert(TABLE_NAME, COLUMNS, VALUES) + return 'answer' - committed = session.run_in_transaction( + return_value = session.run_in_transaction( unit_of_work, 'abc', some_arg='def') - self.assertEqual(committed, now) self.assertEqual(len(called_with), 2) for index, (txn, args, kw) in enumerate(called_with): self.assertIsInstance(txn, Transaction) - if index == 1: - self.assertEqual(txn.committed, committed) - else: - self.assertIsNone(txn.committed) + self.assertEqual(return_value, 'answer') self.assertEqual(args, ('abc',)) self.assertEqual(kw, {'some_arg': 'def'}) @@ -621,17 +618,15 @@ def unit_of_work(txn, *args, **kw): time_module = _FauxTimeModule() with _Monkey(MUT, time=time_module): - committed = session.run_in_transaction( - unit_of_work, 'abc', some_arg='def') + session.run_in_transaction(unit_of_work, 'abc', some_arg='def') self.assertEqual(time_module._slept, RETRY_SECONDS + RETRY_NANOS / 1.0e9) - self.assertEqual(committed, now) self.assertEqual(len(called_with), 2) for index, (txn, args, kw) in enumerate(called_with): self.assertIsInstance(txn, Transaction) if index == 1: - self.assertEqual(txn.committed, committed) + self.assertEqual(txn.committed, now) else: self.assertIsNone(txn.committed) self.assertEqual(args, ('abc',)) @@ -688,9 +683,8 @@ def unit_of_work(txn, *args, **kw): time_module = _FauxTimeModule() with _Monkey(MUT, time=time_module): - committed = session.run_in_transaction(unit_of_work) + session.run_in_transaction(unit_of_work) - self.assertEqual(committed, now) self.assertEqual(time_module._slept, RETRY_SECONDS + RETRY_NANOS / 1.0e9) self.assertEqual(len(called_with), 2) From abfec7008e6040136a16a3a0f7dc72b5579c721e Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Tue, 8 Aug 2017 10:32:00 -0700 Subject: [PATCH 04/10] Add google.api.core.exceptions (#3738) * Add google.api.core.exceptions * Add google.api.core to coverage report * Alias google.cloud.exceptions to google.api.core.exceptions * Fix lint * Address review comments * Fix typo --- core/google/api/core/exceptions.py | 420 ++++++++++++++++++++ core/google/cloud/exceptions.py | 254 ++---------- core/google/cloud/obselete.py | 2 + core/nox.py | 1 + core/tests/unit/api_core/__init__.py | 0 core/tests/unit/api_core/test_exceptions.py | 201 ++++++++++ 6 files changed, 655 insertions(+), 223 deletions(-) create mode 100644 core/google/api/core/exceptions.py create mode 100644 core/tests/unit/api_core/__init__.py create mode 100644 core/tests/unit/api_core/test_exceptions.py diff --git a/core/google/api/core/exceptions.py b/core/google/api/core/exceptions.py new file mode 100644 index 000000000000..c25816abce34 --- /dev/null +++ b/core/google/api/core/exceptions.py @@ -0,0 +1,420 @@ +# Copyright 2014 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exceptions raised by Google API core & clients. + +This module provides base classes for all errors raised by libraries based +on :mod:`google.api.core`, including both HTTP and gRPC clients. +""" + +from __future__ import absolute_import +from __future__ import unicode_literals + +import six +from six.moves import http_client + +try: + import grpc +except ImportError: # pragma: NO COVER + grpc = None + +# Lookup tables for mapping exceptions from HTTP and gRPC transports. +# Populated by _APICallErrorMeta +_HTTP_CODE_TO_EXCEPTION = {} +_GRPC_CODE_TO_EXCEPTION = {} + + +class GoogleAPIError(Exception): + """Base class for all exceptions raised by Google API Clients.""" + pass + + +class _GoogleAPICallErrorMeta(type): + """Metaclass for registering GoogleAPICallError subclasses.""" + def __new__(mcs, name, bases, class_dict): + cls = type.__new__(mcs, name, bases, class_dict) + if cls.code is not None: + _HTTP_CODE_TO_EXCEPTION.setdefault(cls.code, cls) + if cls.grpc_status_code is not None: + _GRPC_CODE_TO_EXCEPTION.setdefault(cls.grpc_status_code, cls) + return cls + + +@six.python_2_unicode_compatible +@six.add_metaclass(_GoogleAPICallErrorMeta) +class GoogleAPICallError(GoogleAPIError): + """Base class for exceptions raised by calling API methods. + + Args: + message (str): The exception message. + errors (Sequence[Any]): An optional list of error details. + response (Union[requests.Request, grpc.Call]): The response or + gRPC call metadata. + """ + + code = None + """Optional[int]: The HTTP status code associated with this error. + + This may be ``None`` if the exception does not have a direct mapping + to an HTTP error. + + See http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html + """ + + grpc_status_code = None + """Optional[grpc.StatusCode]: The gRPC status code associated with this + error. + + This may be ``None`` if the exception does not match up to a gRPC error. + """ + + def __init__(self, message, errors=(), response=None): + super(GoogleAPICallError, self).__init__(message) + self.message = message + """str: The exception message.""" + self._errors = errors + self._response = response + + def __str__(self): + return '{} {}'.format(self.code, self.message) + + @property + def errors(self): + """Detailed error information. + + Returns: + Sequence[Any]: A list of additional error details. + """ + return list(self._errors) + + @property + def response(self): + """Optional[Union[requests.Request, grpc.Call]]: The response or + gRPC call metadata.""" + return self._response + + +class Redirection(GoogleAPICallError): + """Base class for for all redirection (HTTP 3xx) responses.""" + + +class MovedPermanently(Redirection): + """Exception mapping a ``301 Moved Permanently`` response.""" + code = http_client.MOVED_PERMANENTLY + + +class NotModified(Redirection): + """Exception mapping a ``304 Not Modified`` response.""" + code = http_client.NOT_MODIFIED + + +class TemporaryRedirect(Redirection): + """Exception mapping a ``307 Temporary Redirect`` response.""" + code = http_client.TEMPORARY_REDIRECT + + +class ResumeIncomplete(Redirection): + """Exception mapping a ``308 Resume Incomplete`` response. + + .. note:: :ref:`http_client.PERMANENT_REDIRECT` is ``308``, but Google APIs + differ in their use of this status code. + """ + code = 308 + + +class ClientError(GoogleAPICallError): + """Base class for all client error (HTTP 4xx) responses.""" + + +class BadRequest(ClientError): + """Exception mapping a ``400 Bad Request`` response.""" + code = http_client.BAD_REQUEST + + +class InvalidArgument(BadRequest): + """Exception mapping a :prop:`grpc.StatusCode.INVALID_ARGUMENT` error.""" + grpc_status_code = ( + grpc.StatusCode.INVALID_ARGUMENT if grpc is not None else None) + + +class FailedPrecondition(BadRequest): + """Exception mapping a :prop:`grpc.StatusCode.FAILED_PRECONDITION` + error.""" + grpc_status_code = ( + grpc.StatusCode.FAILED_PRECONDITION if grpc is not None else None) + + +class OutOfRange(BadRequest): + """Exception mapping a :prop:`grpc.StatusCode.OUT_OF_RANGE` error.""" + grpc_status_code = ( + grpc.StatusCode.OUT_OF_RANGE if grpc is not None else None) + + +class Unauthorized(ClientError): + """Exception mapping a ``401 Unauthorized`` response.""" + code = http_client.UNAUTHORIZED + + +class Unauthenticated(Unauthorized): + """Exception mapping a :prop:`grpc.StatusCode.UNAUTHENTICATED` error.""" + grpc_status_code = ( + grpc.StatusCode.UNAUTHENTICATED if grpc is not None else None) + + +class Forbidden(ClientError): + """Exception mapping a ``403 Forbidden`` response.""" + code = http_client.FORBIDDEN + + +class PermissionDenied(Forbidden): + """Exception mapping a :prop:`grpc.StatusCode.PERMISSION_DENIED` error.""" + grpc_status_code = ( + grpc.StatusCode.PERMISSION_DENIED if grpc is not None else None) + + +class NotFound(ClientError): + """Exception mapping a ``404 Not Found`` response or a + :prop:`grpc.StatusCode.NOT_FOUND` error.""" + code = http_client.NOT_FOUND + grpc_status_code = ( + grpc.StatusCode.NOT_FOUND if grpc is not None else None) + + +class MethodNotAllowed(ClientError): + """Exception mapping a ``405 Method Not Allowed`` response.""" + code = http_client.METHOD_NOT_ALLOWED + + +class Conflict(ClientError): + """Exception mapping a ``409 Conflict`` response.""" + code = http_client.CONFLICT + + +class AlreadyExists(Conflict): + """Exception mapping a :prop:`grpc.StatusCode.ALREADY_EXISTS` error.""" + grpc_status_code = ( + grpc.StatusCode.ALREADY_EXISTS if grpc is not None else None) + + +class Aborted(Conflict): + """Exception mapping a :prop:`grpc.StatusCode.ABORTED` error.""" + grpc_status_code = ( + grpc.StatusCode.ABORTED if grpc is not None else None) + + +class LengthRequired(ClientError): + """Exception mapping a ``411 Length Required`` response.""" + code = http_client.LENGTH_REQUIRED + + +class PreconditionFailed(ClientError): + """Exception mapping a ``412 Precondition Failed`` response.""" + code = http_client.PRECONDITION_FAILED + + +class RequestRangeNotSatisfiable(ClientError): + """Exception mapping a ``416 Request Range Not Satisfiable`` response.""" + code = http_client.REQUESTED_RANGE_NOT_SATISFIABLE + + +class TooManyRequests(ClientError): + """Exception mapping a ``429 Too Many Requests`` response.""" + # http_client does not define a constant for this in Python 2. + code = 429 + + +class ResourceExhausted(TooManyRequests): + """Exception mapping a :prop:`grpc.StatusCode.RESOURCE_EXHAUSTED` error.""" + grpc_status_code = ( + grpc.StatusCode.RESOURCE_EXHAUSTED if grpc is not None else None) + + +class Cancelled(ClientError): + """Exception mapping a :prop:`grpc.StatusCode.CANCELLED` error.""" + # This maps to HTTP status code 499. See + # https://github.com/googleapis/googleapis/blob/master/google/rpc\ + # /code.proto + code = 499 + grpc_status_code = grpc.StatusCode.CANCELLED if grpc is not None else None + + +class ServerError(GoogleAPICallError): + """Base for 5xx responses.""" + + +class InternalServerError(ServerError): + """Exception mapping a ``500 Internal Server Error`` response. or a + :prop:`grpc.StatusCode.INTERNAL` error.""" + code = http_client.INTERNAL_SERVER_ERROR + grpc_status_code = grpc.StatusCode.INTERNAL if grpc is not None else None + + +class Unknown(ServerError): + """Exception mapping a :prop:`grpc.StatusCode.UNKNOWN` error.""" + grpc_status_code = grpc.StatusCode.UNKNOWN if grpc is not None else None + + +class DataLoss(ServerError): + """Exception mapping a :prop:`grpc.StatusCode.DATA_LOSS` error.""" + grpc_status_code = grpc.StatusCode.DATA_LOSS if grpc is not None else None + + +class MethodNotImplemented(ServerError): + """Exception mapping a ``501 Not Implemented`` response or a + :prop:`grpc.StatusCode.UNIMPLEMENTED` error.""" + code = http_client.NOT_IMPLEMENTED + grpc_status_code = ( + grpc.StatusCode.UNIMPLEMENTED if grpc is not None else None) + + +class BadGateway(ServerError): + """Exception mapping a ``502 Bad Gateway`` response.""" + code = http_client.BAD_GATEWAY + + +class ServiceUnavailable(ServerError): + """Exception mapping a ``503 Service Unavailable`` response or a + :prop:`grpc.StatusCode.UNAVAILABLE` error.""" + code = http_client.SERVICE_UNAVAILABLE + grpc_status_code = ( + grpc.StatusCode.UNAVAILABLE if grpc is not None else None) + + +class GatewayTimeout(ServerError): + """Exception mapping a ``504 Gateway Timeout`` response.""" + code = http_client.GATEWAY_TIMEOUT + + +class DeadlineExceeded(GatewayTimeout): + """Exception mapping a :prop:`grpc.StatusCode.DEADLINE_EXCEEDED` error.""" + grpc_status_code = ( + grpc.StatusCode.DEADLINE_EXCEEDED if grpc is not None else None) + + +def exception_class_for_http_status(status_code): + """Return the exception class for a specific HTTP status code. + + Args: + status_code (int): The HTTP status code. + + Returns: + type: the appropriate subclass of :class:`GoogleAPICallError`. + """ + return _HTTP_CODE_TO_EXCEPTION.get(status_code, GoogleAPICallError) + + +def from_http_status(status_code, message, **kwargs): + """Create a :class:`GoogleAPICallError` from an HTTP status code. + + Args: + status_code (int): The HTTP status code. + message (str): The exception message. + kwargs: Additional arguments passed to the :class:`GoogleAPICallError` + constructor. + + Returns: + GoogleAPICallError: An instance of the appropriate subclass of + :class:`GoogleAPICallError`. + """ + error_class = exception_class_for_http_status(status_code) + error = error_class(message, **kwargs) + + if error.code is None: + error.code = status_code + + return error + + +def from_http_response(response): + """Create a :class:`GoogleAPICallError` from a :class:`requests.Response`. + + Args: + response (requests.Response): The HTTP response. + + Returns: + GoogleAPICallError: An instance of the appropriate subclass of + :class:`GoogleAPICallError`, with the message and errors populated + from the response. + """ + try: + payload = response.json() + except ValueError: + payload = {'error': {'message': response.text or 'unknown error'}} + + error_message = payload.get('error', {}).get('message', 'unknown error') + errors = payload.get('error', {}).get('errors', ()) + + message = '{method} {url}: {error}'.format( + method=response.request.method, + url=response.request.url, + error=error_message) + + exception = from_http_status( + response.status_code, message, errors=errors, response=response) + return exception + + +def exception_class_for_grpc_status(status_code): + """Return the exception class for a specific :class:`grpc.StatusCode`. + + Args: + status_code (grpc.StatusCode): The gRPC status code. + + Returns: + type: the appropriate subclass of :class:`GoogleAPICallError`. + """ + return _GRPC_CODE_TO_EXCEPTION.get(status_code, GoogleAPICallError) + + +def from_grpc_status(status_code, message, **kwargs): + """Create a :class:`GoogleAPICallError` from a :class:`grpc.StatusCode`. + + Args: + status_code (grpc.StatusCode): The gRPC status code. + message (str): The exception message. + kwargs: Additional arguments passed to the :class:`GoogleAPICallError` + constructor. + + Returns: + GoogleAPICallError: An instance of the appropriate subclass of + :class:`GoogleAPICallError`. + """ + error_class = exception_class_for_grpc_status(status_code) + error = error_class(message, **kwargs) + + if error.grpc_status_code is None: + error.grpc_status_code = status_code + + return error + + +def from_grpc_error(rpc_exc): + """Create a :class:`GoogleAPICallError` from a :class:`grpc.RpcError`. + + Args: + rpc_exc (grpc.RpcError): The gRPC error. + + Returns: + GoogleAPICallError: An instance of the appropriate subclass of + :class:`GoogleAPICallError`. + """ + if isinstance(rpc_exc, grpc.Call): + return from_grpc_status( + rpc_exc.code(), + rpc_exc.details(), + errors=(rpc_exc,), + response=rpc_exc) + else: + return GoogleAPICallError( + str(rpc_exc), errors=(rpc_exc,), response=rpc_exc) diff --git a/core/google/cloud/exceptions.py b/core/google/cloud/exceptions.py index 2e7eca3be98d..a5d82be30452 100644 --- a/core/google/cloud/exceptions.py +++ b/core/google/cloud/exceptions.py @@ -12,240 +12,48 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Custom exceptions for :mod:`google.cloud` package. +# pylint: disable=invalid-name +# pylint recognizies all of these aliases as constants and thinks they have +# invalid names. -See https://cloud.google.com/storage/docs/json_api/v1/status-codes -""" +"""Custom exceptions for :mod:`google.cloud` package.""" # Avoid the grpc and google.cloud.grpc collision. from __future__ import absolute_import -import copy - -import six - -from google.cloud._helpers import _to_bytes +from google.api.core import exceptions try: from grpc._channel import _Rendezvous except ImportError: # pragma: NO COVER _Rendezvous = None -_HTTP_CODE_TO_EXCEPTION = {} # populated at end of module - - -# pylint: disable=invalid-name GrpcRendezvous = _Rendezvous """Exception class raised by gRPC stable.""" -# pylint: enable=invalid-name - - -class GoogleCloudError(Exception): - """Base error class for Google Cloud errors (abstract). - - Each subclass represents a single type of HTTP error response. - """ - code = None - """HTTP status code. Concrete subclasses *must* define. - - See http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html - """ - - def __init__(self, message, errors=()): - super(GoogleCloudError, self).__init__(message) - self.message = message - self._errors = errors - - def __str__(self): - result = u'%d %s' % (self.code, self.message) - if six.PY2: - result = _to_bytes(result, 'utf-8') - return result - - @property - def errors(self): - """Detailed error information. - - :rtype: list(dict) - :returns: a list of mappings describing each error. - """ - return [copy.deepcopy(error) for error in self._errors] - - -class Redirection(GoogleCloudError): - """Base for 3xx responses - - This class is abstract. - """ - - -class MovedPermanently(Redirection): - """Exception mapping a '301 Moved Permanently' response.""" - code = 301 - - -class NotModified(Redirection): - """Exception mapping a '304 Not Modified' response.""" - code = 304 - - -class TemporaryRedirect(Redirection): - """Exception mapping a '307 Temporary Redirect' response.""" - code = 307 - - -class ResumeIncomplete(Redirection): - """Exception mapping a '308 Resume Incomplete' response.""" - code = 308 - - -class ClientError(GoogleCloudError): - """Base for 4xx responses - - This class is abstract - """ - - -class BadRequest(ClientError): - """Exception mapping a '400 Bad Request' response.""" - code = 400 - - -class Unauthorized(ClientError): - """Exception mapping a '401 Unauthorized' response.""" - code = 401 - - -class Forbidden(ClientError): - """Exception mapping a '403 Forbidden' response.""" - code = 403 - - -class NotFound(ClientError): - """Exception mapping a '404 Not Found' response.""" - code = 404 - - -class MethodNotAllowed(ClientError): - """Exception mapping a '405 Method Not Allowed' response.""" - code = 405 - - -class Conflict(ClientError): - """Exception mapping a '409 Conflict' response.""" - code = 409 - - -class LengthRequired(ClientError): - """Exception mapping a '411 Length Required' response.""" - code = 411 - - -class PreconditionFailed(ClientError): - """Exception mapping a '412 Precondition Failed' response.""" - code = 412 - - -class RequestRangeNotSatisfiable(ClientError): - """Exception mapping a '416 Request Range Not Satisfiable' response.""" - code = 416 - - -class TooManyRequests(ClientError): - """Exception mapping a '429 Too Many Requests' response.""" - code = 429 - - -class ServerError(GoogleCloudError): - """Base for 5xx responses: (abstract)""" - - -class InternalServerError(ServerError): - """Exception mapping a '500 Internal Server Error' response.""" - code = 500 - - -class MethodNotImplemented(ServerError): - """Exception mapping a '501 Not Implemented' response.""" - code = 501 - - -class BadGateway(ServerError): - """Exception mapping a '502 Bad Gateway' response.""" - code = 502 - - -class ServiceUnavailable(ServerError): - """Exception mapping a '503 Service Unavailable' response.""" - code = 503 - - -class GatewayTimeout(ServerError): - """Exception mapping a `504 Gateway Timeout'` response.""" - code = 504 - - -def from_http_status(status_code, message, errors=()): - """Create a :class:`GoogleCloudError` from an HTTP status code. - - Args: - status_code (int): The HTTP status code. - message (str): The exception message. - errors (Sequence[Any]): A list of additional error information. - - Returns: - GoogleCloudError: An instance of the appropriate subclass of - :class:`GoogleCloudError`. - """ - error_class = _HTTP_CODE_TO_EXCEPTION.get(status_code, GoogleCloudError) - error = error_class(message, errors) - - if error.code is None: - error.code = status_code - - return error - - -def from_http_response(response): - """Create a :class:`GoogleCloudError` from a :class:`requests.Response`. - - Args: - response (requests.Response): The HTTP response. - - Returns: - GoogleCloudError: An instance of the appropriate subclass of - :class:`GoogleCloudError`, with the message and errors populated - from the response. - """ - try: - payload = response.json() - except ValueError: - payload = {'error': {'message': response.text or 'unknown error'}} - - error_message = payload.get('error', {}).get('message', 'unknown error') - errors = payload.get('error', {}).get('errors', ()) - - message = '{method} {url}: {error}'.format( - method=response.request.method, - url=response.request.url, - error=error_message) - - exception = from_http_status( - response.status_code, message, errors=errors) - exception.response = response - return exception - - -def _walk_subclasses(klass): - """Recursively walk subclass tree.""" - for sub in klass.__subclasses__(): - yield sub - for subsub in _walk_subclasses(sub): - yield subsub - -# Build the code->exception class mapping. -for _eklass in _walk_subclasses(GoogleCloudError): - code = getattr(_eklass, 'code', None) - if code is not None: - _HTTP_CODE_TO_EXCEPTION[code] = _eklass +# Aliases to moved classes. +GoogleCloudError = exceptions.GoogleAPICallError +Redirection = exceptions.Redirection +MovedPermanently = exceptions.MovedPermanently +NotModified = exceptions.NotModified +TemporaryRedirect = exceptions.TemporaryRedirect +ResumeIncomplete = exceptions.ResumeIncomplete +ClientError = exceptions.ClientError +BadRequest = exceptions.BadRequest +Unauthorized = exceptions.Unauthorized +Forbidden = exceptions.Forbidden +NotFound = exceptions.NotFound +MethodNotAllowed = exceptions.MethodNotAllowed +Conflict = exceptions.Conflict +LengthRequired = exceptions.LengthRequired +PreconditionFailed = exceptions.PreconditionFailed +RequestRangeNotSatisfiable = exceptions.RequestRangeNotSatisfiable +TooManyRequests = exceptions.TooManyRequests +ServerError = exceptions.ServerError +InternalServerError = exceptions.InternalServerError +MethodNotImplemented = exceptions.MethodNotImplemented +BadGateway = exceptions.BadGateway +ServiceUnavailable = exceptions.ServiceUnavailable +GatewayTimeout = exceptions.GatewayTimeout +from_http_status = exceptions.from_http_status +from_http_response = exceptions.from_http_response diff --git a/core/google/cloud/obselete.py b/core/google/cloud/obselete.py index 9af28cd85d52..cd70025946f7 100644 --- a/core/google/cloud/obselete.py +++ b/core/google/cloud/obselete.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Helpers for deprecated code and modules.""" + import warnings import pkg_resources diff --git a/core/nox.py b/core/nox.py index 1dca10eb9b69..b795ddfce7a6 100644 --- a/core/nox.py +++ b/core/nox.py @@ -43,6 +43,7 @@ def unit_tests(session, python_version): 'py.test', '--quiet', '--cov=google.cloud', + '--cov=google.api.core', '--cov=tests.unit', '--cov-append', '--cov-config=.coveragerc', diff --git a/core/tests/unit/api_core/__init__.py b/core/tests/unit/api_core/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/core/tests/unit/api_core/test_exceptions.py b/core/tests/unit/api_core/test_exceptions.py new file mode 100644 index 000000000000..f29873e7b3d8 --- /dev/null +++ b/core/tests/unit/api_core/test_exceptions.py @@ -0,0 +1,201 @@ +# Copyright 2014 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import grpc +import mock +import requests +from six.moves import http_client + +from google.api.core import exceptions + + +def test_create_google_cloud_error(): + exception = exceptions.GoogleAPICallError('Testing') + exception.code = 600 + assert str(exception) == '600 Testing' + assert exception.message == 'Testing' + assert exception.errors == [] + assert exception.response is None + + +def test_create_google_cloud_error_with_args(): + error = { + 'domain': 'global', + 'location': 'test', + 'locationType': 'testing', + 'message': 'Testing', + 'reason': 'test', + } + response = mock.sentinel.response + exception = exceptions.GoogleAPICallError( + 'Testing', [error], response=response) + exception.code = 600 + assert str(exception) == '600 Testing' + assert exception.message == 'Testing' + assert exception.errors == [error] + assert exception.response == response + + +def test_from_http_status(): + message = 'message' + exception = exceptions.from_http_status(http_client.NOT_FOUND, message) + assert exception.code == http_client.NOT_FOUND + assert exception.message == message + assert exception.errors == [] + + +def test_from_http_status_with_errors_and_response(): + message = 'message' + errors = ['1', '2'] + response = mock.sentinel.response + exception = exceptions.from_http_status( + http_client.NOT_FOUND, message, errors=errors, response=response) + + assert isinstance(exception, exceptions.NotFound) + assert exception.code == http_client.NOT_FOUND + assert exception.message == message + assert exception.errors == errors + assert exception.response == response + + +def test_from_http_status_unknown_code(): + message = 'message' + status_code = 156 + exception = exceptions.from_http_status(status_code, message) + assert exception.code == status_code + assert exception.message == message + + +def make_response(content): + response = requests.Response() + response._content = content + response.status_code = http_client.NOT_FOUND + response.request = requests.Request( + method='POST', url='https://example.com').prepare() + return response + + +def test_from_http_response_no_content(): + response = make_response(None) + + exception = exceptions.from_http_response(response) + + assert isinstance(exception, exceptions.NotFound) + assert exception.code == http_client.NOT_FOUND + assert exception.message == 'POST https://example.com/: unknown error' + assert exception.response == response + + +def test_from_http_response_text_content(): + response = make_response(b'message') + + exception = exceptions.from_http_response(response) + + assert isinstance(exception, exceptions.NotFound) + assert exception.code == http_client.NOT_FOUND + assert exception.message == 'POST https://example.com/: message' + + +def test_from_http_response_json_content(): + response = make_response(json.dumps({ + 'error': { + 'message': 'json message', + 'errors': ['1', '2'] + } + }).encode('utf-8')) + + exception = exceptions.from_http_response(response) + + assert isinstance(exception, exceptions.NotFound) + assert exception.code == http_client.NOT_FOUND + assert exception.message == 'POST https://example.com/: json message' + assert exception.errors == ['1', '2'] + + +def test_from_http_response_bad_json_content(): + response = make_response(json.dumps({'meep': 'moop'}).encode('utf-8')) + + exception = exceptions.from_http_response(response) + + assert isinstance(exception, exceptions.NotFound) + assert exception.code == http_client.NOT_FOUND + assert exception.message == 'POST https://example.com/: unknown error' + + +def test_from_grpc_status(): + message = 'message' + exception = exceptions.from_grpc_status( + grpc.StatusCode.OUT_OF_RANGE, message) + assert isinstance(exception, exceptions.BadRequest) + assert isinstance(exception, exceptions.OutOfRange) + assert exception.code == http_client.BAD_REQUEST + assert exception.grpc_status_code == grpc.StatusCode.OUT_OF_RANGE + assert exception.message == message + assert exception.errors == [] + + +def test_from_grpc_status_with_errors_and_response(): + message = 'message' + response = mock.sentinel.response + errors = ['1', '2'] + exception = exceptions.from_grpc_status( + grpc.StatusCode.OUT_OF_RANGE, message, + errors=errors, response=response) + + assert isinstance(exception, exceptions.OutOfRange) + assert exception.message == message + assert exception.errors == errors + assert exception.response == response + + +def test_from_grpc_status_unknown_code(): + message = 'message' + exception = exceptions.from_grpc_status( + grpc.StatusCode.OK, message) + assert exception.grpc_status_code == grpc.StatusCode.OK + assert exception.message == message + + +def test_from_grpc_error(): + message = 'message' + error = mock.create_autospec(grpc.Call, instance=True) + error.code.return_value = grpc.StatusCode.INVALID_ARGUMENT + error.details.return_value = message + + exception = exceptions.from_grpc_error(error) + + assert isinstance(exception, exceptions.BadRequest) + assert isinstance(exception, exceptions.InvalidArgument) + assert exception.code == http_client.BAD_REQUEST + assert exception.grpc_status_code == grpc.StatusCode.INVALID_ARGUMENT + assert exception.message == message + assert exception.errors == [error] + assert exception.response == error + + +def test_from_grpc_error_non_call(): + message = 'message' + error = mock.create_autospec(grpc.RpcError, instance=True) + error.__str__.return_value = message + + exception = exceptions.from_grpc_error(error) + + assert isinstance(exception, exceptions.GoogleAPICallError) + assert exception.code is None + assert exception.grpc_status_code is None + assert exception.message == message + assert exception.errors == [error] + assert exception.response == error From 90e6fe704d8632673ccb7b9ee5532a08cda96f9d Mon Sep 17 00:00:00 2001 From: Luke Sneeringer Date: Tue, 8 Aug 2017 10:36:47 -0700 Subject: [PATCH 05/10] Bump GAX dependency to 0.15.14. (#3752) --- language/setup.py | 2 +- speech/setup.py | 2 +- videointelligence/setup.py | 2 +- vision/setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/language/setup.py b/language/setup.py index 0b7152fd89fd..7180ed0b2cfb 100644 --- a/language/setup.py +++ b/language/setup.py @@ -52,7 +52,7 @@ REQUIREMENTS = [ 'google-cloud-core >= 0.26.0, < 0.27dev', - 'google-gax >= 0.15.13, < 0.16dev', + 'google-gax >= 0.15.14, < 0.16dev', 'googleapis-common-protos[grpc] >= 1.5.2, < 2.0dev', ] EXTRAS_REQUIRE = { diff --git a/speech/setup.py b/speech/setup.py index 6587ceec4779..8bb1208572f3 100644 --- a/speech/setup.py +++ b/speech/setup.py @@ -53,7 +53,7 @@ REQUIREMENTS = [ 'google-cloud-core >= 0.26.0, < 0.27dev', - 'google-gax >= 0.15.13, < 0.16dev', + 'google-gax >= 0.15.14, < 0.16dev', 'googleapis-common-protos[grpc] >= 1.5.2, < 2.0dev', ] diff --git a/videointelligence/setup.py b/videointelligence/setup.py index a47f897e3855..3c7b64622a25 100644 --- a/videointelligence/setup.py +++ b/videointelligence/setup.py @@ -43,7 +43,7 @@ packages=find_packages(exclude=('tests*',)), install_requires=( 'googleapis-common-protos >= 1.5.2, < 2.0dev', - 'google-gax >= 0.15.12, < 0.16dev', + 'google-gax >= 0.15.14, < 0.16dev', 'six >= 1.10.0', ), url='https://github.com/GoogleCloudPlatform/google-cloud-python', diff --git a/vision/setup.py b/vision/setup.py index 7567a30d0e53..c4fbeaeaffab 100644 --- a/vision/setup.py +++ b/vision/setup.py @@ -26,7 +26,7 @@ REQUIREMENTS = [ 'google-cloud-core >= 0.26.0, < 0.27dev', - 'google-gax >= 0.15.13, < 0.16dev', + 'google-gax >= 0.15.14, < 0.16dev', 'googleapis-common-protos[grpc] >= 1.5.2, < 2.0dev', ] EXTRAS_REQUIRE = { From aaaaf7d635794260adee82b4b634d5cb2c7a4a8d Mon Sep 17 00:00:00 2001 From: Luke Sneeringer Date: Tue, 8 Aug 2017 12:36:20 -0700 Subject: [PATCH 06/10] Reference valid input formats in API docs. (#3758) --- bigquery/google/cloud/bigquery/table.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bigquery/google/cloud/bigquery/table.py b/bigquery/google/cloud/bigquery/table.py index ffbd47ca6c4c..c6bf5db893ab 100644 --- a/bigquery/google/cloud/bigquery/table.py +++ b/bigquery/google/cloud/bigquery/table.py @@ -1045,9 +1045,10 @@ def upload_from_file(self, :param file_obj: A file handle opened in binary mode for reading. :type source_format: str - :param source_format: one of 'CSV' or 'NEWLINE_DELIMITED_JSON'. - job configuration option; see - :meth:`google.cloud.bigquery.job.LoadJob` + :param source_format: Any supported format. The full list of supported + formats is documented under the + ``configuration.extract.destinationFormat`` property on this page: + https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs :type rewind: bool :param rewind: If True, seek to the beginning of the file handle before From 1ff7708fba64f289a957214dd089f224b74f2467 Mon Sep 17 00:00:00 2001 From: Luke Sneeringer Date: Tue, 8 Aug 2017 12:37:24 -0700 Subject: [PATCH 07/10] Make exclude_from_indexes a set, and public API. (#3756) --- datastore/google/cloud/datastore/entity.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/datastore/google/cloud/datastore/entity.py b/datastore/google/cloud/datastore/entity.py index dc8a60b038be..e74d5aa640ee 100644 --- a/datastore/google/cloud/datastore/entity.py +++ b/datastore/google/cloud/datastore/entity.py @@ -129,8 +129,9 @@ class Entity(dict): def __init__(self, key=None, exclude_from_indexes=()): super(Entity, self).__init__() self.key = key - self._exclude_from_indexes = set(_ensure_tuple_or_list( + self.exclude_from_indexes = set(_ensure_tuple_or_list( 'exclude_from_indexes', exclude_from_indexes)) + """Names of fields which are *not* to be indexed for this entity.""" # NOTE: This will be populated when parsing a protobuf in # google.cloud.datastore.helpers.entity_from_protobuf. self._meanings = {} @@ -148,7 +149,7 @@ def __eq__(self, other): return False return (self.key == other.key and - self._exclude_from_indexes == other._exclude_from_indexes and + self.exclude_from_indexes == other.exclude_from_indexes and self._meanings == other._meanings and super(Entity, self).__eq__(other)) @@ -176,15 +177,6 @@ def kind(self): if self.key: return self.key.kind - @property - def exclude_from_indexes(self): - """Names of fields which are *not* to be indexed for this entity. - - :rtype: sequence of field names - :returns: The set of fields excluded from indexes. - """ - return frozenset(self._exclude_from_indexes) - def __repr__(self): if self.key: return '' % (self.key._flat_path, From cdadf4c7dc891ad88bbb53d8a61b6ff019de2c9a Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Tue, 8 Aug 2017 14:03:04 -0700 Subject: [PATCH 08/10] Move google.cloud.future to google.api.core (#3764) --- bigquery/google/cloud/bigquery/job.py | 6 ++-- bigtable/google/cloud/bigtable/cluster.py | 2 +- bigtable/google/cloud/bigtable/instance.py | 2 +- bigtable/tests/unit/test_cluster.py | 4 +-- bigtable/tests/unit/test_instance.py | 4 +-- .../{cloud => api/core}/future/__init__.py | 2 +- .../{cloud => api/core}/future/_helpers.py | 0 .../google/{cloud => api/core}/future/base.py | 0 .../{cloud => api/core}/future/polling.py | 4 +-- .../{cloud/future => api/core}/operation.py | 35 +++++++++++++++---- .../unit/{ => api_core}/future/__init__.py | 0 .../{ => api_core}/future/test__helpers.py | 2 +- .../{ => api_core}/future/test_polling.py | 2 +- .../{future => api_core}/test_operation.py | 2 +- spanner/google/cloud/spanner/database.py | 4 +-- spanner/google/cloud/spanner/instance.py | 4 +-- 16 files changed, 48 insertions(+), 25 deletions(-) rename core/google/{cloud => api/core}/future/__init__.py (93%) rename core/google/{cloud => api/core}/future/_helpers.py (100%) rename core/google/{cloud => api/core}/future/base.py (100%) rename core/google/{cloud => api/core}/future/polling.py (98%) rename core/google/{cloud/future => api/core}/operation.py (91%) rename core/tests/unit/{ => api_core}/future/__init__.py (100%) rename core/tests/unit/{ => api_core}/future/test__helpers.py (96%) rename core/tests/unit/{ => api_core}/future/test_polling.py (98%) rename core/tests/unit/{future => api_core}/test_operation.py (99%) diff --git a/bigquery/google/cloud/bigquery/job.py b/bigquery/google/cloud/bigquery/job.py index 1519e2a0cf6e..43d7fd8f23c3 100644 --- a/bigquery/google/cloud/bigquery/job.py +++ b/bigquery/google/cloud/bigquery/job.py @@ -19,6 +19,7 @@ import six from six.moves import http_client +import google.api.core.future.polling from google.cloud import exceptions from google.cloud.exceptions import NotFound from google.cloud._helpers import _datetime_from_microseconds @@ -31,7 +32,6 @@ from google.cloud.bigquery._helpers import UDFResourcesProperty from google.cloud.bigquery._helpers import _EnumProperty from google.cloud.bigquery._helpers import _TypedProperty -import google.cloud.future.polling _DONE_STATE = 'DONE' _STOPPED_REASON = 'stopped' @@ -140,7 +140,7 @@ class WriteDisposition(_EnumProperty): WRITE_EMPTY = 'WRITE_EMPTY' -class _AsyncJob(google.cloud.future.polling.PollingFuture): +class _AsyncJob(google.api.core.future.polling.PollingFuture): """Base class for asynchronous jobs. :type name: str @@ -496,7 +496,7 @@ def cancelled(self): This always returns False. It's not possible to check if a job was cancelled in the API. This method is here to satisfy the interface - for :class:`google.cloud.future.Future`. + for :class:`google.api.core.future.Future`. :rtype: bool :returns: False diff --git a/bigtable/google/cloud/bigtable/cluster.py b/bigtable/google/cloud/bigtable/cluster.py index 8d15547efae3..09a34e11bb05 100644 --- a/bigtable/google/cloud/bigtable/cluster.py +++ b/bigtable/google/cloud/bigtable/cluster.py @@ -17,11 +17,11 @@ import re +from google.api.core import operation from google.cloud.bigtable._generated import ( instance_pb2 as data_v2_pb2) from google.cloud.bigtable._generated import ( bigtable_instance_admin_pb2 as messages_v2_pb2) -from google.cloud.future import operation _CLUSTER_NAME_RE = re.compile(r'^projects/(?P[^/]+)/' r'instances/(?P[^/]+)/clusters/' diff --git a/bigtable/google/cloud/bigtable/instance.py b/bigtable/google/cloud/bigtable/instance.py index 958f16602953..5e73ed2ba661 100644 --- a/bigtable/google/cloud/bigtable/instance.py +++ b/bigtable/google/cloud/bigtable/instance.py @@ -17,6 +17,7 @@ import re +from google.api.core import operation from google.cloud.bigtable._generated import ( instance_pb2 as data_v2_pb2) from google.cloud.bigtable._generated import ( @@ -26,7 +27,6 @@ from google.cloud.bigtable.cluster import Cluster from google.cloud.bigtable.cluster import DEFAULT_SERVE_NODES from google.cloud.bigtable.table import Table -from google.cloud.future import operation _EXISTING_INSTANCE_LOCATION_ID = 'see-existing-cluster' diff --git a/bigtable/tests/unit/test_cluster.py b/bigtable/tests/unit/test_cluster.py index e244b55d6dff..8ed54846d18e 100644 --- a/bigtable/tests/unit/test_cluster.py +++ b/bigtable/tests/unit/test_cluster.py @@ -233,8 +233,8 @@ def test_reload(self): self.assertEqual(cluster.location, LOCATION) def test_create(self): + from google.api.core import operation from google.longrunning import operations_pb2 - from google.cloud.future import operation from google.cloud.bigtable._generated import ( bigtable_instance_admin_pb2 as messages_v2_pb2) from tests.unit._testing import _FakeStub @@ -275,8 +275,8 @@ def test_create(self): def test_update(self): import datetime + from google.api.core import operation from google.longrunning import operations_pb2 - from google.cloud.future import operation from google.protobuf.any_pb2 import Any from google.cloud._helpers import _datetime_to_pb_timestamp from google.cloud.bigtable._generated import ( diff --git a/bigtable/tests/unit/test_instance.py b/bigtable/tests/unit/test_instance.py index 03c0034fc49e..ce475e0d5a66 100644 --- a/bigtable/tests/unit/test_instance.py +++ b/bigtable/tests/unit/test_instance.py @@ -232,13 +232,13 @@ def test_reload(self): def test_create(self): import datetime + from google.api.core import operation from google.longrunning import operations_pb2 from google.protobuf.any_pb2 import Any from google.cloud.bigtable._generated import ( bigtable_instance_admin_pb2 as messages_v2_pb2) from google.cloud._helpers import _datetime_to_pb_timestamp from tests.unit._testing import _FakeStub - from google.cloud.future import operation from google.cloud.bigtable.cluster import DEFAULT_SERVE_NODES NOW = datetime.datetime.utcnow() @@ -285,11 +285,11 @@ def test_create(self): self.assertEqual(kwargs, {}) def test_create_w_explicit_serve_nodes(self): + from google.api.core import operation from google.longrunning import operations_pb2 from google.cloud.bigtable._generated import ( bigtable_instance_admin_pb2 as messages_v2_pb2) from tests.unit._testing import _FakeStub - from google.cloud.future import operation SERVE_NODES = 5 diff --git a/core/google/cloud/future/__init__.py b/core/google/api/core/future/__init__.py similarity index 93% rename from core/google/cloud/future/__init__.py rename to core/google/api/core/future/__init__.py index e5cf2b20ce7e..a61510d307e6 100644 --- a/core/google/cloud/future/__init__.py +++ b/core/google/api/core/future/__init__.py @@ -14,7 +14,7 @@ """Futures for dealing with asynchronous operations.""" -from google.cloud.future.base import Future +from google.api.core.future.base import Future __all__ = [ 'Future', diff --git a/core/google/cloud/future/_helpers.py b/core/google/api/core/future/_helpers.py similarity index 100% rename from core/google/cloud/future/_helpers.py rename to core/google/api/core/future/_helpers.py diff --git a/core/google/cloud/future/base.py b/core/google/api/core/future/base.py similarity index 100% rename from core/google/cloud/future/base.py rename to core/google/api/core/future/base.py diff --git a/core/google/cloud/future/polling.py b/core/google/api/core/future/polling.py similarity index 98% rename from core/google/cloud/future/polling.py rename to core/google/api/core/future/polling.py index 6b7ae4221f64..40380d6ad938 100644 --- a/core/google/cloud/future/polling.py +++ b/core/google/api/core/future/polling.py @@ -22,8 +22,8 @@ import six import tenacity -from google.cloud.future import _helpers -from google.cloud.future import base +from google.api.core.future import _helpers +from google.api.core.future import base class PollingFuture(base.Future): diff --git a/core/google/cloud/future/operation.py b/core/google/api/core/operation.py similarity index 91% rename from core/google/cloud/future/operation.py rename to core/google/api/core/operation.py index ec430cd9c55b..1cc44f0b3d7b 100644 --- a/core/google/cloud/future/operation.py +++ b/core/google/api/core/operation.py @@ -12,14 +12,36 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Futures for long-running operations returned from Google Cloud APIs.""" +"""Futures for long-running operations returned from Google Cloud APIs. + +These futures can be used to synchronously wait for the result of a +long-running operation using :meth:`Operation.result`: + + +.. code-block:: python + + operation = my_api_client.long_running_method() + result = operation.result() + +Or asynchronously using callbacks and :meth:`Operation.add_done_callback`: + +.. code-block:: python + + operation = my_api_client.long_running_method() + + def my_callback(future): + result = future.result() + + operation.add_done_callback(my_callback) + +""" import functools import threading +from google.api.core import exceptions +from google.api.core.future import polling from google.cloud import _helpers -from google.cloud import exceptions -from google.cloud.future import polling from google.longrunning import operations_pb2 from google.protobuf import json_format from google.rpc import code_pb2 @@ -85,12 +107,13 @@ def _set_result_from_operation(self): self._result_type, self._operation.response) self.set_result(response) elif self._operation.HasField('error'): - exception = exceptions.GoogleCloudError( + exception = exceptions.GoogleAPICallError( self._operation.error.message, - errors=(self._operation.error)) + errors=(self._operation.error), + response=self._operation) self.set_exception(exception) else: - exception = exceptions.GoogleCloudError( + exception = exceptions.GoogleAPICallError( 'Unexpected state: Long-running operation had neither ' 'response nor error set.') self.set_exception(exception) diff --git a/core/tests/unit/future/__init__.py b/core/tests/unit/api_core/future/__init__.py similarity index 100% rename from core/tests/unit/future/__init__.py rename to core/tests/unit/api_core/future/__init__.py diff --git a/core/tests/unit/future/test__helpers.py b/core/tests/unit/api_core/future/test__helpers.py similarity index 96% rename from core/tests/unit/future/test__helpers.py rename to core/tests/unit/api_core/future/test__helpers.py index cbca5ba4d4df..534dd3696cb9 100644 --- a/core/tests/unit/future/test__helpers.py +++ b/core/tests/unit/api_core/future/test__helpers.py @@ -14,7 +14,7 @@ import mock -from google.cloud.future import _helpers +from google.api.core.future import _helpers @mock.patch('threading.Thread', autospec=True) diff --git a/core/tests/unit/future/test_polling.py b/core/tests/unit/api_core/future/test_polling.py similarity index 98% rename from core/tests/unit/future/test_polling.py rename to core/tests/unit/api_core/future/test_polling.py index c8fde1c20385..a359ba1a2152 100644 --- a/core/tests/unit/future/test_polling.py +++ b/core/tests/unit/api_core/future/test_polling.py @@ -19,7 +19,7 @@ import mock import pytest -from google.cloud.future import polling +from google.api.core.future import polling class PollingFutureImpl(polling.PollingFuture): diff --git a/core/tests/unit/future/test_operation.py b/core/tests/unit/api_core/test_operation.py similarity index 99% rename from core/tests/unit/future/test_operation.py rename to core/tests/unit/api_core/test_operation.py index 2d281694001a..2332c50fdf4b 100644 --- a/core/tests/unit/future/test_operation.py +++ b/core/tests/unit/api_core/test_operation.py @@ -15,7 +15,7 @@ import mock -from google.cloud.future import operation +from google.api.core import operation from google.longrunning import operations_pb2 from google.protobuf import struct_pb2 from google.rpc import code_pb2 diff --git a/spanner/google/cloud/spanner/database.py b/spanner/google/cloud/spanner/database.py index acfcefdce891..38dc1c7eaaf8 100644 --- a/spanner/google/cloud/spanner/database.py +++ b/spanner/google/cloud/spanner/database.py @@ -185,7 +185,7 @@ def create(self): See https://cloud.google.com/spanner/reference/rpc/google.spanner.admin.database.v1#google.spanner.admin.database.v1.DatabaseAdmin.CreateDatabase - :rtype: :class:`~google.cloud.future.operation.Operation` + :rtype: :class:`~google.api.core.operation.Operation` :returns: a future used to poll the status of the create request :raises Conflict: if the database already exists :raises NotFound: if the instance owning the database does not exist @@ -269,7 +269,7 @@ def update_ddl(self, ddl_statements): See https://cloud.google.com/spanner/reference/rpc/google.spanner.admin.database.v1#google.spanner.admin.database.v1.DatabaseAdmin.UpdateDatabase - :rtype: :class:`google.cloud.future.operation.Operation` + :rtype: :class:`google.api.core.operation.Operation` :returns: an operation instance :raises NotFound: if the database does not exist :raises GaxError: diff --git a/spanner/google/cloud/spanner/instance.py b/spanner/google/cloud/spanner/instance.py index 5bd4663764f5..4a51c7055731 100644 --- a/spanner/google/cloud/spanner/instance.py +++ b/spanner/google/cloud/spanner/instance.py @@ -198,7 +198,7 @@ def create(self): before calling :meth:`create`. - :rtype: :class:`google.cloud.future.operation.Operation` + :rtype: :class:`google.api.core.operation.Operation` :returns: an operation instance :raises Conflict: if the instance already exists :raises GaxError: @@ -289,7 +289,7 @@ def update(self): before calling :meth:`update`. - :rtype: :class:`google.cloud.future.operation.Operation` + :rtype: :class:`google.api.core.operation.Operation` :returns: an operation instance :raises NotFound: if the instance does not exist :raises GaxError: for other errors returned from the call From 18478f83306554adf963cb227594e6a7899ee5c0 Mon Sep 17 00:00:00 2001 From: Luke Sneeringer Date: Thu, 16 Mar 2017 09:54:46 -0700 Subject: [PATCH 09/10] Non-functional datastore fix for #3152. --- datastore/google/cloud/datastore/helpers.py | 2 +- datastore/tests/unit/test_helpers.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/datastore/google/cloud/datastore/helpers.py b/datastore/google/cloud/datastore/helpers.py index ee4537317030..91b08452812f 100644 --- a/datastore/google/cloud/datastore/helpers.py +++ b/datastore/google/cloud/datastore/helpers.py @@ -133,7 +133,7 @@ def entity_from_protobuf(pb): # Check if ``value_pb`` was excluded from index. Lists need to be # special-cased and we require all ``exclude_from_indexes`` values # in a list agree. - if is_list: + if is_list and len(value) > 0: exclude_values = set(value_pb.exclude_from_indexes for value_pb in value_pb.array_value.values) if len(exclude_values) != 1: diff --git a/datastore/tests/unit/test_helpers.py b/datastore/tests/unit/test_helpers.py index 18ff98e64781..8529f214640c 100644 --- a/datastore/tests/unit/test_helpers.py +++ b/datastore/tests/unit/test_helpers.py @@ -135,6 +135,27 @@ def test_mismatched_value_indexed(self): with self.assertRaises(ValueError): self._call_fut(entity_pb) + def test_index_mismatch_ignores_empty_list(self): + from google.cloud.proto.datastore.v1 import entity_pb2 + from google.cloud.datastore.helpers import _new_value_pb + + _PROJECT = 'PROJECT' + _KIND = 'KIND' + _ID = 1234 + entity_pb = entity_pb2.Entity() + entity_pb.key.partition_id.project_id = _PROJECT + entity_pb.key.path.add(kind=_KIND, id=_ID) + + array_val_pb = _new_value_pb(entity_pb, 'baz') + array_pb = array_val_pb.array_value.values + + # unindexed_value_pb1 = array_pb.add() + # unindexed_value_pb1.integer_value = 10 + + entity = self._call_fut(entity_pb) + entity_dict = dict(entity) + self.assertIsInstance(entity_dict['baz'], list) + def test_entity_no_key(self): from google.cloud.proto.datastore.v1 import entity_pb2 From ef4ed3ab8902015054ec9bd3729affbf3501968d Mon Sep 17 00:00:00 2001 From: Luke Sneeringer Date: Tue, 8 Aug 2017 14:29:37 -0700 Subject: [PATCH 10/10] Fix the unit test. --- datastore/tests/unit/test_helpers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datastore/tests/unit/test_helpers.py b/datastore/tests/unit/test_helpers.py index 8529f214640c..95340c525d48 100644 --- a/datastore/tests/unit/test_helpers.py +++ b/datastore/tests/unit/test_helpers.py @@ -149,12 +149,12 @@ def test_index_mismatch_ignores_empty_list(self): array_val_pb = _new_value_pb(entity_pb, 'baz') array_pb = array_val_pb.array_value.values - # unindexed_value_pb1 = array_pb.add() - # unindexed_value_pb1.integer_value = 10 + unindexed_value_pb1 = array_pb.add() + unindexed_value_pb1.integer_value = 10 entity = self._call_fut(entity_pb) entity_dict = dict(entity) - self.assertIsInstance(entity_dict['baz'], list) + self.assertEqual(entity_dict['baz'], [10]) def test_entity_no_key(self): from google.cloud.proto.datastore.v1 import entity_pb2