From 128db625f072e682b9148d4f17cca3e84528112a Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 28 Nov 2016 16:50:30 -0500 Subject: [PATCH 01/20] Add 'ScalarQueryParameter' class. Holds name, type, and value for scalar query parameters, and handles marshalling them to / from JSON representation mandated by the BigQuery API. Toward #2551. --- bigquery/google/cloud/bigquery/_helpers.py | 63 +++++++++++++++++ bigquery/unit_tests/test__helpers.py | 81 ++++++++++++++++++++++ 2 files changed, 144 insertions(+) diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index 89eb390993c6..5bcd2b3e8257 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -241,3 +241,66 @@ def _build_udf_resources(resources): udf = {resource.udf_type: resource.value} udfs.append(udf) return udfs + + +class ScalarQueryParameter(object): + """Named / positional query parameters for scalar values. + + :type name: str or None + :param name: Parameter name, used via `@foo` syntax. If None, the + paramter can only be addressed via position (`?`). + + :type type_: str + :param type_: name of parameter type. One of `'STRING'`, `'INT64'`, + `'FLOAT64'`, `'BOOLEAN'`, `'TIMESTAMP'`, or `'DATE'`. + + :type value: str, int, float, bool, :class:`datetime.datetime`, or + :class:`datetime.date`. + :param value: the scalar parameter value. + """ + def __init__(self, name, type_, value): + self.name = name + self.type_ = type_ + self.value = value + + @classmethod + def positional(cls, type_, value): + """Factory for positional paramters. + + + :type type_: str + :param type_: name of paramter type. One of `'STRING'`, `'INT64'`, + `'FLOAT64'`, `'BOOLEAN'`, `'TIMESTAMP'`, or `'DATE'`. + + :type value: str, int, float, bool, :class:`datetime.datetime`, or + :class:`datetime.date`. + :param value: the scalar parameter value. + """ + return cls(None, type_, value) + + @classmethod + def from_api_repr(cls, resource): + """Factory: construct paramter from JSON resource. + + :type resource: dict + :param resource: JSON mapping of parameter + """ + name = resource.get('name') + type_ = resource['parameterType']['type'] + value = resource['parameterValue']['value'] + converted = _CELLDATA_FROM_JSON[type_](value, None) + return cls(name, type_, converted) + + def to_api_repr(self): + """Construct JSON API representation for the parameter.""" + resource = { + 'parameterType': { + 'type': self.type_, + }, + 'parameterValue': { + 'value': self.value, + }, + } + if self.name is not None: + resource['name'] = self.name + return resource diff --git a/bigquery/unit_tests/test__helpers.py b/bigquery/unit_tests/test__helpers.py index 46c58c8ea405..8108a47831ab 100644 --- a/bigquery/unit_tests/test__helpers.py +++ b/bigquery/unit_tests/test__helpers.py @@ -491,6 +491,87 @@ def test_instance_setter_w_bad_udfs(self): self.assertEqual(instance.udf_resources, []) +class Test_ScalarQueryParameter(unittest.TestCase): + + @staticmethod + def _get_target_class(): + from google.cloud.bigquery._helpers import ScalarQueryParameter + return ScalarQueryParameter + + def _make_one(self, *args, **kw): + return self._get_target_class()(*args, **kw) + + def test_ctor_scalar(self): + parm = self._make_one(name='foo', type_='INT64', value=123) + self.assertEqual(parm.name, 'foo') + self.assertEqual(parm.type_, 'INT64') + self.assertEqual(parm.value, 123) + + def test_positional_scalar(self): + klass = self._get_target_class() + parm = klass.positional(type_='INT64', value=123) + self.assertEqual(parm.name, None) + self.assertEqual(parm.type_, 'INT64') + self.assertEqual(parm.value, 123) + + def test_from_api_repr_w_name(self): + RESOURCE = { + 'name': 'foo', + 'parameterType': { + 'type': 'INT64', + }, + 'parameterValue': { + 'value': 123, + }, + } + klass = self._get_target_class() + parm = klass.from_api_repr(RESOURCE) + self.assertEqual(parm.name, 'foo') + self.assertEqual(parm.type_, 'INT64') + self.assertEqual(parm.value, 123) + + def test_from_api_repr_wo_name(self): + RESOURCE = { + 'parameterType': { + 'type': 'INT64', + }, + 'parameterValue': { + 'value': 123, + }, + } + klass = self._get_target_class() + parm = klass.from_api_repr(RESOURCE) + self.assertEqual(parm.name, None) + self.assertEqual(parm.type_, 'INT64') + self.assertEqual(parm.value, 123) + + def test_to_api_repr_w_name(self): + EXPECTED = { + 'name': 'foo', + 'parameterType': { + 'type': 'INT64', + }, + 'parameterValue': { + 'value': 123, + }, + } + parm = self._make_one(name='foo', type_='INT64', value=123) + self.assertEqual(parm.to_api_repr(), EXPECTED) + + def test_to_api_repr_wo_name(self): + EXPECTED = { + 'parameterType': { + 'type': 'INT64', + }, + 'parameterValue': { + 'value': 123, + }, + } + klass = self._get_target_class() + parm = klass.positional(type_='INT64', value=123) + self.assertEqual(parm.to_api_repr(), EXPECTED) + + class _Field(object): def __init__(self, mode, name='unknown', field_type='UNKNOWN', fields=()): From 3421d7a983e311c5ea2c7ce16ac08bf30193a6e2 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 28 Nov 2016 17:11:19 -0500 Subject: [PATCH 02/20] Factor out 'AbstractQueryParameter. --- bigquery/google/cloud/bigquery/_helpers.py | 36 ++++++++++++++++++++-- bigquery/unit_tests/test__helpers.py | 21 +++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index 5bcd2b3e8257..ef1b3815b379 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -243,7 +243,29 @@ def _build_udf_resources(resources): return udfs -class ScalarQueryParameter(object): +class AbstractQueryParameter(object): + """Base class for named / positional query parameters. + """ + @classmethod + def from_api_repr(cls, resource): + """Factory: construct paramter from JSON resource. + + :type resource: dict + :param resource: JSON mapping of parameter + + :rtype: :class:`ScalarQueryParameter` + """ + raise NotImplementedError + + def to_api_repr(self): + """Construct JSON API representation for the parameter. + + :rtype: dict + """ + raise NotImplementedError + + +class ScalarQueryParameter(AbstractQueryParameter): """Named / positional query parameters for scalar values. :type name: str or None @@ -275,6 +297,9 @@ def positional(cls, type_, value): :type value: str, int, float, bool, :class:`datetime.datetime`, or :class:`datetime.date`. :param value: the scalar parameter value. + + :rtype: :class:`ScalarQueryParameter` + :returns: instance w/o name """ return cls(None, type_, value) @@ -284,6 +309,9 @@ def from_api_repr(cls, resource): :type resource: dict :param resource: JSON mapping of parameter + + :rtype: :class:`ScalarQueryParameter` + :returns: instance """ name = resource.get('name') type_ = resource['parameterType']['type'] @@ -292,7 +320,11 @@ def from_api_repr(cls, resource): return cls(name, type_, converted) def to_api_repr(self): - """Construct JSON API representation for the parameter.""" + """Construct JSON API representation for the parameter. + + :rtype: dict + :returns: JSON mapping + """ resource = { 'parameterType': { 'type': self.type_, diff --git a/bigquery/unit_tests/test__helpers.py b/bigquery/unit_tests/test__helpers.py index 8108a47831ab..120574f9808b 100644 --- a/bigquery/unit_tests/test__helpers.py +++ b/bigquery/unit_tests/test__helpers.py @@ -491,6 +491,27 @@ def test_instance_setter_w_bad_udfs(self): self.assertEqual(instance.udf_resources, []) +class Test_AbstractQueryParameter(unittest.TestCase): + + @staticmethod + def _get_target_class(): + from google.cloud.bigquery._helpers import AbstractQueryParameter + return AbstractQueryParameter + + def _make_one(self, *args, **kw): + return self._get_target_class()(*args, **kw) + + def test_from_api_virtual(self): + klass = self._get_target_class() + with self.assertRaises(NotImplementedError): + klass.from_api_repr({}) + + def test_to_api_virtual(self): + parm = self._make_one() + with self.assertRaises(NotImplementedError): + parm.to_api_repr() + + class Test_ScalarQueryParameter(unittest.TestCase): @staticmethod From 57c61c17e04163237a206be5453d31356cde33eb Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 28 Nov 2016 21:32:48 -0500 Subject: [PATCH 03/20] Add 'ArrayQueryParameter' class. Holds name, type, and value for array query parameters, and handles marshalling them to / from JSON representation mandated by the BigQuery API. Toward #2551. --- bigquery/google/cloud/bigquery/_helpers.py | 73 ++++++++++++++++++ bigquery/unit_tests/test__helpers.py | 87 +++++++++++++++++++++- 2 files changed, 157 insertions(+), 3 deletions(-) diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index ef1b3815b379..dc6f10934b16 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -336,3 +336,76 @@ def to_api_repr(self): if self.name is not None: resource['name'] = self.name return resource + + +class ArrayQueryParameter(AbstractQueryParameter): + """Named / positional query parameters for array values. + + :type name: str or None + :param name: Parameter name, used via `@foo` syntax. If None, the + paramter can only be addressed via position (`?`). + + :type array_type: str + :param array_type: + name of type of array elements. One of `'STRING'`, `'INT64'`, + `'FLOAT64'`, `'BOOLEAN'`, `'TIMESTAMP'`, or `'DATE'`. + + :type values: list of appropriate scalar type. + :param values: the parameter array values. + """ + def __init__(self, name, array_type, values): + self.name = name + self.array_type = array_type + self.values = values + + @classmethod + def positional(cls, array_type, values): + """Factory for positional paramters. + + :type array_type: str + :param array_type: + name of type of array elements. One of `'STRING'`, `'INT64'`, + `'FLOAT64'`, `'BOOLEAN'`, `'TIMESTAMP'`, or `'DATE'`. + + :type values: list of appropriate scalar type + :param values: the parameter array values. + + :rtype: :class:`ArrayQueryParameter` + :returns: instance w/o name + """ + return cls(None, array_type, values) + + @classmethod + def from_api_repr(cls, resource): + """Factory: construct paramter from JSON resource. + + :type resource: dict + :param resource: JSON mapping of parameter + + :rtype: :class:`ArrayQueryParameter` + :returns: instance + """ + name = resource.get('name') + array_type = resource['parameterType']['arrayType'] + values = resource['parameterValue']['arrayValues'] + converted = [ + _CELLDATA_FROM_JSON[array_type](value, None) for value in values] + return cls(name, array_type, converted) + + def to_api_repr(self): + """Construct JSON API representation for the parameter. + + :rtype: dict + :returns: JSON mapping + """ + resource = { + 'parameterType': { + 'arrayType': self.array_type, + }, + 'parameterValue': { + 'arrayValues': self.values, + }, + } + if self.name is not None: + resource['name'] = self.name + return resource diff --git a/bigquery/unit_tests/test__helpers.py b/bigquery/unit_tests/test__helpers.py index 120574f9808b..f1f23f062b0e 100644 --- a/bigquery/unit_tests/test__helpers.py +++ b/bigquery/unit_tests/test__helpers.py @@ -522,13 +522,13 @@ def _get_target_class(): def _make_one(self, *args, **kw): return self._get_target_class()(*args, **kw) - def test_ctor_scalar(self): + def test_ctor(self): parm = self._make_one(name='foo', type_='INT64', value=123) self.assertEqual(parm.name, 'foo') self.assertEqual(parm.type_, 'INT64') self.assertEqual(parm.value, 123) - def test_positional_scalar(self): + def test_positional(self): klass = self._get_target_class() parm = klass.positional(type_='INT64', value=123) self.assertEqual(parm.name, None) @@ -557,7 +557,7 @@ def test_from_api_repr_wo_name(self): 'type': 'INT64', }, 'parameterValue': { - 'value': 123, + 'value': '123', }, } klass = self._get_target_class() @@ -593,6 +593,87 @@ def test_to_api_repr_wo_name(self): self.assertEqual(parm.to_api_repr(), EXPECTED) +class Test_ArrayQueryParameter(unittest.TestCase): + + @staticmethod + def _get_target_class(): + from google.cloud.bigquery._helpers import ArrayQueryParameter + return ArrayQueryParameter + + def _make_one(self, *args, **kw): + return self._get_target_class()(*args, **kw) + + def test_ctor(self): + parm = self._make_one(name='foo', array_type='INT64', values=[1, 2]) + self.assertEqual(parm.name, 'foo') + self.assertEqual(parm.array_type, 'INT64') + self.assertEqual(parm.values, [1, 2]) + + def test_positional(self): + klass = self._get_target_class() + parm = klass.positional(array_type='INT64', values=[1, 2]) + self.assertEqual(parm.name, None) + self.assertEqual(parm.array_type, 'INT64') + self.assertEqual(parm.values, [1, 2]) + + def test_from_api_repr_w_name(self): + RESOURCE = { + 'name': 'foo', + 'parameterType': { + 'arrayType': 'INT64', + }, + 'parameterValue': { + 'arrayValues': ['1', '2'], + }, + } + klass = self._get_target_class() + parm = klass.from_api_repr(RESOURCE) + self.assertEqual(parm.name, 'foo') + self.assertEqual(parm.array_type, 'INT64') + self.assertEqual(parm.values, [1, 2]) + + def test_from_api_repr_wo_name(self): + RESOURCE = { + 'parameterType': { + 'arrayType': 'INT64', + }, + 'parameterValue': { + 'arrayValues': ['1', '2'], + }, + } + klass = self._get_target_class() + parm = klass.from_api_repr(RESOURCE) + self.assertEqual(parm.name, None) + self.assertEqual(parm.array_type, 'INT64') + self.assertEqual(parm.values, [1, 2]) + + def test_to_api_repr_w_name(self): + EXPECTED = { + 'name': 'foo', + 'parameterType': { + 'arrayType': 'INT64', + }, + 'parameterValue': { + 'arrayValues': [1, 2], + }, + } + parm = self._make_one(name='foo', array_type='INT64', values=[1, 2]) + self.assertEqual(parm.to_api_repr(), EXPECTED) + + def test_to_api_repr_wo_name(self): + EXPECTED = { + 'parameterType': { + 'arrayType': 'INT64', + }, + 'parameterValue': { + 'arrayValues': [1, 2], + }, + } + klass = self._get_target_class() + parm = klass.positional(array_type='INT64', values=[1, 2]) + self.assertEqual(parm.to_api_repr(), EXPECTED) + + class _Field(object): def __init__(self, mode, name='unknown', field_type='UNKNOWN', fields=()): From 6650c59cbe4b12afdd52cbeea6e306807a25c79e Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 29 Nov 2016 11:59:44 -0500 Subject: [PATCH 04/20] Add 'StructQueryParameter' class. Holds name, types, and values for Struct query parameters, and handles marshalling them to / from JSON representation mandated by the BigQuery API. Toward #2551. --- bigquery/google/cloud/bigquery/_helpers.py | 73 ++++++++++++++ bigquery/unit_tests/test__helpers.py | 106 +++++++++++++++++++++ 2 files changed, 179 insertions(+) diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index dc6f10934b16..2889b811e576 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -409,3 +409,76 @@ def to_api_repr(self): if self.name is not None: resource['name'] = self.name return resource + + +class StructQueryParameter(AbstractQueryParameter): + """Named / positional query parameters for struct values. + + :type name: str or None + :param name: Parameter name, used via `@foo` syntax. If None, the + paramter can only be addressed via position (`?`). + + :type sub_parms: tuple of :class:`ScalarQueryParameter` + :param sub_parms: the sub-parameters for the struct + """ + def __init__(self, name, *sub_parms): + self.name = name + self._order = [sub.name for sub in sub_parms] + self.struct_types = {sub.name: sub.type_ for sub in sub_parms} + self.struct_values = {sub.name: sub.value for sub in sub_parms} + + @classmethod + def positional(cls, *sub_parms): + """Factory for positional paramters. + + :type sub_parms: tuple of :class:`ScalarQueryParameter` + :param sub_parms:s the sub-parameters for the struct + + :rtype: :class:`StructQueryParameter` + :returns: instance w/o name + """ + return cls(None, *sub_parms) + + @classmethod + def from_api_repr(cls, resource): + """Factory: construct paramter from JSON resource. + + :type resource: dict + :param resource: JSON mapping of parameter + + :rtype: :class:`StructQueryParameter` + :returns: instance + """ + name = resource.get('name') + instance = cls(name) + types = instance.struct_types = { + item['name']: item['type'] + for item in resource['parameterType']['structTypes'] + } + struct_values = resource['parameterValue']['structValues'] + values = instance.struct_values = {} + for key, value in struct_values.items(): + values[key] = _CELLDATA_FROM_JSON[types[key]](value, None) + return instance + + def to_api_repr(self): + """Construct JSON API representation for the parameter. + + :rtype: dict + :returns: JSON mapping + """ + types = [ + {'name': name, 'type': self.struct_types[name]} + for name in self._order + ] + resource = { + 'parameterType': { + 'structTypes': types, + }, + 'parameterValue': { + 'structValues': self.struct_values, + }, + } + if self.name is not None: + resource['name'] = self.name + return resource diff --git a/bigquery/unit_tests/test__helpers.py b/bigquery/unit_tests/test__helpers.py index f1f23f062b0e..c14eca67ecf7 100644 --- a/bigquery/unit_tests/test__helpers.py +++ b/bigquery/unit_tests/test__helpers.py @@ -674,6 +674,112 @@ def test_to_api_repr_wo_name(self): self.assertEqual(parm.to_api_repr(), EXPECTED) +class Test_StructQueryParameter(unittest.TestCase): + + @staticmethod + def _get_target_class(): + from google.cloud.bigquery._helpers import StructQueryParameter + return StructQueryParameter + + def _make_one(self, *args, **kw): + return self._get_target_class()(*args, **kw) + + @staticmethod + def _make_subparam(name, type_, value): + from google.cloud.bigquery._helpers import ScalarQueryParameter + return ScalarQueryParameter(name, type_, value) + + def test_ctor(self): + sub_1 = self._make_subparam('bar', 'INT64', 123) + sub_2 = self._make_subparam('baz', 'STRING', 'abc') + parm = self._make_one('foo', sub_1, sub_2) + self.assertEqual(parm.name, 'foo') + self.assertEqual(parm.struct_types, {'bar': 'INT64', 'baz': 'STRING'}) + self.assertEqual(parm.struct_values, {'bar': 123, 'baz': 'abc'}) + + def test_positional(self): + sub_1 = self._make_subparam('bar', 'INT64', 123) + sub_2 = self._make_subparam('baz', 'STRING', 'abc') + klass = self._get_target_class() + parm = klass.positional(sub_1, sub_2) + self.assertEqual(parm.name, None) + self.assertEqual(parm.struct_types, {'bar': 'INT64', 'baz': 'STRING'}) + self.assertEqual(parm.struct_values, {'bar': 123, 'baz': 'abc'}) + + def test_from_api_repr_w_name(self): + RESOURCE = { + 'name': 'foo', + 'parameterType': { + 'structTypes': [ + {'name': 'bar', 'type': 'INT64'}, + {'name': 'baz', 'type': 'STRING'}, + ], + }, + 'parameterValue': { + 'structValues': {'bar': 123, 'baz': 'abc'}, + }, + } + klass = self._get_target_class() + parm = klass.from_api_repr(RESOURCE) + self.assertEqual(parm.name, 'foo') + self.assertEqual(parm.struct_types, {'bar': 'INT64', 'baz': 'STRING'}) + self.assertEqual(parm.struct_values, {'bar': 123, 'baz': 'abc'}) + + def test_from_api_repr_wo_name(self): + RESOURCE = { + 'parameterType': { + 'structTypes': [ + {'name': 'bar', 'type': 'INT64'}, + {'name': 'baz', 'type': 'STRING'}, + ], + }, + 'parameterValue': { + 'structValues': {'bar': 123, 'baz': 'abc'}, + }, + } + klass = self._get_target_class() + parm = klass.from_api_repr(RESOURCE) + self.assertEqual(parm.name, None) + self.assertEqual(parm.struct_types, {'bar': 'INT64', 'baz': 'STRING'}) + self.assertEqual(parm.struct_values, {'bar': 123, 'baz': 'abc'}) + + def test_to_api_repr_w_name(self): + EXPECTED = { + 'name': 'foo', + 'parameterType': { + 'structTypes': [ + {'name': 'bar', 'type': 'INT64'}, + {'name': 'baz', 'type': 'STRING'}, + ], + }, + 'parameterValue': { + 'structValues': {'bar': 123, 'baz': 'abc'}, + }, + } + sub_1 = self._make_subparam('bar', 'INT64', 123) + sub_2 = self._make_subparam('baz', 'STRING', 'abc') + parm = self._make_one('foo', sub_1, sub_2) + self.assertEqual(parm.to_api_repr(), EXPECTED) + + def test_to_api_repr_wo_name(self): + EXPECTED = { + 'parameterType': { + 'structTypes': [ + {'name': 'bar', 'type': 'INT64'}, + {'name': 'baz', 'type': 'STRING'}, + ], + }, + 'parameterValue': { + 'structValues': {'bar': 123, 'baz': 'abc'}, + }, + } + sub_1 = self._make_subparam('bar', 'INT64', 123) + sub_2 = self._make_subparam('baz', 'STRING', 'abc') + klass = self._get_target_class() + parm = klass.positional(sub_1, sub_2) + self.assertEqual(parm.to_api_repr(), EXPECTED) + + class _Field(object): def __init__(self, mode, name='unknown', field_type='UNKNOWN', fields=()): From fcff570c9f4945663f9b4d893ca7f833131e8985 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 29 Nov 2016 12:20:51 -0500 Subject: [PATCH 05/20] Remove '_build_udf_resources' helper. It is just a list comprehension, really. --- bigquery/google/cloud/bigquery/_helpers.py | 15 --------------- bigquery/google/cloud/bigquery/job.py | 7 ++++--- bigquery/google/cloud/bigquery/query.py | 6 ++++-- 3 files changed, 8 insertions(+), 20 deletions(-) diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index 2889b811e576..c0cb95791de6 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -228,21 +228,6 @@ def __set__(self, instance, value): instance._udf_resources = tuple(value) -def _build_udf_resources(resources): - """ - :type resources: sequence of :class:`UDFResource` - :param resources: fields to be appended. - - :rtype: mapping - :returns: a mapping describing userDefinedFunctionResources for the query. - """ - udfs = [] - for resource in resources: - udf = {resource.udf_type: resource.value} - udfs.append(udf) - return udfs - - class AbstractQueryParameter(object): """Base class for named / positional query parameters. """ diff --git a/bigquery/google/cloud/bigquery/job.py b/bigquery/google/cloud/bigquery/job.py index 203dd2df6dd0..d933e5ce9fef 100644 --- a/bigquery/google/cloud/bigquery/job.py +++ b/bigquery/google/cloud/bigquery/job.py @@ -26,7 +26,6 @@ from google.cloud.bigquery._helpers import UDFResourcesProperty from google.cloud.bigquery._helpers import _EnumProperty from google.cloud.bigquery._helpers import _TypedProperty -from google.cloud.bigquery._helpers import _build_udf_resources class Compression(_EnumProperty): @@ -1032,8 +1031,10 @@ def _populate_config_resource(self, configuration): if self.maximum_bytes_billed is not None: configuration['maximumBytesBilled'] = self.maximum_bytes_billed if len(self._udf_resources) > 0: - configuration[self._UDF_KEY] = _build_udf_resources( - self._udf_resources) + configuration[self._UDF_KEY] = [ + {udf_resource.udf_type: udf_resource.value} + for udf_resource in self._udf_resources + ] def _build_resource(self): """Generate a resource for :meth:`begin`.""" diff --git a/bigquery/google/cloud/bigquery/query.py b/bigquery/google/cloud/bigquery/query.py index fa1b1da63883..579ca34a4831 100644 --- a/bigquery/google/cloud/bigquery/query.py +++ b/bigquery/google/cloud/bigquery/query.py @@ -21,7 +21,6 @@ from google.cloud.bigquery.dataset import Dataset from google.cloud.bigquery.job import QueryJob from google.cloud.bigquery.table import _parse_schema_resource -from google.cloud.bigquery._helpers import _build_udf_resources from google.cloud.bigquery._helpers import UDFResourcesProperty @@ -314,7 +313,10 @@ def _build_resource(self): resource['dryRun'] = self.dry_run if len(self._udf_resources) > 0: - resource[self._UDF_KEY] = _build_udf_resources(self._udf_resources) + resource[self._UDF_KEY] = [ + {udf_resource.udf_type: udf_resource.value} + for udf_resource in self._udf_resources + ] return resource From 2e062289c2bd981dde648ff90afa7db05248b013 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 29 Nov 2016 12:21:16 -0500 Subject: [PATCH 06/20] Add 'QueryParametersProperty' descriptor class. --- bigquery/google/cloud/bigquery/_helpers.py | 17 ++++++ bigquery/unit_tests/test__helpers.py | 68 ++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index c0cb95791de6..dc97a84652a9 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -467,3 +467,20 @@ def to_api_repr(self): if self.name is not None: resource['name'] = self.name return resource + + +class QueryParametersProperty(object): + """Custom property type, holding query parameter instances.""" + + def __get__(self, instance, owner): + """Descriptor protocol: accessor""" + if instance is None: + return self + return list(instance._query_parameters) + + def __set__(self, instance, value): + """Descriptor protocol: mutator""" + if not all(isinstance(u, AbstractQueryParameter) for u in value): + raise ValueError( + "query parameters must be derived from AbstractQueryParameter") + instance._query_parameters = tuple(value) diff --git a/bigquery/unit_tests/test__helpers.py b/bigquery/unit_tests/test__helpers.py index c14eca67ecf7..f00ec5bcfbb5 100644 --- a/bigquery/unit_tests/test__helpers.py +++ b/bigquery/unit_tests/test__helpers.py @@ -780,6 +780,74 @@ def test_to_api_repr_wo_name(self): self.assertEqual(parm.to_api_repr(), EXPECTED) +class Test_QueryParametersProperty(unittest.TestCase): + + @staticmethod + def _get_target_class(): + from google.cloud.bigquery._helpers import QueryParametersProperty + return QueryParametersProperty + + def _make_one(self, *args, **kw): + return self._get_target_class()(*args, **kw) + + def _descriptor_and_klass(self): + descriptor = self._make_one() + + class _Test(object): + _query_parameters = () + query_parameters = descriptor + + return descriptor, _Test + + def test_class_getter(self): + descriptor, klass = self._descriptor_and_klass() + self.assertIs(klass.query_parameters, descriptor) + + def test_instance_getter_empty(self): + _, klass = self._descriptor_and_klass() + instance = klass() + self.assertEqual(instance.query_parameters, []) + + def test_instance_getter_w_non_empty_list(self): + from google.cloud.bigquery._helpers import ScalarQueryParameter + query_parameters = [ScalarQueryParameter("foo", 'INT64', 123)] + _, klass = self._descriptor_and_klass() + instance = klass() + instance._query_parameters = tuple(query_parameters) + + self.assertEqual(instance.query_parameters, query_parameters) + + def test_instance_setter_w_empty_list(self): + from google.cloud.bigquery._helpers import ScalarQueryParameter + query_parameters = [ScalarQueryParameter("foo", 'INT64', 123)] + _, klass = self._descriptor_and_klass() + instance = klass() + instance._query_parameters = query_parameters + + instance.query_parameters = [] + + self.assertEqual(instance.query_parameters, []) + + def test_instance_setter_w_valid_udf(self): + from google.cloud.bigquery._helpers import ScalarQueryParameter + query_parameters = [ScalarQueryParameter("foo", 'INT64', 123)] + _, klass = self._descriptor_and_klass() + instance = klass() + + instance.query_parameters = query_parameters + + self.assertEqual(instance.query_parameters, query_parameters) + + def test_instance_setter_w_bad_udfs(self): + _, klass = self._descriptor_and_klass() + instance = klass() + + with self.assertRaises(ValueError): + instance.query_parameters = ["foo"] + + self.assertEqual(instance.query_parameters, []) + + class _Field(object): def __init__(self, mode, name='unknown', field_type='UNKNOWN', fields=()): From 968582f9653848d8c0e2470ce384675e775a1bb5 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 29 Nov 2016 13:34:57 -0500 Subject: [PATCH 07/20] Add 'query_parameters' property to 'QueryResults' and 'QueryJob'. --- bigquery/google/cloud/bigquery/job.py | 23 +++- bigquery/google/cloud/bigquery/query.py | 22 +++- bigquery/unit_tests/test_job.py | 148 +++++++++++++++++++++++- bigquery/unit_tests/test_query.py | 118 ++++++++++++++++++- 4 files changed, 306 insertions(+), 5 deletions(-) diff --git a/bigquery/google/cloud/bigquery/job.py b/bigquery/google/cloud/bigquery/job.py index d933e5ce9fef..5eff2b74ef90 100644 --- a/bigquery/google/cloud/bigquery/job.py +++ b/bigquery/google/cloud/bigquery/job.py @@ -23,6 +23,7 @@ from google.cloud.bigquery.table import Table from google.cloud.bigquery.table import _build_schema_resource from google.cloud.bigquery.table import _parse_schema_resource +from google.cloud.bigquery._helpers import QueryParametersProperty from google.cloud.bigquery._helpers import UDFResourcesProperty from google.cloud.bigquery._helpers import _EnumProperty from google.cloud.bigquery._helpers import _TypedProperty @@ -908,14 +909,23 @@ class QueryJob(_AsyncJob): :param udf_resources: An iterable of :class:`google.cloud.bigquery._helpers.UDFResource` (empty by default) + + :type query_parameters: tuple + :param query_parameters: + An iterable of + :class:`google.cloud.bigquery._helpers.AbstractQueryParameter` + (empty by default) """ _JOB_TYPE = 'query' _UDF_KEY = 'userDefinedFunctionResources' + _QUERY_PARAMETERS_KEY = 'queryParameters' - def __init__(self, name, query, client, udf_resources=()): + def __init__(self, name, query, client, + udf_resources=(), query_parameters=()): super(QueryJob, self).__init__(name, client) self.query = query self.udf_resources = udf_resources + self.query_parameters = query_parameters self._configuration = _AsyncQueryConfiguration() allow_large_results = _TypedProperty('allow_large_results', bool) @@ -948,6 +958,8 @@ def __init__(self, name, query, client, udf_resources=()): https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.query.priority """ + query_parameters = QueryParametersProperty() + udf_resources = UDFResourcesProperty() use_query_cache = _TypedProperty('use_query_cache', bool) @@ -1035,6 +1047,15 @@ def _populate_config_resource(self, configuration): {udf_resource.udf_type: udf_resource.value} for udf_resource in self._udf_resources ] + if len(self._query_parameters) > 0: + configuration[self._QUERY_PARAMETERS_KEY] = [ + query_parameter.to_api_repr() + for query_parameter in self._query_parameters + ] + if self._query_parameters[0].name is None: + configuration['parameterMode'] = 'POSITIONAL' + else: + configuration['parameterMode'] = 'NAMED' def _build_resource(self): """Generate a resource for :meth:`begin`.""" diff --git a/bigquery/google/cloud/bigquery/query.py b/bigquery/google/cloud/bigquery/query.py index 579ca34a4831..95d2eabdbdbe 100644 --- a/bigquery/google/cloud/bigquery/query.py +++ b/bigquery/google/cloud/bigquery/query.py @@ -21,6 +21,7 @@ from google.cloud.bigquery.dataset import Dataset from google.cloud.bigquery.job import QueryJob from google.cloud.bigquery.table import _parse_schema_resource +from google.cloud.bigquery._helpers import QueryParametersProperty from google.cloud.bigquery._helpers import UDFResourcesProperty @@ -52,16 +53,24 @@ class QueryResults(object): :param udf_resources: An iterable of :class:`google.cloud.bigquery.job.UDFResource` (empty by default) + + :type query_parameters: tuple + :param query_parameters: + An iterable of + :class:`google.cloud.bigquery._helpers.AbstractQueryParameter` + (empty by default) """ _UDF_KEY = 'userDefinedFunctionResources' + _QUERY_PARAMETERS_KEY = 'queryParameters' - def __init__(self, query, client, udf_resources=()): + def __init__(self, query, client, udf_resources=(), query_parameters=()): self._client = client self._properties = {} self.query = query self._configuration = _SyncQueryConfiguration() self.udf_resources = udf_resources + self.query_parameters = query_parameters self._job = None @classmethod @@ -257,6 +266,8 @@ def schema(self): https://cloud.google.com/bigquery/docs/reference/v2/jobs/query#preserveNulls """ + query_parameters = QueryParametersProperty() + timeout_ms = _TypedProperty('timeout_ms', six.integer_types) """See: https://cloud.google.com/bigquery/docs/reference/v2/jobs/query#timeoutMs @@ -317,6 +328,15 @@ def _build_resource(self): {udf_resource.udf_type: udf_resource.value} for udf_resource in self._udf_resources ] + if len(self._query_parameters) > 0: + resource[self._QUERY_PARAMETERS_KEY] = [ + query_parameter.to_api_repr() + for query_parameter in self._query_parameters + ] + if self._query_parameters[0].name is None: + resource['parameterMode'] = 'POSITIONAL' + else: + resource['parameterMode'] = 'NAMED' return resource diff --git a/bigquery/unit_tests/test_job.py b/bigquery/unit_tests/test_job.py index c73715262ba4..6a5ef792b782 100644 --- a/bigquery/unit_tests/test_job.py +++ b/bigquery/unit_tests/test_job.py @@ -1287,12 +1287,31 @@ def _verifyIntegerResourceProperties(self, job, config): else: self.assertIsNone(job.maximum_bytes_billed) + def _verifyUDFResources(self, job, config): + udf_resources = config.get('userDefinedFunctionResources', ()) + self.assertEqual(len(job.udf_resources), len(udf_resources)) + for found, expected in zip(job.udf_resources, udf_resources): + if 'resourceUri' in expected: + self.assertEqual(found.udf_type, 'resourceUri') + self.assertEqual(found.value, expected['resourceUri']) + else: + self.assertEqual(found.udf_type, 'inlineCode') + self.assertEqual(found.value, expected['inlineCode']) + + def _verifyQueryParameters(self, job, config): + query_parameters = config.get('queryParameters', ()) + self.assertEqual(len(job.query_parameters), len(query_parameters)) + for found, expected in zip(job.query_parameters, query_parameters): + self.assertEqual(found.to_api_repr(), expected) + def _verifyResourceProperties(self, job, resource): self._verifyReadonlyResourceProperties(job, resource) config = resource.get('configuration', {}).get('query') self._verifyBooleanResourceProperties(job, config) self._verifyIntegerResourceProperties(job, config) + self._verifyUDFResources(job, config) + self._verifyQueryParameters(job, config) self.assertEqual(job.query, config['query']) if 'createDisposition' in config: @@ -1330,7 +1349,7 @@ def _verifyResourceProperties(self, job, resource): else: self.assertIsNone(job.write_disposition) - def test_ctor(self): + def test_ctor_defaults(self): client = _Client(self.PROJECT) job = self._make_one(self.JOB_NAME, self.QUERY, client) self.assertEqual(job.query, self.QUERY) @@ -1356,6 +1375,23 @@ def test_ctor(self): self.assertIsNone(job.maximum_billing_tier) self.assertIsNone(job.maximum_bytes_billed) + def test_ctor_w_udf_resources(self): + from google.cloud.bigquery._helpers import UDFResource + RESOURCE_URI = 'gs://some-bucket/js/lib.js' + udf_resources = [UDFResource("resourceUri", RESOURCE_URI)] + client = _Client(self.PROJECT) + job = self._make_one(self.JOB_NAME, self.QUERY, client, + udf_resources=udf_resources) + self.assertEqual(job.udf_resources, udf_resources) + + def test_ctor_w_query_parameters(self): + from google.cloud.bigquery._helpers import ScalarQueryParameter + query_parameters = [ScalarQueryParameter("foo", 'INT64', 123)] + client = _Client(self.PROJECT) + job = self._make_one(self.JOB_NAME, self.QUERY, client, + query_parameters=query_parameters) + self.assertEqual(job.query_parameters, query_parameters) + def test_from_api_repr_missing_identity(self): self._setUpConstants() client = _Client(self.PROJECT) @@ -1520,7 +1556,7 @@ def test_begin_w_alternate_client(self): self.assertEqual(req['data'], SENT) self._verifyResourceProperties(job, RESOURCE) - def test_begin_w_bound_client_and_udf(self): + def test_begin_w_udf(self): from google.cloud.bigquery._helpers import UDFResource RESOURCE_URI = 'gs://some-bucket/js/lib.js' PATH = 'projects/%s/jobs' % self.PROJECT @@ -1530,8 +1566,15 @@ def test_begin_w_bound_client_and_udf(self): del RESOURCE['etag'] del RESOURCE['selfLink'] del RESOURCE['user_email'] + RESOURCE['configuration']['query']['userDefinedFunctionResources'] = [ + {'resourceUri': RESOURCE_URI}, + ] conn = _Connection(RESOURCE) client = _Client(project=self.PROJECT, connection=conn) + udf_resources = [ + UDFResource("resourceUri", RESOURCE_URI), + UDFResource("inlineCode", INLINE_UDF_CODE), + ] job = self._make_one(self.JOB_NAME, self.QUERY, client, udf_resources=[ UDFResource("resourceUri", RESOURCE_URI) @@ -1561,6 +1604,107 @@ def test_begin_w_bound_client_and_udf(self): self.assertEqual(req['data'], SENT) self._verifyResourceProperties(job, RESOURCE) + def test_begin_w_named_query_parameter(self): + from google.cloud.bigquery._helpers import ScalarQueryParameter + query_parameters = [ScalarQueryParameter('foo', 'INT64', 123)] + PATH = 'projects/%s/jobs' % self.PROJECT + RESOURCE = self._makeResource() + # Ensure None for missing server-set props + del RESOURCE['statistics']['creationTime'] + del RESOURCE['etag'] + del RESOURCE['selfLink'] + del RESOURCE['user_email'] + config = RESOURCE['configuration']['query'] + config['parameterMode'] = 'NAMED' + config['queryParameters'] = [ + { + 'name': 'foo', + 'parameterType': { + 'type': 'INT64', + }, + 'parameterValue': { + 'value': 123, + }, + }, + ] + conn = _Connection(RESOURCE) + client = _Client(project=self.PROJECT, connection=conn) + job = self._make_one(self.JOB_NAME, self.QUERY, client, + query_parameters=query_parameters) + + job.begin() + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'POST') + self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(job.query_parameters, query_parameters) + SENT = { + 'jobReference': { + 'projectId': self.PROJECT, + 'jobId': self.JOB_NAME, + }, + 'configuration': { + 'query': { + 'query': self.QUERY, + 'parameterMode': 'NAMED', + 'queryParameters': config['queryParameters'], + }, + }, + } + self.assertEqual(req['data'], SENT) + self._verifyResourceProperties(job, RESOURCE) + + def test_begin_w_positional_query_parameter(self): + from google.cloud.bigquery._helpers import ScalarQueryParameter + query_parameters = [ScalarQueryParameter.positional('INT64', 123)] + PATH = 'projects/%s/jobs' % self.PROJECT + RESOURCE = self._makeResource() + # Ensure None for missing server-set props + del RESOURCE['statistics']['creationTime'] + del RESOURCE['etag'] + del RESOURCE['selfLink'] + del RESOURCE['user_email'] + config = RESOURCE['configuration']['query'] + config['parameterMode'] = 'POSITIONAL' + config['queryParameters'] = [ + { + 'parameterType': { + 'type': 'INT64', + }, + 'parameterValue': { + 'value': 123, + }, + }, + ] + conn = _Connection(RESOURCE) + client = _Client(project=self.PROJECT, connection=conn) + job = self._make_one(self.JOB_NAME, self.QUERY, client, + query_parameters=query_parameters) + + job.begin() + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'POST') + self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(job.query_parameters, query_parameters) + SENT = { + 'jobReference': { + 'projectId': self.PROJECT, + 'jobId': self.JOB_NAME, + }, + 'configuration': { + 'query': { + 'query': self.QUERY, + 'parameterMode': 'POSITIONAL', + 'queryParameters': config['queryParameters'], + }, + }, + } + self.assertEqual(req['data'], SENT) + self._verifyResourceProperties(job, RESOURCE) + def test_exists_miss_w_bound_client(self): PATH = 'projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) conn = _Connection() diff --git a/bigquery/unit_tests/test_query.py b/bigquery/unit_tests/test_query.py index 43aedf334ce0..f12c33c9f36b 100644 --- a/bigquery/unit_tests/test_query.py +++ b/bigquery/unit_tests/test_query.py @@ -100,6 +100,23 @@ def _verifyRows(self, query, resource): self.assertEqual(f_row, tuple([cell['v'] for cell in e_row['f']])) + def _verifyUDFResources(self, query, resource): + udf_resources = resource.get('userDefinedFunctionResources', ()) + self.assertEqual(len(query.udf_resources), len(udf_resources)) + for found, expected in zip(query.udf_resources, udf_resources): + if 'resourceUri' in expected: + self.assertEqual(found.udf_type, 'resourceUri') + self.assertEqual(found.value, expected['resourceUri']) + else: + self.assertEqual(found.udf_type, 'inlineCode') + self.assertEqual(found.value, expected['inlineCode']) + + def _verifyQueryParameters(self, query, resource): + query_parameters = resource.get('queryParameters', ()) + self.assertEqual(len(query.query_parameters), len(query_parameters)) + for found, expected in zip(query.query_parameters, query_parameters): + self.assertEqual(found.to_api_repr(), expected) + def _verifyResourceProperties(self, query, resource): self.assertEqual(query.cache_hit, resource.get('cacheHit')) self.assertEqual(query.complete, resource.get('jobComplete')) @@ -114,10 +131,12 @@ def _verifyResourceProperties(self, query, resource): else: self.assertIsNone(query.name) + self._verifyUDFResources(query, resource) + self._verifyQueryParameters(query, resource) self._verifySchema(query, resource) self._verifyRows(query, resource) - def test_ctor(self): + def test_ctor_defaults(self): client = _Client(self.PROJECT) query = self._make_one(self.QUERY, client) self.assertEqual(query.query, self.QUERY) @@ -128,10 +147,12 @@ def test_ctor(self): self.assertIsNone(query.errors) self.assertIsNone(query.name) self.assertIsNone(query.page_token) + self.assertEqual(query.query_parameters, []) self.assertEqual(query.rows, []) self.assertIsNone(query.schema) self.assertIsNone(query.total_rows) self.assertIsNone(query.total_bytes_processed) + self.assertEqual(query.udf_resources, []) self.assertIsNone(query.default_dataset) self.assertIsNone(query.max_results) @@ -139,6 +160,22 @@ def test_ctor(self): self.assertIsNone(query.use_query_cache) self.assertIsNone(query.use_legacy_sql) + def test_ctor_w_udf_resources(self): + from google.cloud.bigquery._helpers import UDFResource + RESOURCE_URI = 'gs://some-bucket/js/lib.js' + udf_resources = [UDFResource("resourceUri", RESOURCE_URI)] + client = _Client(self.PROJECT) + query = self._make_one(self.QUERY, client, udf_resources=udf_resources) + self.assertEqual(query.udf_resources, udf_resources) + + def test_ctor_w_query_parameters(self): + from google.cloud.bigquery._helpers import ScalarQueryParameter + query_parameters = [ScalarQueryParameter("foo", 'INT64', 123)] + client = _Client(self.PROJECT) + query = self._make_one(self.QUERY, client, + query_parameters=query_parameters) + self.assertEqual(query.query_parameters, query_parameters) + def test_from_query_job(self): from google.cloud.bigquery.dataset import Dataset from google.cloud.bigquery.job import QueryJob @@ -293,6 +330,9 @@ def test_run_w_inline_udf(self): INLINE_UDF_CODE = 'var someCode = "here";' PATH = 'projects/%s/queries' % self.PROJECT RESOURCE = self._makeResource(complete=False) + RESOURCE['userDefinedFunctionResources'] = [ + {'inlineCode': INLINE_UDF_CODE}, + ] conn = _Connection(RESOURCE) client = _Client(project=self.PROJECT, connection=conn) query = self._make_one(self.QUERY, client) @@ -315,6 +355,9 @@ def test_run_w_udf_resource_uri(self): RESOURCE_URI = 'gs://some-bucket/js/lib.js' PATH = 'projects/%s/queries' % self.PROJECT RESOURCE = self._makeResource(complete=False) + RESOURCE['userDefinedFunctionResources'] = [ + {'resourceUri': RESOURCE_URI}, + ] conn = _Connection(RESOURCE) client = _Client(project=self.PROJECT, connection=conn) query = self._make_one(self.QUERY, client) @@ -338,6 +381,10 @@ def test_run_w_mixed_udfs(self): INLINE_UDF_CODE = 'var someCode = "here";' PATH = 'projects/%s/queries' % self.PROJECT RESOURCE = self._makeResource(complete=False) + RESOURCE['userDefinedFunctionResources'] = [ + {'resourceUri': RESOURCE_URI}, + {'inlineCode': INLINE_UDF_CODE}, + ] conn = _Connection(RESOURCE) client = _Client(project=self.PROJECT, connection=conn) query = self._make_one(self.QUERY, client) @@ -360,6 +407,75 @@ def test_run_w_mixed_udfs(self): self.assertEqual(req['data'], SENT) self._verifyResourceProperties(query, RESOURCE) + def test_run_w_named_query_paramter(self): + from google.cloud.bigquery._helpers import ScalarQueryParameter + PATH = 'projects/%s/queries' % self.PROJECT + RESOURCE = self._makeResource(complete=False) + RESOURCE['parameterMode'] = 'NAMED' + RESOURCE['queryParameters'] = [ + { + 'name': 'foo', + 'parameterType': { + 'type': 'INT64', + }, + 'parameterValue': { + 'value': 123, + }, + }, + ] + query_parameters = [ScalarQueryParameter('foo', 'INT64', 123)] + conn = _Connection(RESOURCE) + client = _Client(project=self.PROJECT, connection=conn) + query = self._make_one(self.QUERY, client, + query_parameters=query_parameters) + query.run() + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'POST') + self.assertEqual(req['path'], '/%s' % PATH) + SENT = { + 'query': self.QUERY, + 'parameterMode': 'NAMED', + 'queryParameters': RESOURCE['queryParameters'], + } + self.assertEqual(req['data'], SENT) + self._verifyResourceProperties(query, RESOURCE) + + def test_run_w_positional_query_paramter(self): + from google.cloud.bigquery._helpers import ScalarQueryParameter + PATH = 'projects/%s/queries' % self.PROJECT + RESOURCE = self._makeResource(complete=False) + RESOURCE['parameterMode'] = 'POSITIONAL' + RESOURCE['queryParameters'] = [ + { + 'parameterType': { + 'type': 'INT64', + }, + 'parameterValue': { + 'value': 123, + }, + }, + ] + query_parameters = [ScalarQueryParameter.positional('INT64', 123)] + conn = _Connection(RESOURCE) + client = _Client(project=self.PROJECT, connection=conn) + query = self._make_one(self.QUERY, client, + query_parameters=query_parameters) + query.run() + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'POST') + self.assertEqual(req['path'], '/%s' % PATH) + SENT = { + 'query': self.QUERY, + 'parameterMode': 'POSITIONAL', + 'queryParameters': RESOURCE['queryParameters'], + } + self.assertEqual(req['data'], SENT) + self._verifyResourceProperties(query, RESOURCE) + def test_fetch_data_query_not_yet_run(self): conn = _Connection() client = _Client(project=self.PROJECT, connection=conn) From 52b25ecf23ccb14a08697880b12ebcc6f3af30c3 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 29 Nov 2016 13:46:59 -0500 Subject: [PATCH 08/20] Plumb 'udf_resources'/'query_parameters' through client query factories. --- bigquery/google/cloud/bigquery/client.py | 35 ++++++++-- bigquery/unit_tests/test_client.py | 89 ++++++++++++++++++++++-- 2 files changed, 114 insertions(+), 10 deletions(-) diff --git a/bigquery/google/cloud/bigquery/client.py b/bigquery/google/cloud/bigquery/client.py index d16a9d9349d2..4af5b8fc4910 100644 --- a/bigquery/google/cloud/bigquery/client.py +++ b/bigquery/google/cloud/bigquery/client.py @@ -275,7 +275,8 @@ def extract_table_to_storage(self, job_name, source, *destination_uris): return ExtractTableToStorageJob(job_name, source, destination_uris, client=self) - def run_async_query(self, job_name, query): + def run_async_query(self, job_name, query, + udf_resources=(), query_parameters=()): """Construct a job for running a SQL query asynchronously. See: @@ -287,21 +288,47 @@ def run_async_query(self, job_name, query): :type query: str :param query: SQL query to be executed + :type udf_resources: tuple + :param udf_resources: An iterable of + :class:`google.cloud.bigquery._helpers.UDFResource` + (empty by default) + + :type query_parameters: tuple + :param query_parameters: + An iterable of + :class:`google.cloud.bigquery._helpers.AbstractQueryParameter` + (empty by default) + :rtype: :class:`google.cloud.bigquery.job.QueryJob` :returns: a new ``QueryJob`` instance """ - return QueryJob(job_name, query, client=self) + return QueryJob(job_name, query, client=self, + udf_resources=udf_resources, + query_parameters=query_parameters) - def run_sync_query(self, query): + def run_sync_query(self, query, udf_resources=(), query_parameters=()): """Run a SQL query synchronously. :type query: str :param query: SQL query to be executed + :type udf_resources: tuple + :param udf_resources: An iterable of + :class:`google.cloud.bigquery._helpers.UDFResource` + (empty by default) + + :type query_parameters: tuple + :param query_parameters: + An iterable of + :class:`google.cloud.bigquery._helpers.AbstractQueryParameter` + (empty by default) + :rtype: :class:`google.cloud.bigquery.query.QueryResults` :returns: a new ``QueryResults`` instance """ - return QueryResults(query, client=self) + return QueryResults(query, client=self, + udf_resources=udf_resources, + query_parameters=query_parameters) # pylint: disable=unused-argument diff --git a/bigquery/unit_tests/test_client.py b/bigquery/unit_tests/test_client.py index 61ad81227aee..45f49b0f831e 100644 --- a/bigquery/unit_tests/test_client.py +++ b/bigquery/unit_tests/test_client.py @@ -470,7 +470,7 @@ def test_extract_table_to_storage(self): self.assertEqual(job.source, source) self.assertEqual(list(job.destination_uris), [DESTINATION]) - def test_run_async_query(self): + def test_run_async_query_defaults(self): from google.cloud.bigquery.job import QueryJob PROJECT = 'PROJECT' JOB = 'job_name' @@ -483,19 +483,96 @@ def test_run_async_query(self): self.assertIs(job._client, client) self.assertEqual(job.name, JOB) self.assertEqual(job.query, QUERY) + self.assertEqual(job.udf_resources, []) + self.assertEqual(job.query_parameters, []) - def test_run_sync_query(self): - from google.cloud.bigquery.query import QueryResults + def test_run_async_w_udf_resources(self): + from google.cloud.bigquery._helpers import UDFResource + from google.cloud.bigquery.job import QueryJob + RESOURCE_URI = 'gs://some-bucket/js/lib.js' + PROJECT = 'PROJECT' + JOB = 'job_name' + QUERY = 'select count(*) from persons' + creds = _Credentials() + http = object() + client = self._make_one(project=PROJECT, credentials=creds, http=http) + udf_resources = [UDFResource("resourceUri", RESOURCE_URI)] + job = client.run_async_query(JOB, QUERY, udf_resources=udf_resources) + self.assertIsInstance(job, QueryJob) + self.assertIs(job._client, client) + self.assertEqual(job.name, JOB) + self.assertEqual(job.query, QUERY) + self.assertEqual(job.udf_resources, udf_resources) + self.assertEqual(job.query_parameters, []) + + def test_run_async_w_query_parameters(self): + from google.cloud.bigquery._helpers import ScalarQueryParameter + from google.cloud.bigquery.job import QueryJob PROJECT = 'PROJECT' + JOB = 'job_name' QUERY = 'select count(*) from persons' creds = _Credentials() http = object() client = self._make_one(project=PROJECT, credentials=creds, http=http) - job = client.run_sync_query(QUERY) - self.assertIsInstance(job, QueryResults) + query_parameters = [ScalarQueryParameter('foo', 'INT64', 123)] + job = client.run_async_query(JOB, QUERY, + query_parameters=query_parameters) + self.assertIsInstance(job, QueryJob) self.assertIs(job._client, client) - self.assertIsNone(job.name) + self.assertEqual(job.name, JOB) self.assertEqual(job.query, QUERY) + self.assertEqual(job.udf_resources, []) + self.assertEqual(job.query_parameters, query_parameters) + + def test_run_sync_query_defaults(self): + from google.cloud.bigquery.query import QueryResults + PROJECT = 'PROJECT' + QUERY = 'select count(*) from persons' + creds = _Credentials() + http = object() + client = self._make_one(project=PROJECT, credentials=creds, http=http) + query = client.run_sync_query(QUERY) + self.assertIsInstance(query, QueryResults) + self.assertIs(query._client, client) + self.assertIsNone(query.name) + self.assertEqual(query.query, QUERY) + self.assertEqual(query.udf_resources, []) + self.assertEqual(query.query_parameters, []) + + def test_run_sync_query_w_udf_resources(self): + from google.cloud.bigquery._helpers import UDFResource + from google.cloud.bigquery.query import QueryResults + RESOURCE_URI = 'gs://some-bucket/js/lib.js' + PROJECT = 'PROJECT' + QUERY = 'select count(*) from persons' + creds = _Credentials() + http = object() + client = self._make_one(project=PROJECT, credentials=creds, http=http) + udf_resources = [UDFResource("resourceUri", RESOURCE_URI)] + query = client.run_sync_query(QUERY, udf_resources=udf_resources) + self.assertIsInstance(query, QueryResults) + self.assertIs(query._client, client) + self.assertIsNone(query.name) + self.assertEqual(query.query, QUERY) + self.assertEqual(query.udf_resources, udf_resources) + self.assertEqual(query.query_parameters, []) + + def test_run_sync_query_w_query_parameters(self): + from google.cloud.bigquery._helpers import ScalarQueryParameter + from google.cloud.bigquery.query import QueryResults + PROJECT = 'PROJECT' + QUERY = 'select count(*) from persons' + creds = _Credentials() + http = object() + client = self._make_one(project=PROJECT, credentials=creds, http=http) + query_parameters = [ScalarQueryParameter('foo', 'INT64', 123)] + query = client.run_sync_query(QUERY, query_parameters=query_parameters) + self.assertIsInstance(query, QueryResults) + self.assertIs(query._client, client) + self.assertIsNone(query.name) + self.assertEqual(query.query, QUERY) + self.assertEqual(query.udf_resources, []) + self.assertEqual(query.query_parameters, query_parameters) class _Credentials(object): From 90899079a3233b1b89409f1fa2195230aa9d0d14 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 29 Nov 2016 13:49:03 -0500 Subject: [PATCH 09/20] Expose concrete query parameter classes as package APIs. --- bigquery/google/cloud/bigquery/__init__.py | 3 +++ bigquery/unit_tests/test_job.py | 15 ++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/bigquery/google/cloud/bigquery/__init__.py b/bigquery/google/cloud/bigquery/__init__.py index cde9432d83e2..1fbebba5a089 100644 --- a/bigquery/google/cloud/bigquery/__init__.py +++ b/bigquery/google/cloud/bigquery/__init__.py @@ -28,3 +28,6 @@ from google.cloud.bigquery.dataset import Dataset from google.cloud.bigquery.schema import SchemaField from google.cloud.bigquery.table import Table +from google.cloud.bigquery._helpers import ArrayQueryParameter +from google.cloud.bigquery._helpers import ScalarQueryParameter +from google.cloud.bigquery._helpers import StructQueryParameter diff --git a/bigquery/unit_tests/test_job.py b/bigquery/unit_tests/test_job.py index 6a5ef792b782..b182e99bb5cc 100644 --- a/bigquery/unit_tests/test_job.py +++ b/bigquery/unit_tests/test_job.py @@ -1559,6 +1559,7 @@ def test_begin_w_alternate_client(self): def test_begin_w_udf(self): from google.cloud.bigquery._helpers import UDFResource RESOURCE_URI = 'gs://some-bucket/js/lib.js' + INLINE_UDF_CODE = 'var someCode = "here";' PATH = 'projects/%s/jobs' % self.PROJECT RESOURCE = self._makeResource() # Ensure None for missing server-set props @@ -1568,6 +1569,7 @@ def test_begin_w_udf(self): del RESOURCE['user_email'] RESOURCE['configuration']['query']['userDefinedFunctionResources'] = [ {'resourceUri': RESOURCE_URI}, + {'inlineCode': INLINE_UDF_CODE}, ] conn = _Connection(RESOURCE) client = _Client(project=self.PROJECT, connection=conn) @@ -1576,9 +1578,7 @@ def test_begin_w_udf(self): UDFResource("inlineCode", INLINE_UDF_CODE), ] job = self._make_one(self.JOB_NAME, self.QUERY, client, - udf_resources=[ - UDFResource("resourceUri", RESOURCE_URI) - ]) + udf_resources=udf_resources) job.begin() @@ -1586,8 +1586,7 @@ def test_begin_w_udf(self): req = conn._requested[0] self.assertEqual(req['method'], 'POST') self.assertEqual(req['path'], '/%s' % PATH) - self.assertEqual(job.udf_resources, - [UDFResource("resourceUri", RESOURCE_URI)]) + self.assertEqual(job.udf_resources, udf_resources) SENT = { 'jobReference': { 'projectId': self.PROJECT, @@ -1596,8 +1595,10 @@ def test_begin_w_udf(self): 'configuration': { 'query': { 'query': self.QUERY, - 'userDefinedFunctionResources': - [{'resourceUri': RESOURCE_URI}] + 'userDefinedFunctionResources': [ + {'resourceUri': RESOURCE_URI}, + {'inlineCode': INLINE_UDF_CODE}, + ] }, }, } From 9fd7f76f10ee9e32c17170110bfaf5785f71eade Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 29 Nov 2016 16:17:13 -0500 Subject: [PATCH 10/20] Add a snippet documenting use of query parameters. --- docs/bigquery-usage.rst | 6 ++++++ docs/bigquery_snippets.py | 34 +++++++++++++++++++++++++++------- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/docs/bigquery-usage.rst b/docs/bigquery-usage.rst index 937fbf73f315..aaefcaaf0455 100644 --- a/docs/bigquery-usage.rst +++ b/docs/bigquery-usage.rst @@ -209,6 +209,12 @@ Run a query which can be expected to complete within bounded time: :start-after: [START client_run_sync_query] :end-before: [END client_run_sync_query] +Run a query using a named query parameter: + +.. literalinclude:: bigquery_snippets.py + :start-after: [START client_run_sync_query_w_param] + :end-before: [END client_run_sync_query_w_param] + If the rows returned by the query do not fit into the initial response, then we need to fetch the remaining rows via :meth:`~google.cloud.bigquery.query.QueryResults.fetch_data`: diff --git a/docs/bigquery_snippets.py b/docs/bigquery_snippets.py index 4231e64eae91..6f4816ab7497 100644 --- a/docs/bigquery_snippets.py +++ b/docs/bigquery_snippets.py @@ -461,13 +461,9 @@ def do_something_with(_): pass # [START client_list_jobs] - jobs, token = client.list_jobs() # API request - while True: - for job in jobs: - do_something_with(job) - if token is None: - break - jobs, token = client.list_jobs(page_token=token) # API request + job_iterator = client.list_jobs() # API request + for job in job_iterator: + do_something_with(job) # [END client_list_jobs] @@ -489,6 +485,30 @@ def client_run_sync_query(client, _): # [END client_run_sync_query] +@snippet +def client_run_sync_query_w_param(client, _): + """Run a synchronous query using a query parameter""" + QUERY_W_PARM = ( + 'SELECT name FROM `bigquery-public-data.usa_names.usa_1910_2013` ' + 'WHERE state = @state') + LIMIT = 100 + LIMITED = '%s LIMIT %d' % (QUERY_W_PARM, LIMIT) + TIMEOUT_MS = 1000 + + # [START client_run_sync_query_w_param] + from google.cloud.bigquery import ScalarQueryParameter + param = ScalarQueryParameter('state', 'STRING', 'TX') + query = client.run_sync_query(LIMITED, query_parameters=[param]) + query.use_legacy_sql = False + query.timeout_ms = TIMEOUT_MS + query.run() # API request + + assert query.complete + assert len(query.rows) == LIMIT + assert [field.name for field in query.schema] == ['name'] + # [END client_run_sync_query_w_param] + + @snippet def client_run_sync_query_paged(client, _): """Run a synchronous query with paged results.""" From 1df82a0ed1b3b3dcbe95b0d9ae46a54597b03c5c Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 29 Nov 2016 17:45:25 -0500 Subject: [PATCH 11/20] Address code review. --- bigquery/google/cloud/bigquery/__init__.py | 6 +++--- bigquery/google/cloud/bigquery/_helpers.py | 21 ++++++++++----------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/bigquery/google/cloud/bigquery/__init__.py b/bigquery/google/cloud/bigquery/__init__.py index 1fbebba5a089..9b5809af6cf7 100644 --- a/bigquery/google/cloud/bigquery/__init__.py +++ b/bigquery/google/cloud/bigquery/__init__.py @@ -23,11 +23,11 @@ """ +from google.cloud.bigquery._helpers import ArrayQueryParameter +from google.cloud.bigquery._helpers import ScalarQueryParameter +from google.cloud.bigquery._helpers import StructQueryParameter from google.cloud.bigquery.client import Client from google.cloud.bigquery.dataset import AccessGrant from google.cloud.bigquery.dataset import Dataset from google.cloud.bigquery.schema import SchemaField from google.cloud.bigquery.table import Table -from google.cloud.bigquery._helpers import ArrayQueryParameter -from google.cloud.bigquery._helpers import ScalarQueryParameter -from google.cloud.bigquery._helpers import StructQueryParameter diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index dc97a84652a9..f956458b530d 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -254,8 +254,8 @@ class ScalarQueryParameter(AbstractQueryParameter): """Named / positional query parameters for scalar values. :type name: str or None - :param name: Parameter name, used via `@foo` syntax. If None, the - paramter can only be addressed via position (`?`). + :param name: Parameter name, used via ``@foo`` syntax. If None, the + paramter can only be addressed via position (``?``). :type type_: str :param type_: name of parameter type. One of `'STRING'`, `'INT64'`, @@ -274,7 +274,6 @@ def __init__(self, name, type_, value): def positional(cls, type_, value): """Factory for positional paramters. - :type type_: str :param type_: name of paramter type. One of `'STRING'`, `'INT64'`, `'FLOAT64'`, `'BOOLEAN'`, `'TIMESTAMP'`, or `'DATE'`. @@ -284,7 +283,7 @@ def positional(cls, type_, value): :param value: the scalar parameter value. :rtype: :class:`ScalarQueryParameter` - :returns: instance w/o name + :returns: instance without name """ return cls(None, type_, value) @@ -327,8 +326,8 @@ class ArrayQueryParameter(AbstractQueryParameter): """Named / positional query parameters for array values. :type name: str or None - :param name: Parameter name, used via `@foo` syntax. If None, the - paramter can only be addressed via position (`?`). + :param name: Parameter name, used via ``@foo`` syntax. If None, the + paramter can only be addressed via position (``?``). :type array_type: str :param array_type: @@ -356,7 +355,7 @@ def positional(cls, array_type, values): :param values: the parameter array values. :rtype: :class:`ArrayQueryParameter` - :returns: instance w/o name + :returns: instance without name """ return cls(None, array_type, values) @@ -400,8 +399,8 @@ class StructQueryParameter(AbstractQueryParameter): """Named / positional query parameters for struct values. :type name: str or None - :param name: Parameter name, used via `@foo` syntax. If None, the - paramter can only be addressed via position (`?`). + :param name: Parameter name, used via ``@foo`` syntax. If None, the + paramter can only be addressed via position (``?``). :type sub_parms: tuple of :class:`ScalarQueryParameter` :param sub_parms: the sub-parameters for the struct @@ -417,10 +416,10 @@ def positional(cls, *sub_parms): """Factory for positional paramters. :type sub_parms: tuple of :class:`ScalarQueryParameter` - :param sub_parms:s the sub-parameters for the struct + :param sub_parms: the sub-parameters for the struct :rtype: :class:`StructQueryParameter` - :returns: instance w/o name + :returns: instance without name """ return cls(None, *sub_parms) From 6191ca69b13a2ed52a50f8254fa3e59720c5ed75 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 30 Nov 2016 10:58:04 -0500 Subject: [PATCH 12/20] Document expected types for QueryParametersProperty setter/getter. Addresses: https://github.com/GoogleCloudPlatform/google-cloud-python/pull/2776#discussion_r90132851 --- bigquery/google/cloud/bigquery/_helpers.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index f956458b530d..ebb2b0536a05 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -472,13 +472,24 @@ class QueryParametersProperty(object): """Custom property type, holding query parameter instances.""" def __get__(self, instance, owner): - """Descriptor protocol: accessor""" + """Descriptor protocol: accessor + + :rtype: list of instances of classes derived from + :class:`AbstractQueryParameter`. + :returns: the descriptor, if accessed via the class, or the instance's + query paramters. + """ if instance is None: return self return list(instance._query_parameters) def __set__(self, instance, value): - """Descriptor protocol: mutator""" + """Descriptor protocol: mutator + + :type value: list of instances of classes derived from + :class:`AbstractQueryParameter`. + :param value: new query parameters for the instance. + """ if not all(isinstance(u, AbstractQueryParameter) for u in value): raise ValueError( "query parameters must be derived from AbstractQueryParameter") From 89f5a5837012403812683c81389d7a8ab3107b20 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 30 Nov 2016 10:58:54 -0500 Subject: [PATCH 13/20] Use 'collections.OrderedDict' vs. hand-rolling order. Addresses: https://github.com/GoogleCloudPlatform/google-cloud-python/pull/2776#discussion_r90132319. --- bigquery/google/cloud/bigquery/_helpers.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index ebb2b0536a05..c83db152a0a1 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -14,6 +14,8 @@ """Shared helper functions for BigQuery API classes.""" +from collections import OrderedDict + from google.cloud._helpers import _datetime_from_microseconds from google.cloud._helpers import _date_from_iso8601_date @@ -407,8 +409,8 @@ class StructQueryParameter(AbstractQueryParameter): """ def __init__(self, name, *sub_parms): self.name = name - self._order = [sub.name for sub in sub_parms] - self.struct_types = {sub.name: sub.type_ for sub in sub_parms} + self.struct_types = OrderedDict( + (sub.name, sub.type_) for sub in sub_parms) self.struct_values = {sub.name: sub.value for sub in sub_parms} @classmethod @@ -452,8 +454,8 @@ def to_api_repr(self): :returns: JSON mapping """ types = [ - {'name': name, 'type': self.struct_types[name]} - for name in self._order + {'name': key, 'type': value} + for key, value in self.struct_types.items() ] resource = { 'parameterType': { @@ -474,6 +476,13 @@ class QueryParametersProperty(object): def __get__(self, instance, owner): """Descriptor protocol: accessor + :type instance: :class:`QueryParametersProperty` + :param instance: instance owning the property (None if accessed via + the class). + + :type owner: type + :param owner: the class owning the property. + :rtype: list of instances of classes derived from :class:`AbstractQueryParameter`. :returns: the descriptor, if accessed via the class, or the instance's @@ -486,6 +495,10 @@ def __get__(self, instance, owner): def __set__(self, instance, value): """Descriptor protocol: mutator + :type instance: :class:`QueryParametersProperty` + :param instance: instance owning the property (None if accessed via + the class). + :type value: list of instances of classes derived from :class:`AbstractQueryParameter`. :param value: new query parameters for the instance. From a4ff96493ed98eea15e7f4ea85c787c246575ec6 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 30 Nov 2016 11:06:02 -0500 Subject: [PATCH 14/20] PEP8. Addresses: https://github.com/GoogleCloudPlatform/google-cloud-python/pull/2776#discussion_r90133287. --- bigquery/unit_tests/test_job.py | 4 ++-- bigquery/unit_tests/test_query.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bigquery/unit_tests/test_job.py b/bigquery/unit_tests/test_job.py index b182e99bb5cc..6d6f1295d354 100644 --- a/bigquery/unit_tests/test_job.py +++ b/bigquery/unit_tests/test_job.py @@ -1287,7 +1287,7 @@ def _verifyIntegerResourceProperties(self, job, config): else: self.assertIsNone(job.maximum_bytes_billed) - def _verifyUDFResources(self, job, config): + def _verify_udf_resources(self, job, config): udf_resources = config.get('userDefinedFunctionResources', ()) self.assertEqual(len(job.udf_resources), len(udf_resources)) for found, expected in zip(job.udf_resources, udf_resources): @@ -1310,7 +1310,7 @@ def _verifyResourceProperties(self, job, resource): config = resource.get('configuration', {}).get('query') self._verifyBooleanResourceProperties(job, config) self._verifyIntegerResourceProperties(job, config) - self._verifyUDFResources(job, config) + self._verify_udf_resources(job, config) self._verifyQueryParameters(job, config) self.assertEqual(job.query, config['query']) diff --git a/bigquery/unit_tests/test_query.py b/bigquery/unit_tests/test_query.py index f12c33c9f36b..3dfc795a6072 100644 --- a/bigquery/unit_tests/test_query.py +++ b/bigquery/unit_tests/test_query.py @@ -100,7 +100,7 @@ def _verifyRows(self, query, resource): self.assertEqual(f_row, tuple([cell['v'] for cell in e_row['f']])) - def _verifyUDFResources(self, query, resource): + def _verify_udf_resources(self, query, resource): udf_resources = resource.get('userDefinedFunctionResources', ()) self.assertEqual(len(query.udf_resources), len(udf_resources)) for found, expected in zip(query.udf_resources, udf_resources): @@ -131,7 +131,7 @@ def _verifyResourceProperties(self, query, resource): else: self.assertIsNone(query.name) - self._verifyUDFResources(query, resource) + self._verify_udf_resources(query, resource) self._verifyQueryParameters(query, resource) self._verifySchema(query, resource) self._verifyRows(query, resource) From d4d451ea5541800e1f05eaaab0a33b02e2fcc720 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 30 Nov 2016 11:07:12 -0500 Subject: [PATCH 15/20] Correct notation of where API requests occur. Addresses: https://github.com/GoogleCloudPlatform/google-cloud-python/pull/2776#discussion_r90133576 --- docs/bigquery_snippets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/bigquery_snippets.py b/docs/bigquery_snippets.py index 6f4816ab7497..8f9abd584772 100644 --- a/docs/bigquery_snippets.py +++ b/docs/bigquery_snippets.py @@ -461,8 +461,8 @@ def do_something_with(_): pass # [START client_list_jobs] - job_iterator = client.list_jobs() # API request - for job in job_iterator: + job_iterator = client.list_jobs() + for job in job_iterator: # API request(s) do_something_with(job) # [END client_list_jobs] From 0f61970f8853ea052b25170e6aca1d9627ea9cc7 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 30 Nov 2016 11:13:05 -0500 Subject: [PATCH 16/20] s/parm/param/. Addresses: https://github.com/GoogleCloudPlatform/google-cloud-python/pull/2776#discussion_r90133740 --- bigquery/google/cloud/bigquery/_helpers.py | 18 +-- bigquery/unit_tests/test__helpers.py | 124 ++++++++++----------- docs/bigquery_snippets.py | 4 +- 3 files changed, 73 insertions(+), 73 deletions(-) diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index c83db152a0a1..769ab391f4d2 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -404,26 +404,26 @@ class StructQueryParameter(AbstractQueryParameter): :param name: Parameter name, used via ``@foo`` syntax. If None, the paramter can only be addressed via position (``?``). - :type sub_parms: tuple of :class:`ScalarQueryParameter` - :param sub_parms: the sub-parameters for the struct + :type sub_params: tuple of :class:`ScalarQueryParameter` + :param sub_params: the sub-parameters for the struct """ - def __init__(self, name, *sub_parms): + def __init__(self, name, *sub_params): self.name = name self.struct_types = OrderedDict( - (sub.name, sub.type_) for sub in sub_parms) - self.struct_values = {sub.name: sub.value for sub in sub_parms} + (sub.name, sub.type_) for sub in sub_params) + self.struct_values = {sub.name: sub.value for sub in sub_params} @classmethod - def positional(cls, *sub_parms): + def positional(cls, *sub_params): """Factory for positional paramters. - :type sub_parms: tuple of :class:`ScalarQueryParameter` - :param sub_parms: the sub-parameters for the struct + :type sub_params: tuple of :class:`ScalarQueryParameter` + :param sub_params: the sub-parameters for the struct :rtype: :class:`StructQueryParameter` :returns: instance without name """ - return cls(None, *sub_parms) + return cls(None, *sub_params) @classmethod def from_api_repr(cls, resource): diff --git a/bigquery/unit_tests/test__helpers.py b/bigquery/unit_tests/test__helpers.py index f00ec5bcfbb5..9281172e82ed 100644 --- a/bigquery/unit_tests/test__helpers.py +++ b/bigquery/unit_tests/test__helpers.py @@ -507,9 +507,9 @@ def test_from_api_virtual(self): klass.from_api_repr({}) def test_to_api_virtual(self): - parm = self._make_one() + param = self._make_one() with self.assertRaises(NotImplementedError): - parm.to_api_repr() + param.to_api_repr() class Test_ScalarQueryParameter(unittest.TestCase): @@ -523,17 +523,17 @@ def _make_one(self, *args, **kw): return self._get_target_class()(*args, **kw) def test_ctor(self): - parm = self._make_one(name='foo', type_='INT64', value=123) - self.assertEqual(parm.name, 'foo') - self.assertEqual(parm.type_, 'INT64') - self.assertEqual(parm.value, 123) + param = self._make_one(name='foo', type_='INT64', value=123) + self.assertEqual(param.name, 'foo') + self.assertEqual(param.type_, 'INT64') + self.assertEqual(param.value, 123) def test_positional(self): klass = self._get_target_class() - parm = klass.positional(type_='INT64', value=123) - self.assertEqual(parm.name, None) - self.assertEqual(parm.type_, 'INT64') - self.assertEqual(parm.value, 123) + param = klass.positional(type_='INT64', value=123) + self.assertEqual(param.name, None) + self.assertEqual(param.type_, 'INT64') + self.assertEqual(param.value, 123) def test_from_api_repr_w_name(self): RESOURCE = { @@ -546,10 +546,10 @@ def test_from_api_repr_w_name(self): }, } klass = self._get_target_class() - parm = klass.from_api_repr(RESOURCE) - self.assertEqual(parm.name, 'foo') - self.assertEqual(parm.type_, 'INT64') - self.assertEqual(parm.value, 123) + param = klass.from_api_repr(RESOURCE) + self.assertEqual(param.name, 'foo') + self.assertEqual(param.type_, 'INT64') + self.assertEqual(param.value, 123) def test_from_api_repr_wo_name(self): RESOURCE = { @@ -561,10 +561,10 @@ def test_from_api_repr_wo_name(self): }, } klass = self._get_target_class() - parm = klass.from_api_repr(RESOURCE) - self.assertEqual(parm.name, None) - self.assertEqual(parm.type_, 'INT64') - self.assertEqual(parm.value, 123) + param = klass.from_api_repr(RESOURCE) + self.assertEqual(param.name, None) + self.assertEqual(param.type_, 'INT64') + self.assertEqual(param.value, 123) def test_to_api_repr_w_name(self): EXPECTED = { @@ -576,8 +576,8 @@ def test_to_api_repr_w_name(self): 'value': 123, }, } - parm = self._make_one(name='foo', type_='INT64', value=123) - self.assertEqual(parm.to_api_repr(), EXPECTED) + param = self._make_one(name='foo', type_='INT64', value=123) + self.assertEqual(param.to_api_repr(), EXPECTED) def test_to_api_repr_wo_name(self): EXPECTED = { @@ -589,8 +589,8 @@ def test_to_api_repr_wo_name(self): }, } klass = self._get_target_class() - parm = klass.positional(type_='INT64', value=123) - self.assertEqual(parm.to_api_repr(), EXPECTED) + param = klass.positional(type_='INT64', value=123) + self.assertEqual(param.to_api_repr(), EXPECTED) class Test_ArrayQueryParameter(unittest.TestCase): @@ -604,17 +604,17 @@ def _make_one(self, *args, **kw): return self._get_target_class()(*args, **kw) def test_ctor(self): - parm = self._make_one(name='foo', array_type='INT64', values=[1, 2]) - self.assertEqual(parm.name, 'foo') - self.assertEqual(parm.array_type, 'INT64') - self.assertEqual(parm.values, [1, 2]) + param = self._make_one(name='foo', array_type='INT64', values=[1, 2]) + self.assertEqual(param.name, 'foo') + self.assertEqual(param.array_type, 'INT64') + self.assertEqual(param.values, [1, 2]) def test_positional(self): klass = self._get_target_class() - parm = klass.positional(array_type='INT64', values=[1, 2]) - self.assertEqual(parm.name, None) - self.assertEqual(parm.array_type, 'INT64') - self.assertEqual(parm.values, [1, 2]) + param = klass.positional(array_type='INT64', values=[1, 2]) + self.assertEqual(param.name, None) + self.assertEqual(param.array_type, 'INT64') + self.assertEqual(param.values, [1, 2]) def test_from_api_repr_w_name(self): RESOURCE = { @@ -627,10 +627,10 @@ def test_from_api_repr_w_name(self): }, } klass = self._get_target_class() - parm = klass.from_api_repr(RESOURCE) - self.assertEqual(parm.name, 'foo') - self.assertEqual(parm.array_type, 'INT64') - self.assertEqual(parm.values, [1, 2]) + param = klass.from_api_repr(RESOURCE) + self.assertEqual(param.name, 'foo') + self.assertEqual(param.array_type, 'INT64') + self.assertEqual(param.values, [1, 2]) def test_from_api_repr_wo_name(self): RESOURCE = { @@ -642,10 +642,10 @@ def test_from_api_repr_wo_name(self): }, } klass = self._get_target_class() - parm = klass.from_api_repr(RESOURCE) - self.assertEqual(parm.name, None) - self.assertEqual(parm.array_type, 'INT64') - self.assertEqual(parm.values, [1, 2]) + param = klass.from_api_repr(RESOURCE) + self.assertEqual(param.name, None) + self.assertEqual(param.array_type, 'INT64') + self.assertEqual(param.values, [1, 2]) def test_to_api_repr_w_name(self): EXPECTED = { @@ -657,8 +657,8 @@ def test_to_api_repr_w_name(self): 'arrayValues': [1, 2], }, } - parm = self._make_one(name='foo', array_type='INT64', values=[1, 2]) - self.assertEqual(parm.to_api_repr(), EXPECTED) + param = self._make_one(name='foo', array_type='INT64', values=[1, 2]) + self.assertEqual(param.to_api_repr(), EXPECTED) def test_to_api_repr_wo_name(self): EXPECTED = { @@ -670,8 +670,8 @@ def test_to_api_repr_wo_name(self): }, } klass = self._get_target_class() - parm = klass.positional(array_type='INT64', values=[1, 2]) - self.assertEqual(parm.to_api_repr(), EXPECTED) + param = klass.positional(array_type='INT64', values=[1, 2]) + self.assertEqual(param.to_api_repr(), EXPECTED) class Test_StructQueryParameter(unittest.TestCase): @@ -692,19 +692,19 @@ def _make_subparam(name, type_, value): def test_ctor(self): sub_1 = self._make_subparam('bar', 'INT64', 123) sub_2 = self._make_subparam('baz', 'STRING', 'abc') - parm = self._make_one('foo', sub_1, sub_2) - self.assertEqual(parm.name, 'foo') - self.assertEqual(parm.struct_types, {'bar': 'INT64', 'baz': 'STRING'}) - self.assertEqual(parm.struct_values, {'bar': 123, 'baz': 'abc'}) + param = self._make_one('foo', sub_1, sub_2) + self.assertEqual(param.name, 'foo') + self.assertEqual(param.struct_types, {'bar': 'INT64', 'baz': 'STRING'}) + self.assertEqual(param.struct_values, {'bar': 123, 'baz': 'abc'}) def test_positional(self): sub_1 = self._make_subparam('bar', 'INT64', 123) sub_2 = self._make_subparam('baz', 'STRING', 'abc') klass = self._get_target_class() - parm = klass.positional(sub_1, sub_2) - self.assertEqual(parm.name, None) - self.assertEqual(parm.struct_types, {'bar': 'INT64', 'baz': 'STRING'}) - self.assertEqual(parm.struct_values, {'bar': 123, 'baz': 'abc'}) + param = klass.positional(sub_1, sub_2) + self.assertEqual(param.name, None) + self.assertEqual(param.struct_types, {'bar': 'INT64', 'baz': 'STRING'}) + self.assertEqual(param.struct_values, {'bar': 123, 'baz': 'abc'}) def test_from_api_repr_w_name(self): RESOURCE = { @@ -720,10 +720,10 @@ def test_from_api_repr_w_name(self): }, } klass = self._get_target_class() - parm = klass.from_api_repr(RESOURCE) - self.assertEqual(parm.name, 'foo') - self.assertEqual(parm.struct_types, {'bar': 'INT64', 'baz': 'STRING'}) - self.assertEqual(parm.struct_values, {'bar': 123, 'baz': 'abc'}) + param = klass.from_api_repr(RESOURCE) + self.assertEqual(param.name, 'foo') + self.assertEqual(param.struct_types, {'bar': 'INT64', 'baz': 'STRING'}) + self.assertEqual(param.struct_values, {'bar': 123, 'baz': 'abc'}) def test_from_api_repr_wo_name(self): RESOURCE = { @@ -738,10 +738,10 @@ def test_from_api_repr_wo_name(self): }, } klass = self._get_target_class() - parm = klass.from_api_repr(RESOURCE) - self.assertEqual(parm.name, None) - self.assertEqual(parm.struct_types, {'bar': 'INT64', 'baz': 'STRING'}) - self.assertEqual(parm.struct_values, {'bar': 123, 'baz': 'abc'}) + param = klass.from_api_repr(RESOURCE) + self.assertEqual(param.name, None) + self.assertEqual(param.struct_types, {'bar': 'INT64', 'baz': 'STRING'}) + self.assertEqual(param.struct_values, {'bar': 123, 'baz': 'abc'}) def test_to_api_repr_w_name(self): EXPECTED = { @@ -758,8 +758,8 @@ def test_to_api_repr_w_name(self): } sub_1 = self._make_subparam('bar', 'INT64', 123) sub_2 = self._make_subparam('baz', 'STRING', 'abc') - parm = self._make_one('foo', sub_1, sub_2) - self.assertEqual(parm.to_api_repr(), EXPECTED) + param = self._make_one('foo', sub_1, sub_2) + self.assertEqual(param.to_api_repr(), EXPECTED) def test_to_api_repr_wo_name(self): EXPECTED = { @@ -776,8 +776,8 @@ def test_to_api_repr_wo_name(self): sub_1 = self._make_subparam('bar', 'INT64', 123) sub_2 = self._make_subparam('baz', 'STRING', 'abc') klass = self._get_target_class() - parm = klass.positional(sub_1, sub_2) - self.assertEqual(parm.to_api_repr(), EXPECTED) + param = klass.positional(sub_1, sub_2) + self.assertEqual(param.to_api_repr(), EXPECTED) class Test_QueryParametersProperty(unittest.TestCase): diff --git a/docs/bigquery_snippets.py b/docs/bigquery_snippets.py index 8f9abd584772..2bba9acece81 100644 --- a/docs/bigquery_snippets.py +++ b/docs/bigquery_snippets.py @@ -488,11 +488,11 @@ def client_run_sync_query(client, _): @snippet def client_run_sync_query_w_param(client, _): """Run a synchronous query using a query parameter""" - QUERY_W_PARM = ( + QUERY_W_PARAM = ( 'SELECT name FROM `bigquery-public-data.usa_names.usa_1910_2013` ' 'WHERE state = @state') LIMIT = 100 - LIMITED = '%s LIMIT %d' % (QUERY_W_PARM, LIMIT) + LIMITED = '%s LIMIT %d' % (QUERY_W_PARAM, LIMIT) TIMEOUT_MS = 1000 # [START client_run_sync_query_w_param] From 524baf1226292a6c96e17718257b2fd434ea9d1b Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Thu, 1 Dec 2016 12:02:50 -0500 Subject: [PATCH 17/20] Update docstring to show standard-SQL 'BOOL' type. The back-end raises an exception when passed the 'BOOLEAN' type to define a query parameter. Addresses: https://github.com/GoogleCloudPlatform/google-cloud-python/pull/2776#discussion_r90483766 --- bigquery/google/cloud/bigquery/_helpers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index 769ab391f4d2..3a71fa44c111 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -261,7 +261,7 @@ class ScalarQueryParameter(AbstractQueryParameter): :type type_: str :param type_: name of parameter type. One of `'STRING'`, `'INT64'`, - `'FLOAT64'`, `'BOOLEAN'`, `'TIMESTAMP'`, or `'DATE'`. + `'FLOAT64'`, `'BOOL'`, `'TIMESTAMP'`, or `'DATE'`. :type value: str, int, float, bool, :class:`datetime.datetime`, or :class:`datetime.date`. @@ -278,7 +278,7 @@ def positional(cls, type_, value): :type type_: str :param type_: name of paramter type. One of `'STRING'`, `'INT64'`, - `'FLOAT64'`, `'BOOLEAN'`, `'TIMESTAMP'`, or `'DATE'`. + `'FLOAT64'`, `'BOOL'`, `'TIMESTAMP'`, or `'DATE'`. :type value: str, int, float, bool, :class:`datetime.datetime`, or :class:`datetime.date`. @@ -334,7 +334,7 @@ class ArrayQueryParameter(AbstractQueryParameter): :type array_type: str :param array_type: name of type of array elements. One of `'STRING'`, `'INT64'`, - `'FLOAT64'`, `'BOOLEAN'`, `'TIMESTAMP'`, or `'DATE'`. + `'FLOAT64'`, `'BOOL'`, `'TIMESTAMP'`, or `'DATE'`. :type values: list of appropriate scalar type. :param values: the parameter array values. @@ -351,7 +351,7 @@ def positional(cls, array_type, values): :type array_type: str :param array_type: name of type of array elements. One of `'STRING'`, `'INT64'`, - `'FLOAT64'`, `'BOOLEAN'`, `'TIMESTAMP'`, or `'DATE'`. + `'FLOAT64'`, `'BOOL'`, `'TIMESTAMP'`, or `'DATE'`. :type values: list of appropriate scalar type :param values: the parameter array values. From 4b2cb13d640264ca2ec2572e39cb951ba684b565 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 2 Dec 2016 14:41:26 -0500 Subject: [PATCH 18/20] Indentation fix. --- bigquery/google/cloud/bigquery/_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index 3a71fa44c111..5812dc9f0121 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -500,7 +500,7 @@ def __set__(self, instance, value): the class). :type value: list of instances of classes derived from - :class:`AbstractQueryParameter`. + :class:`AbstractQueryParameter`. :param value: new query parameters for the instance. """ if not all(isinstance(u, AbstractQueryParameter) for u in value): From 99bdcd1bf34575074cd9faec7bf10d44d40cd46a Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 2 Dec 2016 15:51:19 -0500 Subject: [PATCH 19/20] Preserve OrderedDict for struct_types. Addresses: https://github.com/GoogleCloudPlatform/google-cloud-python/pull/2776#discussion_r90710926. --- bigquery/google/cloud/bigquery/_helpers.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index 5812dc9f0121..ab6e41272eef 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -437,14 +437,13 @@ def from_api_repr(cls, resource): """ name = resource.get('name') instance = cls(name) - types = instance.struct_types = { - item['name']: item['type'] - for item in resource['parameterType']['structTypes'] - } + types = instance.struct_types + for item in resource['parameterType']['structTypes']: + types[item['name']] = item['type'] struct_values = resource['parameterValue']['structValues'] - values = instance.struct_values = {} for key, value in struct_values.items(): - values[key] = _CELLDATA_FROM_JSON[types[key]](value, None) + converted = _CELLDATA_FROM_JSON[types[key]](value, None) + instance.struct_values[key] = converted return instance def to_api_repr(self): From 132741ab5f730f7a9352a48240713d4b0b87ff73 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 2 Dec 2016 15:51:56 -0500 Subject: [PATCH 20/20] Normalize '%' formatting. Addresses: https://github.com/GoogleCloudPlatform/google-cloud-python/pull/2776#discussion_r90133344. --- bigquery/unit_tests/test_job.py | 116 ++++++++++++++++---------------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/bigquery/unit_tests/test_job.py b/bigquery/unit_tests/test_job.py index 6d6f1295d354..84ee25418491 100644 --- a/bigquery/unit_tests/test_job.py +++ b/bigquery/unit_tests/test_job.py @@ -426,7 +426,7 @@ def test_begin_w_already_running(self): job.begin() def test_begin_w_bound_client(self): - PATH = 'projects/%s/jobs' % self.PROJECT + PATH = '/projects/%s/jobs' % (self.PROJECT,) RESOURCE = self._makeResource() # Ensure None for missing server-set props del RESOURCE['statistics']['creationTime'] @@ -443,7 +443,7 @@ def test_begin_w_bound_client(self): self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'POST') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) SENT = { 'jobReference': { 'projectId': self.PROJECT, @@ -465,7 +465,7 @@ def test_begin_w_bound_client(self): def test_begin_w_alternate_client(self): from google.cloud.bigquery.schema import SchemaField - PATH = 'projects/%s/jobs' % self.PROJECT + PATH = '/projects/%s/jobs' % (self.PROJECT,) RESOURCE = self._makeResource(ended=True) LOAD_CONFIGURATION = { 'sourceUris': [self.SOURCE1], @@ -519,7 +519,7 @@ def test_begin_w_alternate_client(self): self.assertEqual(len(conn2._requested), 1) req = conn2._requested[0] self.assertEqual(req['method'], 'POST') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) SENT = { 'jobReference': { 'projectId': self.PROJECT, @@ -533,7 +533,7 @@ def test_begin_w_alternate_client(self): self._verifyResourceProperties(job, RESOURCE) def test_exists_miss_w_bound_client(self): - PATH = 'projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) + PATH = '/projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) conn = _Connection() client = _Client(project=self.PROJECT, connection=conn) table = _Table() @@ -544,11 +544,11 @@ def test_exists_miss_w_bound_client(self): self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'GET') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) self.assertEqual(req['query_params'], {'fields': 'id'}) def test_exists_hit_w_alternate_client(self): - PATH = 'projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) + PATH = '/projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) conn1 = _Connection() client1 = _Client(project=self.PROJECT, connection=conn1) conn2 = _Connection({}) @@ -562,11 +562,11 @@ def test_exists_hit_w_alternate_client(self): self.assertEqual(len(conn2._requested), 1) req = conn2._requested[0] self.assertEqual(req['method'], 'GET') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) self.assertEqual(req['query_params'], {'fields': 'id'}) def test_reload_w_bound_client(self): - PATH = 'projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) + PATH = '/projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) RESOURCE = self._makeResource() conn = _Connection(RESOURCE) client = _Client(project=self.PROJECT, connection=conn) @@ -578,11 +578,11 @@ def test_reload_w_bound_client(self): self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'GET') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) self._verifyResourceProperties(job, RESOURCE) def test_reload_w_alternate_client(self): - PATH = 'projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) + PATH = '/projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) RESOURCE = self._makeResource() conn1 = _Connection() client1 = _Client(project=self.PROJECT, connection=conn1) @@ -597,11 +597,11 @@ def test_reload_w_alternate_client(self): self.assertEqual(len(conn2._requested), 1) req = conn2._requested[0] self.assertEqual(req['method'], 'GET') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) self._verifyResourceProperties(job, RESOURCE) def test_cancel_w_bound_client(self): - PATH = 'projects/%s/jobs/%s/cancel' % (self.PROJECT, self.JOB_NAME) + PATH = '/projects/%s/jobs/%s/cancel' % (self.PROJECT, self.JOB_NAME) RESOURCE = self._makeResource(ended=True) RESPONSE = {'job': RESOURCE} conn = _Connection(RESPONSE) @@ -614,11 +614,11 @@ def test_cancel_w_bound_client(self): self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'POST') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) self._verifyResourceProperties(job, RESOURCE) def test_cancel_w_alternate_client(self): - PATH = 'projects/%s/jobs/%s/cancel' % (self.PROJECT, self.JOB_NAME) + PATH = '/projects/%s/jobs/%s/cancel' % (self.PROJECT, self.JOB_NAME) RESOURCE = self._makeResource(ended=True) RESPONSE = {'job': RESOURCE} conn1 = _Connection() @@ -634,7 +634,7 @@ def test_cancel_w_alternate_client(self): self.assertEqual(len(conn2._requested), 1) req = conn2._requested[0] self.assertEqual(req['method'], 'POST') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) self._verifyResourceProperties(job, RESOURCE) @@ -773,7 +773,7 @@ def test_from_api_repr_w_properties(self): self._verifyResourceProperties(dataset, RESOURCE) def test_begin_w_bound_client(self): - PATH = 'projects/%s/jobs' % self.PROJECT + PATH = '/projects/%s/jobs' % (self.PROJECT,) RESOURCE = self._makeResource() # Ensure None for missing server-set props del RESOURCE['statistics']['creationTime'] @@ -791,7 +791,7 @@ def test_begin_w_bound_client(self): self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'POST') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) SENT = { 'jobReference': { 'projectId': self.PROJECT, @@ -816,7 +816,7 @@ def test_begin_w_bound_client(self): self._verifyResourceProperties(job, RESOURCE) def test_begin_w_alternate_client(self): - PATH = 'projects/%s/jobs' % self.PROJECT + PATH = '/projects/%s/jobs' % (self.PROJECT,) RESOURCE = self._makeResource(ended=True) COPY_CONFIGURATION = { 'sourceTables': [{ @@ -850,7 +850,7 @@ def test_begin_w_alternate_client(self): self.assertEqual(len(conn2._requested), 1) req = conn2._requested[0] self.assertEqual(req['method'], 'POST') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) SENT = { 'jobReference': { 'projectId': self.PROJECT, @@ -864,7 +864,7 @@ def test_begin_w_alternate_client(self): self._verifyResourceProperties(job, RESOURCE) def test_exists_miss_w_bound_client(self): - PATH = 'projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) + PATH = '/projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) conn = _Connection() client = _Client(project=self.PROJECT, connection=conn) source = _Table(self.SOURCE_TABLE) @@ -876,11 +876,11 @@ def test_exists_miss_w_bound_client(self): self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'GET') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) self.assertEqual(req['query_params'], {'fields': 'id'}) def test_exists_hit_w_alternate_client(self): - PATH = 'projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) + PATH = '/projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) conn1 = _Connection() client1 = _Client(project=self.PROJECT, connection=conn1) conn2 = _Connection({}) @@ -895,11 +895,11 @@ def test_exists_hit_w_alternate_client(self): self.assertEqual(len(conn2._requested), 1) req = conn2._requested[0] self.assertEqual(req['method'], 'GET') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) self.assertEqual(req['query_params'], {'fields': 'id'}) def test_reload_w_bound_client(self): - PATH = 'projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) + PATH = '/projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) RESOURCE = self._makeResource() conn = _Connection(RESOURCE) client = _Client(project=self.PROJECT, connection=conn) @@ -912,11 +912,11 @@ def test_reload_w_bound_client(self): self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'GET') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) self._verifyResourceProperties(job, RESOURCE) def test_reload_w_alternate_client(self): - PATH = 'projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) + PATH = '/projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) RESOURCE = self._makeResource() conn1 = _Connection() client1 = _Client(project=self.PROJECT, connection=conn1) @@ -932,7 +932,7 @@ def test_reload_w_alternate_client(self): self.assertEqual(len(conn2._requested), 1) req = conn2._requested[0] self.assertEqual(req['method'], 'GET') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) self._verifyResourceProperties(job, RESOURCE) @@ -1071,7 +1071,7 @@ def test_from_api_repr_w_properties(self): self._verifyResourceProperties(dataset, RESOURCE) def test_begin_w_bound_client(self): - PATH = 'projects/%s/jobs' % self.PROJECT + PATH = '/projects/%s/jobs' % (self.PROJECT,) RESOURCE = self._makeResource() # Ensure None for missing server-set props del RESOURCE['statistics']['creationTime'] @@ -1089,7 +1089,7 @@ def test_begin_w_bound_client(self): self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'POST') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) SENT = { 'jobReference': { 'projectId': self.PROJECT, @@ -1110,7 +1110,7 @@ def test_begin_w_bound_client(self): self._verifyResourceProperties(job, RESOURCE) def test_begin_w_alternate_client(self): - PATH = 'projects/%s/jobs' % self.PROJECT + PATH = '/projects/%s/jobs' % (self.PROJECT,) RESOURCE = self._makeResource(ended=True) EXTRACT_CONFIGURATION = { 'sourceTable': { @@ -1144,7 +1144,7 @@ def test_begin_w_alternate_client(self): self.assertEqual(len(conn2._requested), 1) req = conn2._requested[0] self.assertEqual(req['method'], 'POST') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) SENT = { 'jobReference': { 'projectId': self.PROJECT, @@ -1158,7 +1158,7 @@ def test_begin_w_alternate_client(self): self._verifyResourceProperties(job, RESOURCE) def test_exists_miss_w_bound_client(self): - PATH = 'projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) + PATH = '/projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) conn = _Connection() client = _Client(project=self.PROJECT, connection=conn) source = _Table(self.SOURCE_TABLE) @@ -1170,11 +1170,11 @@ def test_exists_miss_w_bound_client(self): self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'GET') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) self.assertEqual(req['query_params'], {'fields': 'id'}) def test_exists_hit_w_alternate_client(self): - PATH = 'projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) + PATH = '/projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) conn1 = _Connection() client1 = _Client(project=self.PROJECT, connection=conn1) conn2 = _Connection({}) @@ -1189,11 +1189,11 @@ def test_exists_hit_w_alternate_client(self): self.assertEqual(len(conn2._requested), 1) req = conn2._requested[0] self.assertEqual(req['method'], 'GET') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) self.assertEqual(req['query_params'], {'fields': 'id'}) def test_reload_w_bound_client(self): - PATH = 'projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) + PATH = '/projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) RESOURCE = self._makeResource() conn = _Connection(RESOURCE) client = _Client(project=self.PROJECT, connection=conn) @@ -1206,11 +1206,11 @@ def test_reload_w_bound_client(self): self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'GET') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) self._verifyResourceProperties(job, RESOURCE) def test_reload_w_alternate_client(self): - PATH = 'projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) + PATH = '/projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) RESOURCE = self._makeResource() conn1 = _Connection() client1 = _Client(project=self.PROJECT, connection=conn1) @@ -1226,7 +1226,7 @@ def test_reload_w_alternate_client(self): self.assertEqual(len(conn2._requested), 1) req = conn2._requested[0] self.assertEqual(req['method'], 'GET') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) self._verifyResourceProperties(job, RESOURCE) @@ -1454,7 +1454,7 @@ def test_results(self): self.assertIs(results._job, job) def test_begin_w_bound_client(self): - PATH = 'projects/%s/jobs' % self.PROJECT + PATH = '/projects/%s/jobs' % (self.PROJECT,) RESOURCE = self._makeResource() # Ensure None for missing server-set props del RESOURCE['statistics']['creationTime'] @@ -1470,7 +1470,7 @@ def test_begin_w_bound_client(self): self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'POST') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) SENT = { 'jobReference': { 'projectId': self.PROJECT, @@ -1488,7 +1488,7 @@ def test_begin_w_bound_client(self): def test_begin_w_alternate_client(self): from google.cloud.bigquery.dataset import Dataset from google.cloud.bigquery.dataset import Table - PATH = 'projects/%s/jobs' % self.PROJECT + PATH = '/projects/%s/jobs' % (self.PROJECT,) TABLE = 'TABLE' DS_NAME = 'DATASET' RESOURCE = self._makeResource(ended=True) @@ -1543,7 +1543,7 @@ def test_begin_w_alternate_client(self): self.assertEqual(len(conn2._requested), 1) req = conn2._requested[0] self.assertEqual(req['method'], 'POST') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) SENT = { 'jobReference': { 'projectId': self.PROJECT, @@ -1560,7 +1560,7 @@ def test_begin_w_udf(self): from google.cloud.bigquery._helpers import UDFResource RESOURCE_URI = 'gs://some-bucket/js/lib.js' INLINE_UDF_CODE = 'var someCode = "here";' - PATH = 'projects/%s/jobs' % self.PROJECT + PATH = '/projects/%s/jobs' % (self.PROJECT,) RESOURCE = self._makeResource() # Ensure None for missing server-set props del RESOURCE['statistics']['creationTime'] @@ -1585,7 +1585,7 @@ def test_begin_w_udf(self): self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'POST') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) self.assertEqual(job.udf_resources, udf_resources) SENT = { 'jobReference': { @@ -1608,7 +1608,7 @@ def test_begin_w_udf(self): def test_begin_w_named_query_parameter(self): from google.cloud.bigquery._helpers import ScalarQueryParameter query_parameters = [ScalarQueryParameter('foo', 'INT64', 123)] - PATH = 'projects/%s/jobs' % self.PROJECT + PATH = '/projects/%s/jobs' % (self.PROJECT,) RESOURCE = self._makeResource() # Ensure None for missing server-set props del RESOURCE['statistics']['creationTime'] @@ -1638,7 +1638,7 @@ def test_begin_w_named_query_parameter(self): self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'POST') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) self.assertEqual(job.query_parameters, query_parameters) SENT = { 'jobReference': { @@ -1659,7 +1659,7 @@ def test_begin_w_named_query_parameter(self): def test_begin_w_positional_query_parameter(self): from google.cloud.bigquery._helpers import ScalarQueryParameter query_parameters = [ScalarQueryParameter.positional('INT64', 123)] - PATH = 'projects/%s/jobs' % self.PROJECT + PATH = '/projects/%s/jobs' % (self.PROJECT,) RESOURCE = self._makeResource() # Ensure None for missing server-set props del RESOURCE['statistics']['creationTime'] @@ -1688,7 +1688,7 @@ def test_begin_w_positional_query_parameter(self): self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'POST') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) self.assertEqual(job.query_parameters, query_parameters) SENT = { 'jobReference': { @@ -1707,7 +1707,7 @@ def test_begin_w_positional_query_parameter(self): self._verifyResourceProperties(job, RESOURCE) def test_exists_miss_w_bound_client(self): - PATH = 'projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) + PATH = '/projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) conn = _Connection() client = _Client(project=self.PROJECT, connection=conn) job = self._make_one(self.JOB_NAME, self.QUERY, client) @@ -1717,11 +1717,11 @@ def test_exists_miss_w_bound_client(self): self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'GET') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) self.assertEqual(req['query_params'], {'fields': 'id'}) def test_exists_hit_w_alternate_client(self): - PATH = 'projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) + PATH = '/projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) conn1 = _Connection() client1 = _Client(project=self.PROJECT, connection=conn1) conn2 = _Connection({}) @@ -1734,13 +1734,13 @@ def test_exists_hit_w_alternate_client(self): self.assertEqual(len(conn2._requested), 1) req = conn2._requested[0] self.assertEqual(req['method'], 'GET') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) self.assertEqual(req['query_params'], {'fields': 'id'}) def test_reload_w_bound_client(self): from google.cloud.bigquery.dataset import Dataset from google.cloud.bigquery.dataset import Table - PATH = 'projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) + PATH = '/projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) DS_NAME = 'DATASET' DEST_TABLE = 'dest_table' RESOURCE = self._makeResource() @@ -1759,11 +1759,11 @@ def test_reload_w_bound_client(self): self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'GET') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) self._verifyResourceProperties(job, RESOURCE) def test_reload_w_alternate_client(self): - PATH = 'projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) + PATH = '/projects/%s/jobs/%s' % (self.PROJECT, self.JOB_NAME) DS_NAME = 'DATASET' DEST_TABLE = 'dest_table' RESOURCE = self._makeResource() @@ -1785,7 +1785,7 @@ def test_reload_w_alternate_client(self): self.assertEqual(len(conn2._requested), 1) req = conn2._requested[0] self.assertEqual(req['method'], 'GET') - self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['path'], PATH) self._verifyResourceProperties(job, RESOURCE)