From ab0bf9c282004e938e63052c2ad73eed2df495c6 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 5 Dec 2016 13:46:53 -0500 Subject: [PATCH 1/3] Allow overriding dataset's project during construction. Closes #2118. --- bigquery/google/cloud/bigquery/client.py | 8 +++-- bigquery/google/cloud/bigquery/dataset.py | 9 ++++-- bigquery/unit_tests/test_client.py | 17 ++++++++++- bigquery/unit_tests/test_dataset.py | 37 ++++++++++++++++++++--- 4 files changed, 62 insertions(+), 9 deletions(-) diff --git a/bigquery/google/cloud/bigquery/client.py b/bigquery/google/cloud/bigquery/client.py index 4af5b8fc4910..0c01578f5ffe 100644 --- a/bigquery/google/cloud/bigquery/client.py +++ b/bigquery/google/cloud/bigquery/client.py @@ -129,16 +129,20 @@ def list_datasets(self, include_all=False, max_results=None, items_key='datasets', page_token=page_token, max_results=max_results, extra_params=extra_params) - def dataset(self, dataset_name): + def dataset(self, dataset_name, project=None): """Construct a dataset bound to this client. :type dataset_name: str :param dataset_name: Name of the dataset. + :type project: str + :param project: (Optional) project ID for the dataset (defaults to + the project of the client). + :rtype: :class:`google.cloud.bigquery.dataset.Dataset` :returns: a new ``Dataset`` instance """ - return Dataset(dataset_name, client=self) + return Dataset(dataset_name, client=self, project=project) def job_from_resource(self, resource): """Detect correct job type from resource and instantiate. diff --git a/bigquery/google/cloud/bigquery/dataset.py b/bigquery/google/cloud/bigquery/dataset.py index e209433b5e10..6f28bd791b05 100644 --- a/bigquery/google/cloud/bigquery/dataset.py +++ b/bigquery/google/cloud/bigquery/dataset.py @@ -101,16 +101,21 @@ class Dataset(object): :type access_grants: list of :class:`AccessGrant` :param access_grants: roles granted to entities for this dataset + + :type project: str + :param project: (Optional) project ID for the dataset (defaults to + the project of the client). """ _access_grants = None - def __init__(self, name, client, access_grants=()): + def __init__(self, name, client, access_grants=(), project=None): self.name = name self._client = client self._properties = {} # Let the @property do validation. self.access_grants = access_grants + self._project = project @property def project(self): @@ -119,7 +124,7 @@ def project(self): :rtype: str :returns: the project (derived from the client). """ - return self._client.project + return self._project or self._client.project @property def path(self): diff --git a/bigquery/unit_tests/test_client.py b/bigquery/unit_tests/test_client.py index 8c76897b3f6d..152714839ebd 100644 --- a/bigquery/unit_tests/test_client.py +++ b/bigquery/unit_tests/test_client.py @@ -176,7 +176,7 @@ def test_list_datasets_explicit_response_missing_datasets_key(self): self.assertEqual(req['query_params'], {'all': True, 'maxResults': 3, 'pageToken': TOKEN}) - def test_dataset(self): + def test_dataset_defaults(self): from google.cloud.bigquery.dataset import Dataset PROJECT = 'PROJECT' DATASET = 'dataset_name' @@ -187,6 +187,21 @@ def test_dataset(self): self.assertIsInstance(dataset, Dataset) self.assertEqual(dataset.name, DATASET) self.assertIs(dataset._client, client) + self.assertEqual(dataset.project, PROJECT) + + def test_dataset_explicit(self): + from google.cloud.bigquery.dataset import Dataset + PROJECT = 'my-project-123' + OTHER_PROJECT = 'other-project-456' + DATASET = 'dataset_name' + creds = object() + http = object() + client = self._make_one(project=PROJECT, credentials=creds, http=http) + dataset = client.dataset(DATASET, project=OTHER_PROJECT) + self.assertIsInstance(dataset, Dataset) + self.assertEqual(dataset.name, DATASET) + self.assertIs(dataset._client, client) + self.assertEqual(dataset.project, OTHER_PROJECT) def test_job_from_resource_unknown_type(self): PROJECT = 'PROJECT' diff --git a/bigquery/unit_tests/test_dataset.py b/bigquery/unit_tests/test_dataset.py index ec7c56722368..015b160687d9 100644 --- a/bigquery/unit_tests/test_dataset.py +++ b/bigquery/unit_tests/test_dataset.py @@ -174,7 +174,7 @@ def _verifyResourceProperties(self, dataset, resource): else: self.assertEqual(dataset.access_grants, []) - def test_ctor(self): + def test_ctor_defaults(self): client = _Client(self.PROJECT) dataset = self._make_one(self.DS_NAME, client) self.assertEqual(dataset.name, self.DS_NAME) @@ -196,13 +196,42 @@ def test_ctor(self): self.assertIsNone(dataset.friendly_name) self.assertIsNone(dataset.location) - def test_access_roles_setter_non_list(self): + def test_ctor_explicit(self): + from google.cloud.bigquery.dataset import AccessGrant + phred = AccessGrant('OWNER', 'userByEmail', 'phred@example.com') + bharney = AccessGrant('OWNER', 'userByEmail', 'bharney@example.com') + grants = [phred, bharney] + OTHER_PROJECT = 'foo-bar-123' + client = _Client(self.PROJECT) + dataset = self._make_one(self.DS_NAME, client, + access_grants=grants, + project=OTHER_PROJECT) + self.assertEqual(dataset.name, self.DS_NAME) + self.assertIs(dataset._client, client) + self.assertEqual(dataset.project, OTHER_PROJECT) + self.assertEqual( + dataset.path, + '/projects/%s/datasets/%s' % (OTHER_PROJECT, self.DS_NAME)) + self.assertEqual(dataset.access_grants, grants) + + self.assertIsNone(dataset.created) + self.assertIsNone(dataset.dataset_id) + self.assertIsNone(dataset.etag) + self.assertIsNone(dataset.modified) + self.assertIsNone(dataset.self_link) + + self.assertIsNone(dataset.default_table_expiration_ms) + self.assertIsNone(dataset.description) + self.assertIsNone(dataset.friendly_name) + self.assertIsNone(dataset.location) + + def test_access_grants_setter_non_list(self): client = _Client(self.PROJECT) dataset = self._make_one(self.DS_NAME, client) with self.assertRaises(TypeError): dataset.access_grants = object() - def test_access_roles_setter_invalid_field(self): + def test_access_grants_setter_invalid_field(self): from google.cloud.bigquery.dataset import AccessGrant client = _Client(self.PROJECT) dataset = self._make_one(self.DS_NAME, client) @@ -210,7 +239,7 @@ def test_access_roles_setter_invalid_field(self): with self.assertRaises(ValueError): dataset.access_grants = [phred, object()] - def test_access_roles_setter(self): + def test_access_grants_setter(self): from google.cloud.bigquery.dataset import AccessGrant client = _Client(self.PROJECT) dataset = self._make_one(self.DS_NAME, client) From d5091bc74ffd7fd3233290e6fef808374d7ed011 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 5 Dec 2016 14:53:47 -0500 Subject: [PATCH 2/3] Add a system test exercising non-default project for dataset. --- system_tests/bigquery.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/system_tests/bigquery.py b/system_tests/bigquery.py index bf913d499afb..d9ac3857bbf6 100644 --- a/system_tests/bigquery.py +++ b/system_tests/bigquery.py @@ -524,3 +524,12 @@ def test_sync_query_w_nested_arrays_and_structs(self): self.assertEqual(len(query.rows), 1) self.assertEqual(len(query.rows[0]), 1) self.assertEqual(query.rows[0][0], example['expected']) + + def test_dump_table_w_public_data(self): + PUBLIC = 'bigquery-public-data' + DATASET_NAME = 'samples' + TABLE_NAME = 'natality' + + dataset = Config.CLIENT.dataset(DATASET_NAME, project=PUBLIC) + table = dataset.table(TABLE_NAME) + self._fetch_single_page(table) From e86617735f968584fb231108013227be2d98a083 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 5 Dec 2016 14:55:45 -0500 Subject: [PATCH 3/3] Freeze project selection in dataset ctor. Addresses: https://travis-ci.org/GoogleCloudPlatform/google-cloud-python/builds/181434795#L1784-L1785 --- bigquery/google/cloud/bigquery/dataset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigquery/google/cloud/bigquery/dataset.py b/bigquery/google/cloud/bigquery/dataset.py index 6f28bd791b05..109f259ae100 100644 --- a/bigquery/google/cloud/bigquery/dataset.py +++ b/bigquery/google/cloud/bigquery/dataset.py @@ -115,7 +115,7 @@ def __init__(self, name, client, access_grants=(), project=None): self._properties = {} # Let the @property do validation. self.access_grants = access_grants - self._project = project + self._project = project or client.project @property def project(self): @@ -124,7 +124,7 @@ def project(self): :rtype: str :returns: the project (derived from the client). """ - return self._project or self._client.project + return self._project @property def path(self):