From a6aa2238fa2ec5ccf81f75f54735dfaebfd1db68 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Tue, 15 Sep 2015 13:57:42 -0700 Subject: [PATCH] Reorganizing storage samples. --- README.md | 10 ----- bigquery/README.md | 8 ---- storage/api/__init__.py | 0 storage/{ => api}/compose_objects.py | 0 storage/{ => api}/list_objects.py | 0 .../{tests => api}/test_compose_objects.py | 2 +- storage/{tests => api}/test_list_objects.py | 2 +- storage/storage_transfer/test_aws_request.py | 39 ---------------- .../storage_transfer/test_create_client.py | 44 ------------------- .../storage_transfer/test_nearline_request.py | 39 ---------------- .../storage_transfer/test_transfer_check.py | 31 ------------- .../README.md | 0 storage/transfer_service/__init__.py | 0 .../aws_request.py | 13 ++++-- .../create_client.py | 0 .../nearline_request.py | 13 ++++-- .../transfer_check.py | 21 +++++---- 17 files changed, 33 insertions(+), 189 deletions(-) create mode 100644 storage/api/__init__.py rename storage/{ => api}/compose_objects.py (100%) rename storage/{ => api}/list_objects.py (100%) rename storage/{tests => api}/test_compose_objects.py (95%) rename storage/{tests => api}/test_list_objects.py (95%) delete mode 100644 storage/storage_transfer/test_aws_request.py delete mode 100644 storage/storage_transfer/test_create_client.py delete mode 100644 storage/storage_transfer/test_nearline_request.py delete mode 100644 storage/storage_transfer/test_transfer_check.py rename storage/{storage_transfer => transfer_service}/README.md (100%) create mode 100644 storage/transfer_service/__init__.py rename storage/{storage_transfer => transfer_service}/aws_request.py (84%) rename storage/{storage_transfer => transfer_service}/create_client.py (100%) rename storage/{storage_transfer => transfer_service}/nearline_request.py (83%) rename storage/{storage_transfer => transfer_service}/transfer_check.py (66%) diff --git a/README.md b/README.md index 2dbaaead8b24..36ab4283270d 100644 --- a/README.md +++ b/README.md @@ -8,16 +8,6 @@ This repository holds the samples used in the python documentation on [cloud.goo For more detailed introduction to a product, check the README in the corresponding folder. -## Running the samples - -Most samples must be run as modules instead of directly, for example: - -``` -$ python -m bigquery.samples.async_query [your-project-id] [your-query] -``` - -Refer to the README in the corresponding folder for any special instructions. - ## Contributing changes * See [CONTRIBUTING.md](CONTRIBUTING.md) diff --git a/bigquery/README.md b/bigquery/README.md index 0385d7cbf63e..75f7b9a6897f 100644 --- a/bigquery/README.md +++ b/bigquery/README.md @@ -2,14 +2,6 @@ This section contains samples for [Google BigQuery](https://cloud.google.com/bigquery). -## Running the samples - -These samples must be run as modules, for example: - -``` -$ python -m bigquery.samples.async_query [your-project-id] [your-query] -``` - ## Other Samples * [Using BigQuery from Google App Engine](../appengine/bigquery). diff --git a/storage/api/__init__.py b/storage/api/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/storage/compose_objects.py b/storage/api/compose_objects.py similarity index 100% rename from storage/compose_objects.py rename to storage/api/compose_objects.py diff --git a/storage/list_objects.py b/storage/api/list_objects.py similarity index 100% rename from storage/list_objects.py rename to storage/api/list_objects.py diff --git a/storage/tests/test_compose_objects.py b/storage/api/test_compose_objects.py similarity index 95% rename from storage/tests/test_compose_objects.py rename to storage/api/test_compose_objects.py index d9ca356904c1..a44b2fc2b64d 100644 --- a/storage/tests/test_compose_objects.py +++ b/storage/api/test_compose_objects.py @@ -13,7 +13,7 @@ # import os -from storage.compose_objects import main +from .compose_objects import main from tests import CloudBaseTest diff --git a/storage/tests/test_list_objects.py b/storage/api/test_list_objects.py similarity index 95% rename from storage/tests/test_list_objects.py rename to storage/api/test_list_objects.py index 32f7d09424a5..83842e5b171d 100644 --- a/storage/tests/test_list_objects.py +++ b/storage/api/test_list_objects.py @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from storage.list_objects import main +from .list_objects import main from tests import CloudBaseTest diff --git a/storage/storage_transfer/test_aws_request.py b/storage/storage_transfer/test_aws_request.py deleted file mode 100644 index f37a7c512ab6..000000000000 --- a/storage/storage_transfer/test_aws_request.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import unittest - -from aws_request import main - -from mock import Mock -from mock import patch - - -class AwsRequestTestCase(unittest.TestCase): - """A test case for creating a TransferJob from AWS S3.""" - - def setUp(self): - patcher1 = patch( - 'storage.storage_transfer.aws_request.create_client') - self.mock_create_client = patcher1.start() - self.addCleanup(patcher1.stop) - self.mock_client = Mock(spec=['transferJobs']) - self.mock_create_client.create_transfer_client.return_value = \ - self.mock_client - - def test_create_aws_request(self): - execute = self.mock_client.transferJobs.return_value.create.return_value \ - .execute - execute.return_value = "" - main() - execute.assert_called_with() diff --git a/storage/storage_transfer/test_create_client.py b/storage/storage_transfer/test_create_client.py deleted file mode 100644 index c74cde2e16eb..000000000000 --- a/storage/storage_transfer/test_create_client.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import unittest - -import create_client - -from mock import Mock -from mock import patch -from oauth2client.client import GoogleCredentials - - -class CheckCreateClientTestCase(unittest.TestCase): - """A test case for client creation.""" - - def setUp(self): - patcher1 = patch( - 'storage.storage_transfer.create_client.GoogleCredentials') - patcher2 = patch( - 'storage.storage_transfer.create_client.discovery.build') - self.mock_google_credentials = patcher1.start() - self.mock_discovery = patcher2.start() - self.addCleanup(patcher1.stop) - self.addCleanup(patcher2.stop) - - self.mock_credentials = Mock(spec=GoogleCredentials) - self.mock_google_credentials.get_application_default.return_value = \ - self.mock_credentials - - def test_create_client(self): - create_client.create_transfer_client() - self.mock_discovery.assert_called_with( - 'storagetransfer', 'v1', - credentials=self.mock_credentials) diff --git a/storage/storage_transfer/test_nearline_request.py b/storage/storage_transfer/test_nearline_request.py deleted file mode 100644 index f70b45e5d5da..000000000000 --- a/storage/storage_transfer/test_nearline_request.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import unittest - -from mock import Mock -from mock import patch - -from nearline_request import main - - -class NearlineRequestTestCase(unittest.TestCase): - """A test case for creating a TransferJob to Nearline for old files.""" - - def setUp(self): - patcher1 = patch( - 'storage.storage_transfer.nearline_request.create_client') - self.mock_create_client = patcher1.start() - self.addCleanup(patcher1.stop) - self.mock_client = Mock(spec=['transferJobs']) - self.mock_create_client.create_transfer_client.return_value = \ - self.mock_client - - def test_create_nearline_request(self): - execute = self.mock_client.transferJobs.return_value.create.return_value \ - .execute - execute.return_value = "" - main() - execute.assert_called_with() diff --git a/storage/storage_transfer/test_transfer_check.py b/storage/storage_transfer/test_transfer_check.py deleted file mode 100644 index cd3c6886979c..000000000000 --- a/storage/storage_transfer/test_transfer_check.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import unittest - -from mock import Mock - -from transfer_check import check_operation - - -class CheckTransferTestCase(unittest.TestCase): - """A test case for querying transfer job completion.""" - - def test_check_operation(self): - mock_client = Mock(spec=['transferOperations']) - execute = mock_client.transferOperations.return_value.list.return_value \ - .execute - project_id = "" - job_name = "" - check_operation(mock_client, project_id, job_name) - execute.assert_called_with() diff --git a/storage/storage_transfer/README.md b/storage/transfer_service/README.md similarity index 100% rename from storage/storage_transfer/README.md rename to storage/transfer_service/README.md diff --git a/storage/transfer_service/__init__.py b/storage/transfer_service/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/storage/storage_transfer/aws_request.py b/storage/transfer_service/aws_request.py similarity index 84% rename from storage/storage_transfer/aws_request.py rename to storage/transfer_service/aws_request.py index 40274175335a..808a5754b956 100644 --- a/storage/storage_transfer/aws_request.py +++ b/storage/transfer_service/aws_request.py @@ -15,13 +15,18 @@ import json import logging -import create_client +from apiclient import discovery +from oauth2client.client import GoogleCredentials + + +logging.basicConfig(level=logging.DEBUG) def main(): """Create a one-off transfer from Amazon S3 to GCS.""" - logging.getLogger().setLevel(logging.DEBUG) - transfer_service_client = create_client.create_transfer_client() + credentials = GoogleCredentials.get_application_default() + storagetransfer = discovery.build( + 'storagetransfer', 'v1', credentials=credentials) # Edit this template with desired parameters. # Specify times below using US Pacific Time Zone. @@ -61,7 +66,7 @@ def main(): } ''' - result = transfer_service_client.transferJobs().create(body=json.loads( + result = storagetransfer.transferJobs().create(body=json.loads( transfer_job)).execute() logging.info('Returned transferJob: %s', json.dumps(result, indent=4)) diff --git a/storage/storage_transfer/create_client.py b/storage/transfer_service/create_client.py similarity index 100% rename from storage/storage_transfer/create_client.py rename to storage/transfer_service/create_client.py diff --git a/storage/storage_transfer/nearline_request.py b/storage/transfer_service/nearline_request.py similarity index 83% rename from storage/storage_transfer/nearline_request.py rename to storage/transfer_service/nearline_request.py index d31bc1907200..44e108df0aa4 100644 --- a/storage/storage_transfer/nearline_request.py +++ b/storage/transfer_service/nearline_request.py @@ -15,13 +15,18 @@ import json import logging -import create_client +from apiclient import discovery +from oauth2client.client import GoogleCredentials + + +logging.basicConfig(level=logging.DEBUG) def main(): """Transfer from standard Cloud Storage to Cloud Storage Nearline.""" - logging.getLogger().setLevel(logging.DEBUG) - transfer_service_client = create_client.create_transfer_client() + credentials = GoogleCredentials.get_application_default() + storagetransfer = discovery.build( + 'storagetransfer', 'v1', credentials=credentials) # Edit this template with desired parameters. # Specify times below using US Pacific Time Zone. @@ -57,7 +62,7 @@ def main(): } } ''' - result = transfer_service_client.transferJobs().create(body=json.loads( + result = storagetransfer.transferJobs().create(body=json.loads( transfer_job)).execute() logging.info('Returned transferJob: %s', json.dumps(result, indent=4)) diff --git a/storage/storage_transfer/transfer_check.py b/storage/transfer_service/transfer_check.py similarity index 66% rename from storage/storage_transfer/transfer_check.py rename to storage/transfer_service/transfer_check.py index 7e9055fe49be..18a87d676438 100644 --- a/storage/storage_transfer/transfer_check.py +++ b/storage/transfer_service/transfer_check.py @@ -15,29 +15,34 @@ import json import logging -import create_client +from apiclient import discovery +from oauth2client.client import GoogleCredentials + + +logging.basicConfig(level=logging.DEBUG) # Edit these values with desired parameters. PROJECT_ID = 'YOUR_PROJECT_ID' JOB_NAME = 'YOUR_JOB_NAME' -def check_operation(transfer_service_client, project_id, job_name): +def check_operation(storagetransfer, project_id, job_name): """Review the transfer operations associated with a transfer job.""" filterString = ( '{{"project_id": "{project_id}", ' - '"job_names": ["{job_name}"]}}').format( - project_id=project_id, job_name=job_name) - return transfer_service_client.transferOperations().list( + '"job_names": ["{job_name}"]}}' + ).format(project_id=project_id, job_name=job_name) + return storagetransfer.transferOperations().list( name="transferOperations", filter=filterString).execute() def main(): - logging.getLogger().setLevel(logging.DEBUG) - transfer_service_client = create_client.create_transfer_client() + credentials = GoogleCredentials.get_application_default() + storagetransfer = discovery.build( + 'storagetransfer', 'v1', credentials=credentials) - result = check_operation(transfer_service_client, PROJECT_ID, JOB_NAME) + result = check_operation(storagetransfer, PROJECT_ID, JOB_NAME) logging.info('Result of transferOperations/list: %s', json.dumps(result, indent=4, sort_keys=True))