From bd4610393d367f0ad557c2fbb86ee283c42593d6 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Mon, 20 Oct 2014 12:55:37 -0700 Subject: [PATCH 1/2] Resolving cyclic imports in storage package. This is accomplished by moving the definitions of BucketIterator and KeyIterator into their respective modules (i.e. storage.bucket and storage.key). This is not so big an issue because the parent class iterator.Iterator does most of the work and the children only require a small tweak. --- gcloud/storage/bucket.py | 27 +++++++++++- gcloud/storage/connection.py | 2 +- gcloud/storage/iterator.py | 54 ----------------------- gcloud/storage/key.py | 27 ++++++++++++ gcloud/storage/test_bucket.py | 40 ++++++++++++++++- gcloud/storage/test_iterator.py | 76 --------------------------------- gcloud/storage/test_key.py | 40 +++++++++++++++++ 7 files changed, 132 insertions(+), 134 deletions(-) diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index f6c97ceef109..e9d26018949c 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -5,8 +5,9 @@ from gcloud.storage import exceptions from gcloud.storage.acl import BucketACL from gcloud.storage.acl import DefaultObjectACL -from gcloud.storage.iterator import KeyIterator +from gcloud.storage.iterator import Iterator from gcloud.storage.key import Key +from gcloud.storage.key import KeyIterator class Bucket(object): @@ -637,3 +638,27 @@ def make_public(self, recursive=False, future=False): for key in self: key.get_acl().all().grant_read() key.save_acl() + + +class BucketIterator(Iterator): + """An iterator listing all buckets. + + You shouldn't have to use this directly, + but instead should use the helper methods + on :class:`gcloud.storage.connection.Connection` objects. + + :type connection: :class:`gcloud.storage.connection.Connection` + :param connection: The connection to use for querying the list of buckets. + """ + + def __init__(self, connection): + super(BucketIterator, self).__init__(connection=connection, path='/b') + + def get_items_from_response(self, response): + """Factory method which yields :class:`.Bucket` items from a response. + + :type response: dict + :param response: The JSON API response for a page of buckets. + """ + for item in response.get('items', []): + yield Bucket.from_dict(item, connection=self.connection) diff --git a/gcloud/storage/connection.py b/gcloud/storage/connection.py index 55b76ce3688d..9fd0b7d84b05 100644 --- a/gcloud/storage/connection.py +++ b/gcloud/storage/connection.py @@ -15,7 +15,7 @@ from gcloud import connection from gcloud.storage import exceptions from gcloud.storage.bucket import Bucket -from gcloud.storage.iterator import BucketIterator +from gcloud.storage.bucket import BucketIterator def _utcnow(): # pragma: NO COVER testing replaces diff --git a/gcloud/storage/iterator.py b/gcloud/storage/iterator.py index 9bc01cb1eb56..144ac2fbabc5 100644 --- a/gcloud/storage/iterator.py +++ b/gcloud/storage/iterator.py @@ -134,60 +134,6 @@ def get_items_from_response(self, response): raise NotImplementedError -class BucketIterator(Iterator): - """An iterator listing all buckets. - - You shouldn't have to use this directly, - but instead should use the helper methods - on :class:`gcloud.storage.connection.Connection` objects. - - :type connection: :class:`gcloud.storage.connection.Connection` - :param connection: The connection to use for querying the list of buckets. - """ - - def __init__(self, connection): - super(BucketIterator, self).__init__(connection=connection, path='/b') - - def get_items_from_response(self, response): - """Factory method which yields :class:`.Bucket` items from a response. - - :type response: dict - :param response: The JSON API response for a page of buckets. - """ - - from gcloud.storage.bucket import Bucket - for item in response.get('items', []): - yield Bucket.from_dict(item, connection=self.connection) - - -class KeyIterator(Iterator): - """An iterator listing keys. - - You shouldn't have to use this directly, - but instead should use the helper methods - on :class:`gcloud.storage.key.Key` objects. - - :type bucket: :class:`gcloud.storage.bucket.Bucket` - :param bucket: The bucket from which to list keys. - """ - - def __init__(self, bucket): - self.bucket = bucket - super(KeyIterator, self).__init__( - connection=bucket.connection, path=bucket.path + '/o') - - def get_items_from_response(self, response): - """Factory method, yields :class:`.storage.key.Key` items from response. - - :type response: dict - :param response: The JSON API response for a page of keys. - """ - - from gcloud.storage.key import Key - for item in response.get('items', []): - yield Key.from_dict(item, bucket=self.bucket) - - class KeyDataIterator(object): """An iterator listing data stored in a key. diff --git a/gcloud/storage/key.py b/gcloud/storage/key.py index fa19a9afc05f..099c8ddbb62e 100644 --- a/gcloud/storage/key.py +++ b/gcloud/storage/key.py @@ -5,6 +5,7 @@ from StringIO import StringIO from gcloud.storage.acl import ObjectACL +from gcloud.storage.iterator import Iterator from gcloud.storage.iterator import KeyDataIterator @@ -459,3 +460,29 @@ def make_public(self): self.get_acl().all().grant_read() self.save_acl() return self + + +class KeyIterator(Iterator): + """An iterator listing keys. + + You shouldn't have to use this directly, + but instead should use the helper methods + on :class:`gcloud.storage.key.Key` objects. + + :type bucket: :class:`gcloud.storage.bucket.Bucket` + :param bucket: The bucket from which to list keys. + """ + + def __init__(self, bucket): + self.bucket = bucket + super(KeyIterator, self).__init__( + connection=bucket.connection, path=bucket.path + '/o') + + def get_items_from_response(self, response): + """Factory method, yields :class:`.storage.key.Key` items from response. + + :type response: dict + :param response: The JSON API response for a page of keys. + """ + for item in response.get('items', []): + yield Key.from_dict(item, bucket=self.bucket) diff --git a/gcloud/storage/test_bucket.py b/gcloud/storage/test_bucket.py index f0ba26346121..8ae603a67cd0 100644 --- a/gcloud/storage/test_bucket.py +++ b/gcloud/storage/test_bucket.py @@ -840,7 +840,7 @@ def test_make_public_w_future(self): def test_make_public_recursive(self): from gcloud.storage.acl import _ACLEntity from gcloud._testing import _Monkey - from gcloud.storage import iterator + from gcloud.storage import key from gcloud.storage import bucket as MUT _saved = [] @@ -863,7 +863,7 @@ def grant_read(self): def save_acl(self): _saved.append((self._bucket, self._name, self._granted)) - class _KeyIterator(iterator.KeyIterator): + class _KeyIterator(key.KeyIterator): def get_items_from_response(self, response): for item in response.get('items', []): yield _Key(self.bucket, item['name']) @@ -892,6 +892,42 @@ def get_items_from_response(self, response): self.assertEqual(kw[1]['query_params'], None) +class TestBucketIterator(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.storage.bucket import BucketIterator + return BucketIterator + + def _makeOne(self, *args, **kw): + return self._getTargetClass()(*args, **kw) + + def test_ctor(self): + connection = _Connection() + iterator = self._makeOne(connection) + self.assertTrue(iterator.connection is connection) + self.assertEqual(iterator.path, '/b') + self.assertEqual(iterator.page_number, 0) + self.assertEqual(iterator.next_page_token, None) + + def test_get_items_from_response_empty(self): + connection = _Connection() + iterator = self._makeOne(connection) + self.assertEqual(list(iterator.get_items_from_response({})), []) + + def test_get_items_from_response_non_empty(self): + from gcloud.storage.bucket import Bucket + KEY = 'key' + response = {'items': [{'name': KEY}]} + connection = _Connection() + iterator = self._makeOne(connection) + buckets = list(iterator.get_items_from_response(response)) + self.assertEqual(len(buckets), 1) + bucket = buckets[0] + self.assertTrue(isinstance(bucket, Bucket)) + self.assertTrue(bucket.connection is connection) + self.assertEqual(bucket.name, KEY) + + class _Connection(object): _delete_ok = False diff --git a/gcloud/storage/test_iterator.py b/gcloud/storage/test_iterator.py index 6dc1c9005bbc..8a0006ed0352 100644 --- a/gcloud/storage/test_iterator.py +++ b/gcloud/storage/test_iterator.py @@ -118,82 +118,6 @@ def test_get_items_from_response_raises_NotImplementedError(self): iterator.get_items_from_response, object()) -class TestBucketIterator(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.storage.iterator import BucketIterator - return BucketIterator - - def _makeOne(self, *args, **kw): - return self._getTargetClass()(*args, **kw) - - def test_ctor(self): - connection = _Connection() - iterator = self._makeOne(connection) - self.assertTrue(iterator.connection is connection) - self.assertEqual(iterator.path, '/b') - self.assertEqual(iterator.page_number, 0) - self.assertEqual(iterator.next_page_token, None) - - def test_get_items_from_response_empty(self): - connection = _Connection() - iterator = self._makeOne(connection) - self.assertEqual(list(iterator.get_items_from_response({})), []) - - def test_get_items_from_response_non_empty(self): - from gcloud.storage.bucket import Bucket - KEY = 'key' - response = {'items': [{'name': KEY}]} - connection = _Connection() - iterator = self._makeOne(connection) - buckets = list(iterator.get_items_from_response(response)) - self.assertEqual(len(buckets), 1) - bucket = buckets[0] - self.assertTrue(isinstance(bucket, Bucket)) - self.assertTrue(bucket.connection is connection) - self.assertEqual(bucket.name, KEY) - - -class TestKeyIterator(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.storage.iterator import KeyIterator - return KeyIterator - - def _makeOne(self, *args, **kw): - return self._getTargetClass()(*args, **kw) - - def test_ctor(self): - connection = _Connection() - bucket = _Bucket(connection) - iterator = self._makeOne(bucket) - self.assertTrue(iterator.bucket is bucket) - self.assertTrue(iterator.connection is connection) - self.assertEqual(iterator.path, '%s/o' % bucket.path) - self.assertEqual(iterator.page_number, 0) - self.assertEqual(iterator.next_page_token, None) - - def test_get_items_from_response_empty(self): - connection = _Connection() - bucket = _Bucket(connection) - iterator = self._makeOne(bucket) - self.assertEqual(list(iterator.get_items_from_response({})), []) - - def test_get_items_from_response_non_empty(self): - from gcloud.storage.key import Key - KEY = 'key' - response = {'items': [{'name': KEY}]} - connection = _Connection() - bucket = _Bucket(connection) - iterator = self._makeOne(bucket) - keys = list(iterator.get_items_from_response(response)) - self.assertEqual(len(keys), 1) - key = keys[0] - self.assertTrue(isinstance(key, Key)) - self.assertTrue(key.connection is connection) - self.assertEqual(key.name, KEY) - - class TestKeyDataIterator(unittest2.TestCase): def _getTargetClass(self): diff --git a/gcloud/storage/test_key.py b/gcloud/storage/test_key.py index 3a5028e1ebb7..9d02a16f8c9a 100644 --- a/gcloud/storage/test_key.py +++ b/gcloud/storage/test_key.py @@ -591,6 +591,46 @@ def test_make_public(self): self.assertEqual(kw[0]['query_params'], {'projection': 'full'}) +class TestKeyIterator(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.storage.key import KeyIterator + return KeyIterator + + def _makeOne(self, *args, **kw): + return self._getTargetClass()(*args, **kw) + + def test_ctor(self): + connection = _Connection() + bucket = _Bucket(connection) + iterator = self._makeOne(bucket) + self.assertTrue(iterator.bucket is bucket) + self.assertTrue(iterator.connection is connection) + self.assertEqual(iterator.path, '%s/o' % bucket.path) + self.assertEqual(iterator.page_number, 0) + self.assertEqual(iterator.next_page_token, None) + + def test_get_items_from_response_empty(self): + connection = _Connection() + bucket = _Bucket(connection) + iterator = self._makeOne(bucket) + self.assertEqual(list(iterator.get_items_from_response({})), []) + + def test_get_items_from_response_non_empty(self): + from gcloud.storage.key import Key + KEY = 'key' + response = {'items': [{'name': KEY}]} + connection = _Connection() + bucket = _Bucket(connection) + iterator = self._makeOne(bucket) + keys = list(iterator.get_items_from_response(response)) + self.assertEqual(len(keys), 1) + key = keys[0] + self.assertTrue(isinstance(key, Key)) + self.assertTrue(key.connection is connection) + self.assertEqual(key.name, KEY) + + class _Connection(object): API_BASE_URL = 'http://example.com' From 8de933f138ab2e7a832772f21113b4bc5e694825 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Mon, 20 Oct 2014 15:20:54 -0700 Subject: [PATCH 2/2] Removed unused _Bucket class from iterator test. --- gcloud/storage/test_iterator.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/gcloud/storage/test_iterator.py b/gcloud/storage/test_iterator.py index 8a0006ed0352..8d441b1dce5c 100644 --- a/gcloud/storage/test_iterator.py +++ b/gcloud/storage/test_iterator.py @@ -313,13 +313,6 @@ def build_api_url(self, path, query_params=None): return urlunsplit(('http', 'example.com', path, qs, '')) -class _Bucket(object): - path = '/b/name' - - def __init__(self, connection): - self.connection = connection - - class _Key(object): CHUNK_SIZE = 10 path = '/b/name/o/key'