From 1fa508f35d05ff211c6916354d0e35a13892fec2 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Mon, 13 Oct 2014 16:11:07 -0700 Subject: [PATCH] Documenting undocumented classes/methods/etc. in gcloud.storage. Also did tiny clean-up along the way in places I noticed code issues. Another thing, I suggest a rename of _bytes_written in storage.iterator.KeyDataIterator to something like _bytes_read or _bytes_consumed, i.e. something which actually matches the action being taken. --- gcloud/storage/bucket.py | 9 +++++++ gcloud/storage/connection.py | 2 ++ gcloud/storage/exceptions.py | 11 ++++---- gcloud/storage/iterator.py | 46 ++++++++++++++++++++++++++++++--- gcloud/storage/key.py | 6 +++++ gcloud/storage/test_iterator.py | 3 ++- 6 files changed, 68 insertions(+), 9 deletions(-) diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index 046944082795..d6ab28f88bdd 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -1,3 +1,5 @@ +"""Create / interact with gcloud storage buckets.""" + import os from gcloud.storage import exceptions @@ -187,6 +189,13 @@ def delete_key(self, key): return key def delete_keys(self, keys): + """Deletes a list of keys from the current bucket. + + Uses :func:`Bucket.delete_key` to delete each individual key. + + :type keys: list of string or :class:`gcloud.storage.key.Key` + :param key: A list of key names or Key objects to delete. + """ # NOTE: boto returns a MultiDeleteResult instance. for key in keys: self.delete_key(key) diff --git a/gcloud/storage/connection.py b/gcloud/storage/connection.py index f8d4d28a1a28..b51058f82edd 100644 --- a/gcloud/storage/connection.py +++ b/gcloud/storage/connection.py @@ -1,3 +1,5 @@ +"""Create / interact with gcloud storage connections.""" + import base64 import datetime import json diff --git a/gcloud/storage/exceptions.py b/gcloud/storage/exceptions.py index 18433b6041a4..80785efe45f3 100644 --- a/gcloud/storage/exceptions.py +++ b/gcloud/storage/exceptions.py @@ -1,8 +1,12 @@ +"""Custom exceptions for gcloud.storage package.""" + + class StorageError(Exception): - pass + """Base error class for gcloud errors.""" class ConnectionError(StorageError): + """Exception corresponding to a bad HTTP/RPC connection.""" def __init__(self, response, content): message = str(response) + content @@ -12,10 +16,7 @@ def __init__(self, response, content): class NotFoundError(ConnectionError): + """Exception corresponding to a 404 not found bad connection.""" def __init__(self, response, content): self.message = 'Request returned a 404. Headers: %s' % (response) - - -class StorageDataError(StorageError): - pass diff --git a/gcloud/storage/iterator.py b/gcloud/storage/iterator.py index 3aecaab3982d..b69c6d70a768 100644 --- a/gcloud/storage/iterator.py +++ b/gcloud/storage/iterator.py @@ -40,6 +40,9 @@ def get_items_from_response(self, response): """ +from gcloud.storage.exceptions import StorageError + + class Iterator(object): """A generic class for iterating through Cloud Storage list responses. @@ -186,6 +189,15 @@ def get_items_from_response(self, response): class KeyDataIterator(object): + """An iterator listing data stored in a key. + + You shouldn't have to use this directly, + but instead should use the helper methods + on :class:`gcloud.storage.key.Key` objects. + + :type key: :class:`gcloud.storage.key.Key` + :param key: The key from which to list data.. + """ def __init__(self, key): self.key = key @@ -196,20 +208,33 @@ def __iter__(self): yield self.get_next_chunk() def reset(self): + """Resets the iterator to the beginning.""" self._bytes_written = 0 self._total_bytes = None def has_more_data(self): + """Determines whether or not this iterator has more data to read. + + :rtype: bool + :returns: Whether the iterator has more data or not. + """ + if self._bytes_written == 0: return True elif not self._total_bytes: # self._total_bytes **should** be set by this point. # If it isn't, something is wrong. - raise ValueError('Size of object is unknown... This is bad.') + raise ValueError('Size of object is unknown.') else: return (self._bytes_written < self._total_bytes) def get_headers(self): + """Gets range header(s) for next chunk of data. + + :rtype: dict + :returns: A dictionary of query parameters. + """ + start = self._bytes_written end = self._bytes_written + self.key.CHUNK_SIZE - 1 @@ -219,10 +244,25 @@ def get_headers(self): return {'Range': 'bytes=%s-%s' % (start, end)} def get_url(self): + """Gets URL to read next chunk of data. + + :rtype: string + :returns: A URL. + """ return self.key.connection.build_api_url( path=self.key.path, query_params={'alt': 'media'}) def get_next_chunk(self): + """Gets the next chunk of data. + + Uses CHUNK_SIZE to determine how much data to get. + + :rtype: string + :returns: The chunk of data read from the key. + :raises: :class:`RuntimeError` if no more data or + :class:`gcloud.storage.exceptions.StorageError` in the + case of an unexpected response status code. + """ if not self.has_more_data(): raise RuntimeError('No more data in this iterator. Try resetting.') @@ -238,5 +278,5 @@ def get_next_chunk(self): return content - # Expected a 200 or a 206... Got something else, which is bad. - raise Exception(response) + # Expected a 200 or a 206. Got something else, which is unknown. + raise StorageError(response) diff --git a/gcloud/storage/key.py b/gcloud/storage/key.py index 0e17f93902a0..73f4f799da40 100644 --- a/gcloud/storage/key.py +++ b/gcloud/storage/key.py @@ -1,3 +1,5 @@ +"""Create / interact with gcloud storage keys.""" + import errno import mimetypes import os @@ -92,6 +94,10 @@ def path(self): @property def public_url(self): + """ + :rtype: `string` + :returns: The public URL for this key. + """ return '{storage_base_url}/{self.bucket.name}/{self.name}'.format( storage_base_url='http://commondatastorage.googleapis.com', self=self) diff --git a/gcloud/storage/test_iterator.py b/gcloud/storage/test_iterator.py index 738b3000b300..ba197d3b727c 100644 --- a/gcloud/storage/test_iterator.py +++ b/gcloud/storage/test_iterator.py @@ -344,12 +344,13 @@ def test_get_next_chunk_206(self): self.assertEqual(kw['headers'], {'Range': 'bytes=0-9'}) def test_get_next_chunk_416(self): + from gcloud.storage.exceptions import StorageError response = _Response(status=416) connection = _Connection((response, '')) key = _Key(connection) iterator = self._makeOne(key) iterator._total_bytes = 1000 - self.assertRaises(Exception, iterator.get_next_chunk) + self.assertRaises(StorageError, iterator.get_next_chunk) class _Response(dict):