Skip to content

Commit

Permalink
Documenting undocumented classes/methods/etc. in gcloud.storage.
Browse files Browse the repository at this point in the history
Also did tiny clean-up along the way in places I noticed
code issues.

Another thing, I suggest a rename of _bytes_written in
storage.iterator.KeyDataIterator to something like
_bytes_read or _bytes_consumed, i.e. something which
actually matches the action being taken.
  • Loading branch information
dhermes committed Oct 14, 2014
1 parent 11c8765 commit 1fa508f
Show file tree
Hide file tree
Showing 6 changed files with 68 additions and 9 deletions.
9 changes: 9 additions & 0 deletions gcloud/storage/bucket.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
"""Create / interact with gcloud storage buckets."""

import os

from gcloud.storage import exceptions
Expand Down Expand Up @@ -187,6 +189,13 @@ def delete_key(self, key):
return key

def delete_keys(self, keys):
"""Deletes a list of keys from the current bucket.
Uses :func:`Bucket.delete_key` to delete each individual key.
:type keys: list of string or :class:`gcloud.storage.key.Key`
:param key: A list of key names or Key objects to delete.
"""
# NOTE: boto returns a MultiDeleteResult instance.
for key in keys:
self.delete_key(key)
Expand Down
2 changes: 2 additions & 0 deletions gcloud/storage/connection.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
"""Create / interact with gcloud storage connections."""

import base64
import datetime
import json
Expand Down
11 changes: 6 additions & 5 deletions gcloud/storage/exceptions.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,12 @@
"""Custom exceptions for gcloud.storage package."""


class StorageError(Exception):
pass
"""Base error class for gcloud errors."""


class ConnectionError(StorageError):
"""Exception corresponding to a bad HTTP/RPC connection."""

def __init__(self, response, content):
message = str(response) + content
Expand All @@ -12,10 +16,7 @@ def __init__(self, response, content):


class NotFoundError(ConnectionError):
"""Exception corresponding to a 404 not found bad connection."""

def __init__(self, response, content):
self.message = 'Request returned a 404. Headers: %s' % (response)


class StorageDataError(StorageError):
pass
46 changes: 43 additions & 3 deletions gcloud/storage/iterator.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ def get_items_from_response(self, response):
"""


from gcloud.storage.exceptions import StorageError


class Iterator(object):
"""A generic class for iterating through Cloud Storage list responses.
Expand Down Expand Up @@ -186,6 +189,15 @@ def get_items_from_response(self, response):


class KeyDataIterator(object):
"""An iterator listing data stored in a key.
You shouldn't have to use this directly,
but instead should use the helper methods
on :class:`gcloud.storage.key.Key` objects.
:type key: :class:`gcloud.storage.key.Key`
:param key: The key from which to list data..
"""

def __init__(self, key):
self.key = key
Expand All @@ -196,20 +208,33 @@ def __iter__(self):
yield self.get_next_chunk()

def reset(self):
"""Resets the iterator to the beginning."""
self._bytes_written = 0
self._total_bytes = None

def has_more_data(self):
"""Determines whether or not this iterator has more data to read.
:rtype: bool
:returns: Whether the iterator has more data or not.
"""

if self._bytes_written == 0:
return True
elif not self._total_bytes:
# self._total_bytes **should** be set by this point.
# If it isn't, something is wrong.
raise ValueError('Size of object is unknown... This is bad.')
raise ValueError('Size of object is unknown.')
else:
return (self._bytes_written < self._total_bytes)

def get_headers(self):
"""Gets range header(s) for next chunk of data.
:rtype: dict
:returns: A dictionary of query parameters.
"""

start = self._bytes_written
end = self._bytes_written + self.key.CHUNK_SIZE - 1

Expand All @@ -219,10 +244,25 @@ def get_headers(self):
return {'Range': 'bytes=%s-%s' % (start, end)}

def get_url(self):
"""Gets URL to read next chunk of data.
:rtype: string
:returns: A URL.
"""
return self.key.connection.build_api_url(
path=self.key.path, query_params={'alt': 'media'})

def get_next_chunk(self):
"""Gets the next chunk of data.
Uses CHUNK_SIZE to determine how much data to get.
:rtype: string
:returns: The chunk of data read from the key.
:raises: :class:`RuntimeError` if no more data or
:class:`gcloud.storage.exceptions.StorageError` in the
case of an unexpected response status code.
"""
if not self.has_more_data():
raise RuntimeError('No more data in this iterator. Try resetting.')

Expand All @@ -238,5 +278,5 @@ def get_next_chunk(self):

return content

# Expected a 200 or a 206... Got something else, which is bad.
raise Exception(response)
# Expected a 200 or a 206. Got something else, which is unknown.
raise StorageError(response)
6 changes: 6 additions & 0 deletions gcloud/storage/key.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
"""Create / interact with gcloud storage keys."""

import errno
import mimetypes
import os
Expand Down Expand Up @@ -92,6 +94,10 @@ def path(self):

@property
def public_url(self):
"""
:rtype: `string`
:returns: The public URL for this key.
"""
return '{storage_base_url}/{self.bucket.name}/{self.name}'.format(
storage_base_url='http://commondatastorage.googleapis.com',
self=self)
Expand Down
3 changes: 2 additions & 1 deletion gcloud/storage/test_iterator.py
Original file line number Diff line number Diff line change
Expand Up @@ -344,12 +344,13 @@ def test_get_next_chunk_206(self):
self.assertEqual(kw['headers'], {'Range': 'bytes=0-9'})

def test_get_next_chunk_416(self):
from gcloud.storage.exceptions import StorageError
response = _Response(status=416)
connection = _Connection((response, ''))
key = _Key(connection)
iterator = self._makeOne(key)
iterator._total_bytes = 1000
self.assertRaises(Exception, iterator.get_next_chunk)
self.assertRaises(StorageError, iterator.get_next_chunk)


class _Response(dict):
Expand Down

0 comments on commit 1fa508f

Please sign in to comment.