From f8c0fb612c8fddb7edca2b27015adaee0b6e90d1 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 22 Feb 2017 16:58:16 -0500 Subject: [PATCH 1/3] Add 'Blob.update_storage_class' API method. Closes #2991. --- storage/google/cloud/storage/blob.py | 32 +++++++++++++++++++++ storage/unit_tests/test_blob.py | 42 ++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/storage/google/cloud/storage/blob.py b/storage/google/cloud/storage/blob.py index 8d30a94b600b..d097c5eaafcc 100644 --- a/storage/google/cloud/storage/blob.py +++ b/storage/google/cloud/storage/blob.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=too-many-lines + """Create / interact with Google Cloud Storage blobs.""" import base64 @@ -73,6 +75,9 @@ class Blob(_PropertyMixin): _CHUNK_SIZE_MULTIPLE = 256 * 1024 """Number (256 KB, in bytes) that must divide the chunk size.""" + _STORAGE_CLASSES = ( + 'STANDARD', 'NEARLINE', 'MULTI_REGIONAL', 'REGIONAL', 'COLDLINE') + def __init__(self, name, bucket, chunk_size=None, encryption_key=None): super(Blob, self).__init__(name=name) @@ -852,6 +857,33 @@ def rewrite(self, source, token=None, client=None): return api_response['rewriteToken'], rewritten, size + def update_storage_class(self, new_class, client=None): + """Update blob's storage class via a rewrite-in-place. + + See: + https://cloud.google.com/storage/docs/per-object-storage-class + + :type new_class: str + :param new_class: new storage class for the object + + :type client: :class:`~google.cloud.storage.client.Client` + :param client: Optional. The client to use. If not passed, falls back + to the ``client`` stored on the blob's bucket. + """ + if new_class not in self._STORAGE_CLASSES: + raise ValueError("Invalid storage class: %s" % (new_class,)) + + client = self._require_client(client) + headers = _get_encryption_headers(self._encryption_key) + headers.update(_get_encryption_headers( + self._encryption_key, source=True)) + + api_response = client._connection.api_request( + method='POST', path=self.path + '/rewriteTo' + self.path, + data={'storageClass': new_class}, headers=headers, + _target_object=self) + self._set_properties(api_response['resource']) + cache_control = _scalar_property('cacheControl') """HTTP 'Cache-Control' header for this object. diff --git a/storage/unit_tests/test_blob.py b/storage/unit_tests/test_blob.py index 9724fd7a4210..f14fc0b2af05 100644 --- a/storage/unit_tests/test_blob.py +++ b/storage/unit_tests/test_blob.py @@ -1446,6 +1446,48 @@ def test_rewrite_same_name_no_key_new_key_w_token(self): self.assertEqual( headers['X-Goog-Encryption-Key-Sha256'], DEST_KEY_HASH_B64) + def test_update_storage_class_invalid(self): + BLOB_NAME = 'blob-name' + bucket = _Bucket() + blob = self._make_one(BLOB_NAME, bucket=bucket) + with self.assertRaises(ValueError): + blob.update_storage_class(u'BOGUS') + + def test_update_storage_class_setter_valid(self): + from six.moves.http_client import OK + BLOB_NAME = 'blob-name' + STORAGE_CLASS = u'NEARLINE' + RESPONSE = { + 'resource': {'storageClass': STORAGE_CLASS}, + } + response = ({'status': OK}, RESPONSE) + connection = _Connection(response) + client = _Client(connection) + bucket = _Bucket(client=client) + blob = self._make_one(BLOB_NAME, bucket=bucket) + + blob.update_storage_class('NEARLINE') + + self.assertEqual(blob.storage_class, 'NEARLINE') + + kw = connection._requested + self.assertEqual(len(kw), 1) + self.assertEqual(kw[0]['method'], 'POST') + PATH = '/b/name/o/%s/rewriteTo/b/name/o/%s' % (BLOB_NAME, BLOB_NAME) + self.assertEqual(kw[0]['path'], PATH) + self.assertNotIn('query_params', kw[0]) + SENT = {'storageClass': STORAGE_CLASS} + self.assertEqual(kw[0]['data'], SENT) + + headers = { + key.title(): str(value) for key, value in kw[0]['headers'].items()} + self.assertNotIn('X-Goog-Copy-Source-Encryption-Algorithm', headers) + self.assertNotIn('X-Goog-Copy-Source-Encryption-Key', headers) + self.assertNotIn('X-Goog-Copy-Source-Encryption-Key-Sha256', headers) + self.assertNotIn('X-Goog-Encryption-Algorithm', headers) + self.assertNotIn('X-Goog-Encryption-Key', headers) + self.assertNotIn('X-Goog-Encryption-Key-Sha256', headers) + def test_cache_control_getter(self): BLOB_NAME = 'blob-name' bucket = _Bucket() From fb17115f753d4becbc518558dee6502ab1b0ad0c Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Thu, 23 Feb 2017 14:46:17 -0500 Subject: [PATCH 2/3] Add unit test for 'Blob.update_storage_class' w/ enc. key. Addresses: https://github.com/GoogleCloudPlatform/google-cloud-python/pull/3051#discussion_r102594592 --- storage/unit_tests/test_blob.py | 55 ++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/storage/unit_tests/test_blob.py b/storage/unit_tests/test_blob.py index f14fc0b2af05..d0de26ef3bbc 100644 --- a/storage/unit_tests/test_blob.py +++ b/storage/unit_tests/test_blob.py @@ -1453,7 +1453,7 @@ def test_update_storage_class_invalid(self): with self.assertRaises(ValueError): blob.update_storage_class(u'BOGUS') - def test_update_storage_class_setter_valid(self): + def test_update_storage_class_wo_encryption_key(self): from six.moves.http_client import OK BLOB_NAME = 'blob-name' STORAGE_CLASS = u'NEARLINE' @@ -1481,6 +1481,7 @@ def test_update_storage_class_setter_valid(self): headers = { key.title(): str(value) for key, value in kw[0]['headers'].items()} + # Blob has no key, and therefore the relevant headers are not sent. self.assertNotIn('X-Goog-Copy-Source-Encryption-Algorithm', headers) self.assertNotIn('X-Goog-Copy-Source-Encryption-Key', headers) self.assertNotIn('X-Goog-Copy-Source-Encryption-Key-Sha256', headers) @@ -1488,6 +1489,58 @@ def test_update_storage_class_setter_valid(self): self.assertNotIn('X-Goog-Encryption-Key', headers) self.assertNotIn('X-Goog-Encryption-Key-Sha256', headers) + def test_update_storage_class_w_encryption_key(self): + import base64 + import hashlib + from six.moves.http_client import OK + + BLOB_NAME = 'blob-name' + BLOB_KEY = b'01234567890123456789012345678901' # 32 bytes + BLOB_KEY_B64 = base64.b64encode(BLOB_KEY).rstrip().decode('ascii') + BLOB_KEY_HASH = hashlib.sha256(BLOB_KEY).digest() + BLOB_KEY_HASH_B64 = base64.b64encode( + BLOB_KEY_HASH).rstrip().decode('ascii') + STORAGE_CLASS = u'NEARLINE' + RESPONSE = { + 'resource': {'storageClass': STORAGE_CLASS}, + } + response = ({'status': OK}, RESPONSE) + connection = _Connection(response) + client = _Client(connection) + bucket = _Bucket(client=client) + blob = self._make_one( + BLOB_NAME, bucket=bucket, encryption_key=BLOB_KEY) + + blob.update_storage_class('NEARLINE') + + self.assertEqual(blob.storage_class, 'NEARLINE') + + kw = connection._requested + self.assertEqual(len(kw), 1) + self.assertEqual(kw[0]['method'], 'POST') + PATH = '/b/name/o/%s/rewriteTo/b/name/o/%s' % (BLOB_NAME, BLOB_NAME) + self.assertEqual(kw[0]['path'], PATH) + self.assertNotIn('query_params', kw[0]) + SENT = {'storageClass': STORAGE_CLASS} + self.assertEqual(kw[0]['data'], SENT) + + headers = { + key.title(): str(value) for key, value in kw[0]['headers'].items()} + # Blob has key, and therefore the relevant headers are sent. + self.assertEqual( + headers['X-Goog-Copy-Source-Encryption-Algorithm'], 'AES256') + self.assertEqual( + headers['X-Goog-Copy-Source-Encryption-Key'], BLOB_KEY_B64) + self.assertEqual( + headers['X-Goog-Copy-Source-Encryption-Key-Sha256'], + BLOB_KEY_HASH_B64) + self.assertEqual( + headers['X-Goog-Encryption-Algorithm'], 'AES256') + self.assertEqual( + headers['X-Goog-Encryption-Key'], BLOB_KEY_B64) + self.assertEqual( + headers['X-Goog-Encryption-Key-Sha256'], BLOB_KEY_HASH_B64) + def test_cache_control_getter(self): BLOB_NAME = 'blob-name' bucket = _Bucket() From 7239c9e3bf3298a5f83ca451765d515e001b9cee Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 24 Feb 2017 13:01:47 -0500 Subject: [PATCH 3/3] Document '{Bucket,Blob}._STORAGE_CLASSES'. Re-order to match canonical order in docs. Add docstrings with links to relevant docs. Explain why the two lists differ. --- storage/google/cloud/storage/blob.py | 22 +++++++++++++++++++++- storage/google/cloud/storage/bucket.py | 16 ++++++++++++++-- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/storage/google/cloud/storage/blob.py b/storage/google/cloud/storage/blob.py index d097c5eaafcc..6af15e3e9a66 100644 --- a/storage/google/cloud/storage/blob.py +++ b/storage/google/cloud/storage/blob.py @@ -76,7 +76,27 @@ class Blob(_PropertyMixin): """Number (256 KB, in bytes) that must divide the chunk size.""" _STORAGE_CLASSES = ( - 'STANDARD', 'NEARLINE', 'MULTI_REGIONAL', 'REGIONAL', 'COLDLINE') + 'NEARLINE', + 'MULTI_REGIONAL', + 'REGIONAL', + 'COLDLINE', + 'STANDARD', # alias for MULTI_REGIONAL/REGIONAL, based on location + ) + """Allowed values for :attr:`storage_class`. + + See: + https://cloud.google.com/storage/docs/json_api/v1/objects#storageClass + https://cloud.google.com/storage/docs/per-object-storage-class + + .. note:: + This list does not include 'DURABLE_REDUCED_AVAILABILITY', which + is only documented for buckets (and deprectated. + + .. note:: + The documentation does *not* mention 'STANDARD', but it is the value + assigned by the back-end for objects created in buckets with 'STANDARD' + set as their 'storage_class'. + """ def __init__(self, name, bucket, chunk_size=None, encryption_key=None): super(Blob, self).__init__(name=name) diff --git a/storage/google/cloud/storage/bucket.py b/storage/google/cloud/storage/bucket.py index 04417094f258..f54791785d6b 100644 --- a/storage/google/cloud/storage/bucket.py +++ b/storage/google/cloud/storage/bucket.py @@ -90,8 +90,20 @@ class Bucket(_PropertyMixin): This is used in Bucket.delete() and Bucket.make_public(). """ - _STORAGE_CLASSES = ('STANDARD', 'NEARLINE', 'DURABLE_REDUCED_AVAILABILITY', - 'MULTI_REGIONAL', 'REGIONAL', 'COLDLINE') + _STORAGE_CLASSES = ( + 'MULTI_REGIONAL', + 'REGIONAL', + 'NEARLINE', + 'COLDLINE', + 'STANDARD', # alias for MULTI_REGIONAL/REGIONAL, based on location + 'DURABLE_REDUCED_AVAILABILITY', # deprecated + ) + """Allowed values for :attr:`storage_class`. + + See: + https://cloud.google.com/storage/docs/json_api/v1/buckets#storageClass + https://cloud.google.com/storage/docs/storage-classes + """ def __init__(self, client, name=None): super(Bucket, self).__init__(name=name)