Implementing storage regression tests to match gcloud-node.

Note meant to be merged. Some missing features were
uncovered or hard to use APIs and I want to discuss here
and then map out the plan (i.e. how to break this up).

CONTRIBUTING.rst

@@ -160,7 +160,10 @@ Running Regression Tests
   so you'll need to provide some environment variables to facilitate
   authentication to your project:
 
+  - ``GCLOUD_TESTS_PROJECT_ID``: Developers Console project ID (e.g.
+    bamboo-shift-455).
   - ``GCLOUD_TESTS_DATASET_ID``: The name of the dataset your tests connect to.
+    This is typically the same as ``GCLOUD_TESTS_PROJECT_ID``.
   - ``GCLOUD_TESTS_CLIENT_EMAIL``: The email for the service account you're
     authenticating with
   - ``GCLOUD_TESTS_KEY_FILE``: The path to an encrypted key file.

gcloud/storage/bucket.py

@@ -301,7 +301,8 @@ def upload_file(self, filename, key=None):
         if key is None:
             key = os.path.basename(filename)
         key = self.new_key(key)
-        return key.upload_from_filename(filename)
+        key.upload_from_filename(filename)
+        return key
 
     def upload_file_object(self, file_obj, key=None):
         """Shortcut method to upload a file object into this bucket.

regression/local_test_setup.sample

@@ -1,3 +1,4 @@
-export GCLOUD_TESTS_DATASET_ID="my-dataset"
+export GCLOUD_TESTS_PROJECT_ID="my-project"
+export GCLOUD_TESTS_DATASET_ID=${GCLOUD_TESTS_PROJECT_ID}
 export GCLOUD_TESTS_CLIENT_EMAIL="[email protected]"
 export GCLOUD_TESTS_KEY_FILE="path.key"

regression/regression_utils.py

@@ -2,37 +2,58 @@
 import sys
 
 from gcloud import datastore
+from gcloud import storage
 
 
 # Defaults from shell environ. May be None.
+PROJECT_ID = os.getenv('GCLOUD_TESTS_PROJECT_ID')
 DATASET_ID = os.getenv('GCLOUD_TESTS_DATASET_ID')
 CLIENT_EMAIL = os.getenv('GCLOUD_TESTS_CLIENT_EMAIL')
 KEY_FILENAME = os.getenv('GCLOUD_TESTS_KEY_FILE')
-DATASETS = {}
+CACHED_RETURN_VALS = {}
 
 ENVIRON_ERROR_MSG = """\
 To run the regression tests, you need to set some environment variables.
 Please check the Contributing guide for instructions.
 """
 
 
-def get_environ():
-    if DATASET_ID is None or CLIENT_EMAIL is None or KEY_FILENAME is None:
-        print >> sys.stderr, ENVIRON_ERROR_MSG
-        sys.exit(1)
+def get_environ(require_datastore=False, require_storage=False):
+    if require_datastore:
+        if DATASET_ID is None or CLIENT_EMAIL is None or KEY_FILENAME is None:
+            print >> sys.stderr, ENVIRON_ERROR_MSG
+            sys.exit(1)
+
+    if require_storage:
+        if PROJECT_ID is None or CLIENT_EMAIL is None or KEY_FILENAME is None:
+            print >> sys.stderr, ENVIRON_ERROR_MSG
+            sys.exit(1)
 
     return {
+        'project_id': PROJECT_ID,
         'dataset_id': DATASET_ID,
         'client_email': CLIENT_EMAIL,
         'key_filename': KEY_FILENAME,
     }
 
 
 def get_dataset():
-    environ = get_environ()
+    environ = get_environ(require_datastore=True)
     get_dataset_args = (environ['dataset_id'], environ['client_email'],
                         environ['key_filename'])
-    if get_dataset_args not in DATASETS:
+    key = ('get_dataset', get_dataset_args)
+    if key not in CACHED_RETURN_VALS:
+        # Cache return value for the environment.
+        CACHED_RETURN_VALS[key] = datastore.get_dataset(*get_dataset_args)
+    return CACHED_RETURN_VALS[key]
+
+
+def get_storage_connection():
+    environ = get_environ(require_storage=True)
+    get_connection_args = (environ['project_id'], environ['client_email'],
+                           environ['key_filename'])
+    key = ('get_storage_connection', get_connection_args)
+    if key not in CACHED_RETURN_VALS:
         # Cache return value for the environment.
-        DATASETS[get_dataset_args] = datastore.get_dataset(*get_dataset_args)
-    return DATASETS[get_dataset_args]
+        CACHED_RETURN_VALS[key] = storage.get_connection(*get_connection_args)
+    return CACHED_RETURN_VALS[key]

regression/run_regression.py

@@ -11,7 +11,7 @@ def get_parser():
     parser = argparse.ArgumentParser(
         description='GCloud test runner against actual project.')
     parser.add_argument('--package', dest='package',
-                        choices=('datastore',),
+                        choices=('datastore', 'storage'),
                         default='datastore', help='Package to be tested.')
     return parser
 
@@ -27,7 +27,10 @@ def main():
     parser = get_parser()
     args = parser.parse_args()
     # Make sure environ is set before running test.
-    regression_utils.get_environ()
+    if args.package == 'datastore':
+        regression_utils.get_environ(require_datastore=True)
+    elif args.package == 'storage':
+        regression_utils.get_environ(require_storage=True)
     test_result = run_module_tests(args.package)
     if not test_result.wasSuccessful():
         sys.exit(1)

regression/storage.py

@@ -0,0 +1,247 @@
+from Crypto.Hash import MD5
+import base64
+import httplib2
+import tempfile
+import time
+import unittest2
+
+from gcloud import storage
+# This assumes the command is being run via tox hence the
+# repository root is the current directory.
+from regression import regression_utils
+
+
+HTTP = httplib2.Http()
+SHARED_BUCKETS = {}
+
+
+def setUpModule():
+    if 'test_bucket' not in SHARED_BUCKETS:
+        connection = regression_utils.get_storage_connection()
+        # %d rounds milliseconds to nearest integer.
+        bucket_name = 'new%d' % (1000 * time.time(),)
+        # In the **very** rare case the bucket name is reserved, this
+        # fails with a ConnectionError.
+        SHARED_BUCKETS['test_bucket'] = connection.create_bucket(bucket_name)
+
+
+def tearDownModule():
+    for bucket in SHARED_BUCKETS.values():
+        # Passing force=True also deletes all files.
+        bucket.delete(force=True)
+
+
+class TestStorage(unittest2.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.connection = regression_utils.get_storage_connection()
+
+
+class TestStorageBuckets(TestStorage):
+
+    def setUp(self):
+        self.case_buckets_to_delete = []
+
+    def tearDown(self):
+        for bucket in self.case_buckets_to_delete:
+            bucket.delete()
+
+    def test_create_bucket(self):
+        new_bucket_name = 'a-new-bucket'
+        self.assertRaises(storage.exceptions.NotFoundError,
+                          self.connection.get_bucket, new_bucket_name)
+        created = self.connection.create_bucket(new_bucket_name)
+        self.case_buckets_to_delete.append(created)
+        self.assertEqual(created.name, new_bucket_name)
+
+    def test_get_buckets(self):
+        buckets_to_create = [
+            'new%d' % (1000 * time.time(),),
+            'newer%d' % (1000 * time.time(),),
+            'newest%d' % (1000 * time.time(),),
+        ]
+        created_buckets = []
+        for bucket_name in buckets_to_create:
+            bucket = self.connection.create_bucket(bucket_name)
+            self.case_buckets_to_delete.append(bucket)
+
+        # Retrieve the buckets.
+        all_buckets = self.connection.get_all_buckets()
+        created_buckets = [bucket for bucket in all_buckets
+                           if bucket.name in buckets_to_create]
+        self.assertEqual(len(created_buckets), len(buckets_to_create))
+
+
+class TestStorageFiles(TestStorage):
+
+    FILES = {
+        'logo': {
+            'path': 'regression/data/CloudPlatform_128px_Retina.png',
+        },
+        'big': {
+            'path': 'regression/data/five-mb-file.zip',
+        },
+    }
+
+    @staticmethod
+    def _get_base64_md5hash(filename):
+        with open(filename, 'rb') as file_obj:
+            hash = MD5.new(data=file_obj.read())
+        digest_bytes = hash.digest()
+        return base64.b64encode(digest_bytes)
+
+    @classmethod
+    def setUpClass(cls):
+        super(TestStorageFiles, cls).setUpClass()
+        for file_data in cls.FILES.values():
+            file_data['hash'] = cls._get_base64_md5hash(file_data['path'])
+        cls.bucket = SHARED_BUCKETS['test_bucket']
+
+    def setUp(self):
+        self.case_keys_to_delete = []
+
+    def tearDown(self):
+        for key in self.case_keys_to_delete:
+            key.delete()
+
+
+class TestStorageWriteFiles(TestStorageFiles):
+
+    def test_large_file_write_from_stream(self):
+        key = self.bucket.new_key('LargeFile')
+        self.assertEqual(key.metadata, {})
+
+        file_data = self.FILES['big']
+        with open(file_data['path'], 'rb') as file_obj:
+            self.bucket.upload_file_object(file_obj, key=key)
+            self.case_keys_to_delete.append(key)
+
+        key.reload_metadata()
+        self.assertEqual(key.metadata['md5Hash'], file_data['hash'])
+
+    def test_write_metadata(self):
+        my_metadata = {'contentType': 'image/png'}
+        key = self.bucket.upload_file(self.FILES['logo']['path'])
+        self.case_keys_to_delete.append(key)
+
+        # NOTE: This should not be necessary. We should be able to pass
+        #       it in to upload_file and also to upload_from_string.
+        key.patch_metadata(my_metadata)
+        self.assertEqual(key.metadata['contentType'],
+                         my_metadata['contentType'])
+
+    def test_direct_write_and_read_into_file(self):
+        key = self.bucket.new_key('MyBuffer')
+        file_contents = 'Hello World'
+        key.upload_from_string(file_contents)
+        self.case_keys_to_delete.append(key)
+
+        same_key = self.bucket.new_key('MyBuffer')
+        temp_filename = tempfile.mktemp()
+        with open(temp_filename, 'w') as file_obj:
+            same_key.get_contents_to_file(file_obj)
+
+        with open(temp_filename, 'rb') as file_obj:
+            stored_contents = file_obj.read()
+
+        self.assertEqual(file_contents, stored_contents)
+
+    def test_copy_existing_file(self):
+        key = self.bucket.upload_file(self.FILES['logo']['path'],
+                                      key='CloudLogo')
+        self.case_keys_to_delete.append(key)
+
+        new_key = self.bucket.copy_key(key, self.bucket, 'CloudLogoCopy')
+        self.case_keys_to_delete.append(new_key)
+
+        base_contents = key.get_contents_as_string()
+        copied_contents = new_key.get_contents_as_string()
+        self.assertEqual(base_contents, copied_contents)
+
+
+class TestStorageListFiles(TestStorageFiles):
+
+    FILENAMES = ['CloudLogo1', 'CloudLogo2', 'CloudLogo3']
+
+    @classmethod
+    def setUpClass(cls):
+        super(TestStorageListFiles, cls).setUpClass()
+        # Make sure bucket empty before beginning.
+        for key in cls.bucket:
+            key.delete()
+
+        logo_path = cls.FILES['logo']['path']
+        key = cls.bucket.upload_file(logo_path, key=cls.FILENAMES[0])
+        cls.suite_keys_to_delete = [key]
+
+        # Copy main key onto remaining in FILENAMES.
+        for filename in cls.FILENAMES[1:]:
+            new_key = cls.bucket.copy_key(key, cls.bucket, filename)
+            cls.suite_keys_to_delete.append(new_key)
+
+    @classmethod
+    def tearDownClass(cls):
+        for key in cls.suite_keys_to_delete:
+            key.delete()
+
+    def test_list_files(self):
+        all_keys = self.bucket.get_all_keys()
+        self.assertEqual(len(all_keys), len(self.FILENAMES))
+
+    def test_paginate_files(self):
+        truncation_size = 1
+        extra_params = {'maxResults': len(self.FILENAMES) - truncation_size}
+        iterator = storage.key._KeyIterator(bucket=self.bucket,
+                                            extra_params=extra_params)
+        response = iterator.get_next_page_response()
+        keys = list(iterator.get_items_from_response(response))
+        self.assertEqual(len(keys), extra_params['maxResults'])
+        self.assertEqual(iterator.page_number, 1)
+        self.assertTrue(iterator.next_page_token is not None)
+
+        response = iterator.get_next_page_response()
+        last_keys = list(iterator.get_items_from_response(response))
+        self.assertEqual(len(last_keys), truncation_size)
+
+
+class TestStorageSignURLs(TestStorageFiles):
+
+    def setUp(self):
+        super(TestStorageSignURLs, self).setUp()
+
+        logo_path = self.FILES['logo']['path']
+        with open(logo_path, 'r') as file_obj:
+            self.LOCAL_FILE = file_obj.read()
+
+        key = self.bucket.new_key('LogoToSign.jpg')
+        key.upload_from_string(self.LOCAL_FILE)
+        self.case_keys_to_delete.append(key)
+
+    def tearDown(self):
+        for key in self.case_keys_to_delete:
+            if key.exists():
+                key.delete()
+
+    def test_create_signed_read_url(self):
+        key = self.bucket.new_key('LogoToSign.jpg')
+        expiration = int(time.time() + 5)
+        signed_url = key.generate_signed_url(expiration, method='GET')
+
+        response, content = HTTP.request(signed_url, method='GET')
+        self.assertEqual(response.status, 200)
+        self.assertEqual(content, self.LOCAL_FILE)
+
+    def test_create_signed_delete_url(self):
+        key = self.bucket.new_key('LogoToSign.jpg')
+        expiration = int(time.time() + 283473274)
+        signed_delete_url = key.generate_signed_url(expiration,
+                                                    method='DELETE')
+
+        response, content = HTTP.request(signed_delete_url, method='DELETE')
+        self.assertEqual(response.status, 204)
+        self.assertEqual(content, '')
+
+        # Check that the key has actually been deleted.
+        self.assertRaises(storage.exceptions.NotFoundError,
+                          key.reload_metadata)

scripts/run_regression.sh

@@ -36,3 +36,4 @@ fi
 
 # Run the regression tests for each tested package.
 python regression/run_regression.py --package datastore
+python regression/run_regression.py --package storage