Add system tests for bucket notifications. (#4021)

storage/google/cloud/storage/bucket.py

@@ -36,6 +36,7 @@
 from google.cloud.storage.blob import Blob
 from google.cloud.storage.blob import _get_encryption_headers
 from google.cloud.storage.notification import BucketNotification
+from google.cloud.storage.notification import NONE_PAYLOAD_FORMAT
 
 
 def _blobs_page_start(iterator, page, response):
@@ -185,7 +186,7 @@ def notification(self, topic_name,
                      custom_attributes=None,
                      event_types=None,
                      blob_name_prefix=None,
-                     payload_format=None):
+                     payload_format=NONE_PAYLOAD_FORMAT):
         """Factory:  create a notification resource for the bucket.
 
         See: :class:`.BucketNotification` for parameters.

storage/google/cloud/storage/notification.py

@@ -70,7 +70,7 @@ class BucketNotification(object):
     """
     def __init__(self, bucket, topic_name,
                  topic_project=None, custom_attributes=None, event_types=None,
-                 blob_name_prefix=None, payload_format=None):
+                 blob_name_prefix=None, payload_format=NONE_PAYLOAD_FORMAT):
         self._bucket = bucket
         self._topic_name = topic_name
 
@@ -87,10 +87,9 @@ def __init__(self, bucket, topic_name,
             self._properties['event_types'] = event_types
 
         if blob_name_prefix is not None:
-            self._properties['blob_name_prefix'] = blob_name_prefix
+            self._properties['object_name_prefix'] = blob_name_prefix
 
-        if payload_format is not None:
-            self._properties['payload_format'] = payload_format
+        self._properties['payload_format'] = payload_format
 
     @classmethod
     def from_api_repr(cls, resource, bucket):
@@ -162,7 +161,7 @@ def event_types(self):
     def blob_name_prefix(self):
         """Prefix of blob names for which notification events are published.
         """
-        return self._properties.get('blob_name_prefix')
+        return self._properties.get('object_name_prefix')
 
     @property
     def payload_format(self):

storage/nox.py

@@ -71,10 +71,11 @@ def system_tests(session, python_version):
     # virutalenv's dist-packages.
     session.install('mock', 'pytest', *LOCAL_DEPS)
     session.install('../test_utils/')
-    session.install('.')
+    session.install('../pubsub')
+    session.install('-e', '.')
 
     # Run py.test against the system tests.
-    session.run('py.test', '--quiet', 'tests/system.py')
+    session.run('py.test', '--quiet', 'tests/system.py', *session.posargs)
 
 
 @nox.session

storage/tests/system.py

@@ -550,3 +550,103 @@ def test_rewrite_rotate_encryption_key(self):
         self.assertEqual(total, len(source_data))
 
         self.assertEqual(dest.download_as_string(), source_data)
+
+
+class TestStorageNotificationCRUD(unittest.TestCase):
+
+    topic = None
+    TOPIC_NAME = 'notification' + unique_resource_id('-')
+    CUSTOM_ATTRIBUTES = {
+        'attr1': 'value1',
+        'attr2': 'value2',
+    }
+    BLOB_NAME_PREFIX = 'blob-name-prefix/'
+
+    @property
+    def topic_path(self):
+        return 'projects/{}/topics/{}'.format(
+            Config.CLIENT.project, self.TOPIC_NAME)
+
+    def _intialize_topic(self):
+        try:
+            from google.cloud.pubsub_v1 import PublisherClient
+        except ImportError:
+            raise unittest.SkipTest("Cannot import pubsub")
+        self.publisher_client = PublisherClient()
+        retry_429(self.publisher_client.create_topic)(self.topic_path)
+        policy = self.publisher_client.get_iam_policy(self.topic_path)
+        binding = policy.bindings.add()
+        binding.role = 'roles/pubsub.publisher'
+        binding.members.append(
+            'serviceAccount:{}'
+            '@gs-project-accounts.iam.gserviceaccount.com'.format(
+                Config.CLIENT.project))
+        self.publisher_client.set_iam_policy(self.topic_path, policy)
+
+
+    def setUp(self):
+        self.case_buckets_to_delete = []
+        self._intialize_topic()
+
+    def tearDown(self):
+        retry_429(self.publisher_client.delete_topic)(self.topic_path)
+        with Config.CLIENT.batch():
+            for bucket_name in self.case_buckets_to_delete:
+                bucket = Config.CLIENT.bucket(bucket_name)
+                retry_429(bucket.delete)()
+
+    @staticmethod
+    def event_types():
+        from google.cloud.storage.notification import (
+            OBJECT_FINALIZE_EVENT_TYPE,
+            OBJECT_DELETE_EVENT_TYPE)
+
+        return [OBJECT_FINALIZE_EVENT_TYPE, OBJECT_DELETE_EVENT_TYPE]
+
+    @staticmethod
+    def payload_format():
+        from google.cloud.storage.notification import (
+            JSON_API_V1_PAYLOAD_FORMAT)
+
+        return JSON_API_V1_PAYLOAD_FORMAT
+
+    def test_notification_minimal(self):
+        new_bucket_name = 'notification-minimal' + unique_resource_id('-')
+        bucket = retry_429(Config.CLIENT.create_bucket)(new_bucket_name)
+        self.case_buckets_to_delete.append(new_bucket_name)
+        self.assertEqual(list(bucket.list_notifications()), [])
+        notification = bucket.notification(self.TOPIC_NAME)
+        retry_429(notification.create)()
+        try:
+            self.assertTrue(notification.exists())
+            self.assertIsNotNone(notification.notification_id)
+            notifications = list(bucket.list_notifications())
+            self.assertEqual(len(notifications), 1)
+            self.assertEqual(notifications[0].topic_name, self.TOPIC_NAME)
+        finally:
+            notification.delete()
+
+    def test_notification_explicit(self):
+        new_bucket_name = 'notification-explicit' + unique_resource_id('-')
+        bucket = retry_429(Config.CLIENT.create_bucket)(new_bucket_name)
+        self.case_buckets_to_delete.append(new_bucket_name)
+        notification = bucket.notification(
+            self.TOPIC_NAME,
+            custom_attributes=self.CUSTOM_ATTRIBUTES,
+            event_types=self.event_types(),
+            blob_name_prefix=self.BLOB_NAME_PREFIX,
+            payload_format=self.payload_format(),
+        )
+        retry_429(notification.create)()
+        try:
+            self.assertTrue(notification.exists())
+            self.assertIsNotNone(notification.notification_id)
+            self.assertEqual(
+                notification.custom_attributes, self.CUSTOM_ATTRIBUTES)
+            self.assertEqual(notification.event_types, self.event_types())
+            self.assertEqual(
+                notification.blob_name_prefix, self.BLOB_NAME_PREFIX)
+            self.assertEqual(
+                notification.payload_format, self.payload_format())
+        finally:
+            notification.delete()

storage/tests/unit/test_bucket.py

@@ -74,6 +74,7 @@ def test_blob(self):
 
     def test_notification_defaults(self):
         from google.cloud.storage.notification import BucketNotification
+        from google.cloud.storage.notification import NONE_PAYLOAD_FORMAT
 
         PROJECT = 'PROJECT'
         BUCKET_NAME = 'BUCKET_NAME'
@@ -89,7 +90,7 @@ def test_notification_defaults(self):
         self.assertIsNone(notification.custom_attributes)
         self.assertIsNone(notification.event_types)
         self.assertIsNone(notification.blob_name_prefix)
-        self.assertIsNone(notification.payload_format)
+        self.assertEqual(notification.payload_format, NONE_PAYLOAD_FORMAT)
 
     def test_notification_explicit(self):
         from google.cloud.storage.notification import (
@@ -392,6 +393,8 @@ def test_list_blobs(self):
     def test_list_notifications(self):
         from google.cloud.storage.notification import BucketNotification
         from google.cloud.storage.notification import _TOPIC_REF_FMT
+        from google.cloud.storage.notification import (
+            JSON_API_V1_PAYLOAD_FORMAT, NONE_PAYLOAD_FORMAT)
 
         NAME = 'name'
 
@@ -405,11 +408,13 @@ def test_list_notifications(self):
             'id': '1',
             'etag': 'DEADBEEF',
             'selfLink': 'https://example.com/notification/1',
+            'payload_format': NONE_PAYLOAD_FORMAT,
         }, {
             'topic': _TOPIC_REF_FMT.format(*topic_refs[1]),
             'id': '2',
             'etag': 'FACECABB',
             'selfLink': 'https://example.com/notification/2',
+            'payload_format': JSON_API_V1_PAYLOAD_FORMAT,
         }]
         connection = _Connection({'items': resources})
         client = _Client(connection)

storage/tests/unit/test_notification.py

@@ -74,6 +74,8 @@ def _make_bucket(self, client, name=BUCKET_NAME):
          return bucket
 
     def test_ctor_defaults(self):
+        from google.cloud.storage.notification import NONE_PAYLOAD_FORMAT
+
         client = self._make_client()
         bucket = self._make_bucket(client)
 
@@ -86,7 +88,7 @@ def test_ctor_defaults(self):
         self.assertIsNone(notification.custom_attributes)
         self.assertIsNone(notification.event_types)
         self.assertIsNone(notification.blob_name_prefix)
-        self.assertIsNone(notification.payload_format)
+        self.assertEqual(notification.payload_format, NONE_PAYLOAD_FORMAT)
 
     def test_ctor_explicit(self):
         client = self._make_client()
@@ -132,6 +134,8 @@ def test_from_api_repr_invalid_topic(self):
             klass.from_api_repr(resource, bucket=bucket)
 
     def test_from_api_repr_minimal(self):
+        from google.cloud.storage.notification import NONE_PAYLOAD_FORMAT
+
         klass = self._get_target_class()
         client = self._make_client()
         bucket = self._make_bucket(client)
@@ -140,6 +144,7 @@ def test_from_api_repr_minimal(self):
             'id': self.NOTIFICATION_ID,
             'etag': self.ETAG,
             'selfLink': self.SELF_LINK,
+            'payload_format': NONE_PAYLOAD_FORMAT,
         }
 
         notification = klass.from_api_repr(resource, bucket=bucket)
@@ -150,7 +155,7 @@ def test_from_api_repr_minimal(self):
         self.assertIsNone(notification.custom_attributes)
         self.assertIsNone(notification.event_types)
         self.assertIsNone(notification.blob_name_prefix)
-        self.assertIsNone(notification.payload_format)
+        self.assertEqual(notification.payload_format, NONE_PAYLOAD_FORMAT)
         self.assertEqual(notification.etag, self.ETAG)
         self.assertEqual(notification.self_link, self.SELF_LINK)
 
@@ -162,7 +167,7 @@ def test_from_api_repr_explicit(self):
             'topic': self.TOPIC_ALT_REF,
             'custom_attributes': self.CUSTOM_ATTRIBUTES,
             'event_types': self.event_types(),
-            'blob_name_prefix': self.BLOB_NAME_PREFIX,
+            'object_name_prefix': self.BLOB_NAME_PREFIX,
             'payload_format': self.payload_format(),
             'id': self.NOTIFICATION_ID,
             'etag': self.ETAG,
@@ -231,6 +236,8 @@ def test_create_w_existing_notification_id(self):
             notification.create()
 
     def test_create_w_defaults(self):
+        from google.cloud.storage.notification import NONE_PAYLOAD_FORMAT
+
         client = self._make_client()
         bucket = self._make_bucket(client)
         notification = self._make_one(
@@ -241,6 +248,7 @@ def test_create_w_defaults(self):
             'id': self.NOTIFICATION_ID,
             'etag': self.ETAG,
             'selfLink': self.SELF_LINK,
+            'payload_format': NONE_PAYLOAD_FORMAT,
         }
 
         notification.create()
@@ -251,10 +259,11 @@ def test_create_w_defaults(self):
         self.assertIsNone(notification.custom_attributes)
         self.assertIsNone(notification.event_types)
         self.assertIsNone(notification.blob_name_prefix)
-        self.assertIsNone(notification.payload_format)
+        self.assertEqual(notification.payload_format, NONE_PAYLOAD_FORMAT)
 
         data = {
             'topic': self.TOPIC_REF,
+            'payload_format': NONE_PAYLOAD_FORMAT,
         }
         api_request.assert_called_once_with(
             method='POST',
@@ -279,7 +288,7 @@ def test_create_w_explicit_client(self):
             'topic': self.TOPIC_ALT_REF,
             'custom_attributes': self.CUSTOM_ATTRIBUTES,
             'event_types': self.event_types(),
-            'blob_name_prefix': self.BLOB_NAME_PREFIX,
+            'object_name_prefix': self.BLOB_NAME_PREFIX,
             'payload_format': self.payload_format(),
             'id': self.NOTIFICATION_ID,
             'etag': self.ETAG,
@@ -302,7 +311,7 @@ def test_create_w_explicit_client(self):
             'topic': self.TOPIC_ALT_REF,
             'custom_attributes': self.CUSTOM_ATTRIBUTES,
             'event_types': self.event_types(),
-            'blob_name_prefix': self.BLOB_NAME_PREFIX,
+            'object_name_prefix': self.BLOB_NAME_PREFIX,
             'payload_format': self.payload_format(),
         }
         api_request.assert_called_once_with(
@@ -386,6 +395,8 @@ def test_reload_miss(self):
         )
 
     def test_reload_hit(self):
+        from google.cloud.storage.notification import NONE_PAYLOAD_FORMAT
+
         client = self._make_client()
         bucket = self._make_bucket(client)
         alt_client = self._make_client()
@@ -397,6 +408,7 @@ def test_reload_hit(self):
             'id': self.NOTIFICATION_ID,
             'etag': self.ETAG,
             'selfLink': self.SELF_LINK,
+            'payload_format': NONE_PAYLOAD_FORMAT,
         }
 
         notification.reload(client=client)
@@ -406,7 +418,7 @@ def test_reload_hit(self):
         self.assertIsNone(notification.custom_attributes)
         self.assertIsNone(notification.event_types)
         self.assertIsNone(notification.blob_name_prefix)
-        self.assertIsNone(notification.payload_format)
+        self.assertEqual(notification.payload_format, NONE_PAYLOAD_FORMAT)
 
         api_request.assert_called_once_with(
             method='GET',