feat: add ingestion Cloud Storage fields and Platform Logging fields …

…to Topic (#1974) * feat: add ingestion Cloud Storage fields and Platform Logging fields to Topic PiperOrigin-RevId: 678307181 Source-Link: googleapis/googleapis@69e9dff Source-Link: googleapis/googleapis-gen@11a52e5 Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiMTFhNTJlNTI3MjMyMjdkM2I2MTIwZjg4ZjFhNTgwZTM5NThlMGNkNSJ9 feat: return listing information for subscriptions created via Analytics Hub PiperOrigin-RevId: 676126585 Source-Link: googleapis/googleapis@de3b4b3 Source-Link: googleapis/googleapis-gen@c3df551 Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiYzNkZjU1MTYyY2NjOTkyOWM5N2JmZWM5ZmExZDViY2QzYjY1YWY0YSJ9 * build: try switching to raw eslint to exclude owl-bot-staging * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com> Co-authored-by: feywind <[email protected]>
googleapis · Sep 30, 2024 · afec9a1 · afec9a1
1 parent 37b9f71
commit afec9a1
Show file tree

Hide file tree

Showing 7 changed files with 2,631 additions and 61 deletions.
diff --git a/README.md b/README.md
@@ -66,15 +66,15 @@ npm install @google-cloud/pubsub
 
 ```javascript
 // Imports the Google Cloud client library
-const {PubSub} = require('@google-cloud/pubsub');
+const { PubSub } = require("@google-cloud/pubsub");
 
 async function quickstart(
-  projectId = 'your-project-id', // Your Google Cloud Platform project ID
-  topicNameOrId = 'my-topic', // Name for the new topic to create
-  subscriptionName = 'my-sub' // Name for the new subscription to create
+projectId = 'your-project-id', // Your Google Cloud Platform project ID
+topicNameOrId = 'my-topic', // Name for the new topic to create
+subscriptionName = 'my-sub' // Name for the new subscription to create
 ) {
   // Instantiates a client
-  const pubsub = new PubSub({projectId});
+  const pubsub = new PubSub({ projectId });
 
   // Creates a new topic
   const [topic] = await pubsub.createTopic(topicNameOrId);
@@ -84,19 +84,19 @@ async function quickstart(
   const [subscription] = await topic.createSubscription(subscriptionName);
 
   // Receive callbacks for new messages on the subscription
-  subscription.on('message', message => {
+  subscription.on('message', (message) => {
     console.log('Received message:', message.data.toString());
     process.exit(0);
   });
 
   // Receive callbacks for errors on the subscription
-  subscription.on('error', error => {
+  subscription.on('error', (error) => {
     console.error('Received error:', error);
     process.exit(1);
   });
 
   // Send a message to the topic
-  topic.publishMessage({data: Buffer.from('Test message!')});
+  topic.publishMessage({ data: Buffer.from('Test message!') });
 }
 
 ```

diff --git a/package.json b/package.json
@@ -46,7 +46,7 @@
     "prelint": "cd samples; npm link ../; npm install",
     "precompile": "gts clean",
     "typeless": "npx typeless-sample-bot --outputpath samples --targets samples --recursive",
-    "posttypeless": "cd samples && npm i && cd .. && npm run fix"
+    "posttypeless": "cd samples && npm i && cd .. && npx eslint --ignore-pattern owl-bot-staging --fix"
   },
   "dependencies": {
     "@google-cloud/paginator": "^5.0.0",

diff --git a/protos/google/pubsub/v1/pubsub.proto b/protos/google/pubsub/v1/pubsub.proto
@@ -237,11 +237,132 @@ message IngestionDataSourceSettings {
     string gcp_service_account = 5 [(google.api.field_behavior) = REQUIRED];
   }
 
+  // Ingestion settings for Cloud Storage.
+  message CloudStorage {
+    // Possible states for ingestion from Cloud Storage.
+    enum State {
+      // Default value. This value is unused.
+      STATE_UNSPECIFIED = 0;
+
+      // Ingestion is active.
+      ACTIVE = 1;
+
+      // Permission denied encountered while calling the Cloud Storage API. This
+      // can happen if the Pub/Sub SA has not been granted the
+      // [appropriate
+      // permissions](https://cloud.google.com/storage/docs/access-control/iam-permissions):
+      // - storage.objects.list: to list the objects in a bucket.
+      // - storage.objects.get: to read the objects in a bucket.
+      // - storage.buckets.get: to verify the bucket exists.
+      CLOUD_STORAGE_PERMISSION_DENIED = 2;
+
+      // Permission denied encountered while publishing to the topic. This can
+      // happen if the Pub/Sub SA has not been granted the [appropriate publish
+      // permissions](https://cloud.google.com/pubsub/docs/access-control#pubsub.publisher)
+      PUBLISH_PERMISSION_DENIED = 3;
+
+      // The provided Cloud Storage bucket doesn't exist.
+      BUCKET_NOT_FOUND = 4;
+
+      // The Cloud Storage bucket has too many objects, ingestion will be
+      // paused.
+      TOO_MANY_OBJECTS = 5;
+    }
+
+    // Configuration for reading Cloud Storage data in text format. Each line of
+    // text as specified by the delimiter will be set to the `data` field of a
+    // Pub/Sub message.
+    message TextFormat {
+      // Optional. When unset, '\n' is used.
+      optional string delimiter = 1 [(google.api.field_behavior) = OPTIONAL];
+    }
+
+    // Configuration for reading Cloud Storage data in Avro binary format. The
+    // bytes of each object will be set to the `data` field of a Pub/Sub
+    // message.
+    message AvroFormat {}
+
+    // Configuration for reading Cloud Storage data written via [Cloud Storage
+    // subscriptions](https://cloud.google.com/pubsub/docs/cloudstorage). The
+    // data and attributes fields of the originally exported Pub/Sub message
+    // will be restored when publishing.
+    message PubSubAvroFormat {}
+
+    // Output only. An output-only field that indicates the state of the Cloud
+    // Storage ingestion source.
+    State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+    // Optional. Cloud Storage bucket. The bucket name must be without any
+    // prefix like "gs://". See the [bucket naming requirements]
+    // (https://cloud.google.com/storage/docs/buckets#naming).
+    string bucket = 2 [(google.api.field_behavior) = OPTIONAL];
+
+    // Defaults to text format.
+    oneof input_format {
+      // Optional. Data from Cloud Storage will be interpreted as text.
+      TextFormat text_format = 3 [(google.api.field_behavior) = OPTIONAL];
+
+      // Optional. Data from Cloud Storage will be interpreted in Avro format.
+      AvroFormat avro_format = 4 [(google.api.field_behavior) = OPTIONAL];
+
+      // Optional. It will be assumed data from Cloud Storage was written via
+      // [Cloud Storage
+      // subscriptions](https://cloud.google.com/pubsub/docs/cloudstorage).
+      PubSubAvroFormat pubsub_avro_format = 5
+          [(google.api.field_behavior) = OPTIONAL];
+    }
+
+    // Optional. Only objects with a larger or equal creation timestamp will be
+    // ingested.
+    google.protobuf.Timestamp minimum_object_create_time = 6
+        [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. Glob pattern used to match objects that will be ingested. If
+    // unset, all objects will be ingested. See the [supported
+    // patterns](https://cloud.google.com/storage/docs/json_api/v1/objects/list#list-objects-and-prefixes-using-glob).
+    string match_glob = 9 [(google.api.field_behavior) = OPTIONAL];
+  }
+
   // Only one source type can have settings set.
   oneof source {
     // Optional. Amazon Kinesis Data Streams.
     AwsKinesis aws_kinesis = 1 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. Cloud Storage.
+    CloudStorage cloud_storage = 2 [(google.api.field_behavior) = OPTIONAL];
+  }
+
+  // Optional. Platform Logs settings. If unset, no Platform Logs will be
+  // generated.
+  PlatformLogsSettings platform_logs_settings = 4
+      [(google.api.field_behavior) = OPTIONAL];
+}
+
+// Settings for Platform Logs produced by Pub/Sub.
+message PlatformLogsSettings {
+  // Severity levels of Platform Logs.
+  enum Severity {
+    // Default value. Logs level is unspecified. Logs will be disabled.
+    SEVERITY_UNSPECIFIED = 0;
+
+    // Logs will be disabled.
+    DISABLED = 1;
+
+    // Debug logs and higher-severity logs will be written.
+    DEBUG = 2;
+
+    // Info logs and higher-severity logs will be written.
+    INFO = 3;
+
+    // Warning logs and higher-severity logs will be written.
+    WARNING = 4;
+
+    // Only error logs will be written.
+    ERROR = 5;
   }
+
+  // Optional. The minimum severity level of Platform Logs that will be written.
+  Severity severity = 1 [(google.api.field_behavior) = OPTIONAL];
 }
 
 // A topic resource.
@@ -774,6 +895,20 @@ message Subscription {
     RESOURCE_ERROR = 2;
   }
 
+  // Information about an associated Analytics Hub subscription
+  // (https://cloud.google.com/bigquery/docs/analytics-hub-manage-subscriptions).
+  message AnalyticsHubSubscriptionInfo {
+    // Optional. The name of the associated Analytics Hub listing resource.
+    // Pattern:
+    // "projects/{project}/locations/{location}/dataExchanges/{data_exchange}/listings/{listing}"
+    string listing = 1 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The name of the associated Analytics Hub subscription resource.
+    // Pattern:
+    // "projects/{project}/locations/{location}/subscriptions/{subscription}"
+    string subscription = 2 [(google.api.field_behavior) = OPTIONAL];
+  }
+
   // Required. The name of the subscription. It must have the format
   // `"projects/{project}/subscriptions/{subscription}"`. `{subscription}` must
   // start with a letter, and contain only letters (`[A-Za-z]`), numbers
@@ -922,6 +1057,11 @@ message Subscription {
   // Output only. An output-only field indicating whether or not the
   // subscription can receive messages.
   State state = 19 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. Information about the associated Analytics Hub subscription.
+  // Only set if the subscritpion is created by Analytics Hub.
+  AnalyticsHubSubscriptionInfo analytics_hub_subscription_info = 23
+      [(google.api.field_behavior) = OUTPUT_ONLY];
 }
 
 // A policy that specifies how Pub/Sub retries message delivery.