[Filebeat][Google workspace] Backport integration fixes

CHANGELOG.next.asciidoc

@@ -58,6 +58,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 - Fix handling of empty array in httpjson input. {pull}32001[32001]
 - Fix reporting of `filebeat.events.active` in log events such that the current value is always reported instead of the difference from the last value. {pull}33597[33597]
 - Fix splitting array of strings/arrays in httpjson input {issue}30345[30345] {pull}33609[33609]
+- Fix Google workspace pagination and document ID generation. {pull}33666[33666]
 
 *Heartbeat*
 - Fix bug affecting let's encrypt and other users of cross-signed certs, where cert expiration was incorrectly calculated. {issue}33215[33215]

x-pack/filebeat/module/google_workspace/admin/config/config.yml

@@ -16,21 +16,42 @@ request.proxy_url: {{ .proxy_url }}
 request.transforms:
   - set:
       target: url.params.startTime
-      value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]'
+      value: '[[.cursor.last_execution_datetime]]'
       default: '[[formatDate (now (parseDuration "-{{.initial_interval}}"))]]'
 response.split:
   target: body.items
   split:
     target: body.events
     keep_parent: true
 response.pagination:
+  - set:
+      target: url.params.startTime
+      value: '[[.last_response.url.params.Get "startTime"]]'
+      fail_on_template_error: true
   - set:
       target: url.params.pageToken
-      value: "[[.last_response.body.nextPageToken]]"
+      value: >-
+        [[- if index .last_response.body "nextPageToken" -]]
+          [[- .last_response.body.nextPageToken -]]
+        [[- end -]]
       fail_on_template_error: true
 cursor:
   last_execution_datetime:
-    value: "[[formatDate now]]"
+    value: >-
+      [[- $time := .last_event.id.time -]]
+      [[- if not (parseDate $time "RFC3339").IsZero -]]
+        [[- $time -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05Z").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05Z") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05.999Z").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05.999Z") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05 MST").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05 MST") -]]
+      [[- else -]]
+        [[- formatDate now -]]
+      [[- end -]]
 
 {{ else if eq .input "file" }}
 type: log

x-pack/filebeat/module/google_workspace/config/common.js

@@ -18,17 +18,27 @@ var googleWorkspace = (function () {
         ignore_missing: true,
     });
 
-    var addID = new processor.Fingerprint({
-        fields: [
+    var addID = function(evt) {
+        var keys = [
             "json.id.time",
             "json.id.uniqueQualifier",
             "json.id.applicationName",
             "json.id.customerId",
-        ],
-        target_field: "@metadata._id",
-        ignore_missing: true,
-        fail_on_error: false,
-    });
+        ];
+        Object.keys(evt.Get("json.events")).forEach(function(evtsKey) {
+            var key = "json.events."+evtsKey;
+            var value = evt.Get(key);
+            if (!Array.isArray(value) && !(typeof value === "object")) {
+                keys.push(key);
+            }
+        });
+        new processor.Fingerprint({
+            fields: keys,
+            target_field: "@metadata._id",
+            ignore_missing: true,
+            fail_on_error: false,
+        }).Run(evt);
+    };
 
     var convertFields = new processor.Convert({
         fields: [

x-pack/filebeat/module/google_workspace/drive/config/config.yml

@@ -16,21 +16,42 @@ request.proxy_url: {{ .proxy_url }}
 request.transforms:
   - set:
       target: url.params.startTime
-      value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]'
+      value: '[[.cursor.last_execution_datetime]]'
       default: '[[formatDate (now (parseDuration "-{{.initial_interval}}"))]]'
 response.split:
   target: body.items
   split:
     target: body.events
     keep_parent: true
 response.pagination:
+  - set:
+      target: url.params.startTime
+      value: '[[.last_response.url.params.Get "startTime"]]'
+      fail_on_template_error: true
   - set:
       target: url.params.pageToken
-      value: "[[.last_response.body.nextPageToken]]"
+      value: >-
+        [[- if index .last_response.body "nextPageToken" -]]
+          [[- .last_response.body.nextPageToken -]]
+        [[- end -]]
       fail_on_template_error: true
 cursor:
   last_execution_datetime:
-    value: "[[formatDate now]]"
+    value: >-
+      [[- $time := .last_event.id.time -]]
+      [[- if not (parseDate $time "RFC3339").IsZero -]]
+        [[- $time -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05Z").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05Z") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05.999Z").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05.999Z") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05 MST").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05 MST") -]]
+      [[- else -]]
+        [[- formatDate now -]]
+      [[- end -]]
 
 {{ else if eq .input "file" }}
 type: log

x-pack/filebeat/module/google_workspace/groups/config/config.yml

@@ -16,21 +16,42 @@ request.proxy_url: {{ .proxy_url }}
 request.transforms:
   - set:
       target: url.params.startTime
-      value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]'
+      value: '[[.cursor.last_execution_datetime]]'
       default: '[[formatDate (now (parseDuration "-{{.initial_interval}}"))]]'
 response.split:
   target: body.items
   split:
     target: body.events
     keep_parent: true
 response.pagination:
+  - set:
+      target: url.params.startTime
+      value: '[[.last_response.url.params.Get "startTime"]]'
+      fail_on_template_error: true
   - set:
       target: url.params.pageToken
-      value: "[[.last_response.body.nextPageToken]]"
+      value: >-
+        [[- if index .last_response.body "nextPageToken" -]]
+          [[- .last_response.body.nextPageToken -]]
+        [[- end -]]
       fail_on_template_error: true
 cursor:
   last_execution_datetime:
-    value: "[[formatDate now]]"
+    value: >-
+      [[- $time := .last_event.id.time -]]
+      [[- if not (parseDate $time "RFC3339").IsZero -]]
+        [[- $time -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05Z").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05Z") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05.999Z").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05.999Z") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05 MST").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05 MST") -]]
+      [[- else -]]
+        [[- formatDate now -]]
+      [[- end -]]
 
 {{ else if eq .input "file" }}
 type: log

x-pack/filebeat/module/google_workspace/login/config/config.yml

@@ -16,21 +16,42 @@ request.proxy_url: {{ .proxy_url }}
 request.transforms:
   - set:
       target: url.params.startTime
-      value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]'
+      value: '[[.cursor.last_execution_datetime]]'
       default: '[[formatDate (now (parseDuration "-{{.initial_interval}}"))]]'
 response.split:
   target: body.items
   split:
     target: body.events
     keep_parent: true
 response.pagination:
+  - set:
+      target: url.params.startTime
+      value: '[[.last_response.url.params.Get "startTime"]]'
+      fail_on_template_error: true
   - set:
       target: url.params.pageToken
-      value: "[[.last_response.body.nextPageToken]]"
+      value: >-
+        [[- if index .last_response.body "nextPageToken" -]]
+          [[- .last_response.body.nextPageToken -]]
+        [[- end -]]
       fail_on_template_error: true
 cursor:
   last_execution_datetime:
-    value: "[[formatDate now]]"
+    value: >-
+      [[- $time := .last_event.id.time -]]
+      [[- if not (parseDate $time "RFC3339").IsZero -]]
+        [[- $time -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05Z").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05Z") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05.999Z").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05.999Z") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05 MST").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05 MST") -]]
+      [[- else -]]
+        [[- formatDate now -]]
+      [[- end -]]
 
 {{ else if eq .input "file" }}
 type: log

x-pack/filebeat/module/google_workspace/saml/config/config.yml

@@ -16,21 +16,42 @@ request.proxy_url: {{ .proxy_url }}
 request.transforms:
   - set:
       target: url.params.startTime
-      value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]'
+      value: '[[.cursor.last_execution_datetime]]'
       default: '[[formatDate (now (parseDuration "-{{.initial_interval}}"))]]'
 response.split:
   target: body.items
   split:
     target: body.events
     keep_parent: true
 response.pagination:
+  - set:
+      target: url.params.startTime
+      value: '[[.last_response.url.params.Get "startTime"]]'
+      fail_on_template_error: true
   - set:
       target: url.params.pageToken
-      value: "[[.last_response.body.nextPageToken]]"
+      value: >-
+        [[- if index .last_response.body "nextPageToken" -]]
+          [[- .last_response.body.nextPageToken -]]
+        [[- end -]]
       fail_on_template_error: true
 cursor:
   last_execution_datetime:
-    value: "[[formatDate now]]"
+    value: >-
+      [[- $time := .last_event.id.time -]]
+      [[- if not (parseDate $time "RFC3339").IsZero -]]
+        [[- $time -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05Z").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05Z") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05.999Z").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05.999Z") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05 MST").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05 MST") -]]
+      [[- else -]]
+        [[- formatDate now -]]
+      [[- end -]]
 
 {{ else if eq .input "file" }}
 type: log

x-pack/filebeat/module/google_workspace/user_accounts/config/config.yml

@@ -16,21 +16,42 @@ request.proxy_url: {{ .proxy_url }}
 request.transforms:
   - set:
       target: url.params.startTime
-      value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]'
+      value: '[[.cursor.last_execution_datetime]]'
       default: '[[formatDate (now (parseDuration "-{{.initial_interval}}"))]]'
 response.split:
   target: body.items
   split:
     target: body.events
     keep_parent: true
 response.pagination:
+  - set:
+      target: url.params.startTime
+      value: '[[.last_response.url.params.Get "startTime"]]'
+      fail_on_template_error: true
   - set:
       target: url.params.pageToken
-      value: "[[.last_response.body.nextPageToken]]"
+      value: >-
+        [[- if index .last_response.body "nextPageToken" -]]
+          [[- .last_response.body.nextPageToken -]]
+        [[- end -]]
       fail_on_template_error: true
 cursor:
   last_execution_datetime:
-    value: "[[formatDate now]]"
+    value: >-
+      [[- $time := .last_event.id.time -]]
+      [[- if not (parseDate $time "RFC3339").IsZero -]]
+        [[- $time -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05Z").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05Z") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05.999Z").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05.999Z") -]]
+      [[- else if not (parseDate $time "2006-01-02T15:04:05 MST").IsZero -]]
+        [[- formatDate (parseDate $time "2006-01-02T15:04:05 MST") -]]
+      [[- else -]]
+        [[- formatDate now -]]
+      [[- end -]]
 
 {{ else if eq .input "file" }}
 type: log

x-pack/filebeat/module/google_workspace/user_accounts/config/pipeline.js

@@ -10,8 +10,34 @@ var userAccounts = (function () {
         evt.Put("event.category", ["iam"]);
     };
 
+    var getParamValue = function(param) {
+        if (param.value) {
+            return param.value;
+        }
+        if (param.multiValue) {
+            return param.multiValue;
+        }
+        if (param.intValue !== null) {
+            return param.intValue;
+        }
+    };
+
+    var flattenParams = function(evt) {
+        var params = evt.Get("json.events.parameters");
+        if (!params || !Array.isArray(params)) {
+            return;
+        }
+
+        params.forEach(function(p){
+            evt.Put("google_workspace.user_accounts."+p.name, getParamValue(p));
+        });
+
+        evt.Delete("json.events.parameters");
+    };
+
     var pipeline = new processor.Chain()
         .Add(categorizeEvent)
+        .Add(flattenParams)
         .Build();
 
     return {