From 00a72d9c06e462152b8175336509671b3ef3522d Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Mon, 14 Nov 2022 10:07:59 +0100 Subject: [PATCH 1/5] Backport integration improvements: - Use event for google workspace fingerprint - Fix pagination and cursor for google workspace data streams --- .../google_workspace/admin/config/config.yml | 27 ++++++++++++++++--- .../module/google_workspace/config/common.js | 24 +++++++++-------- .../google_workspace/drive/config/config.yml | 27 ++++++++++++++++--- .../google_workspace/groups/config/config.yml | 27 ++++++++++++++++--- .../google_workspace/login/config/config.yml | 27 ++++++++++++++++--- .../google_workspace/saml/config/config.yml | 27 ++++++++++++++++--- .../user_accounts/config/config.yml | 27 ++++++++++++++++--- 7 files changed, 157 insertions(+), 29 deletions(-) diff --git a/x-pack/filebeat/module/google_workspace/admin/config/config.yml b/x-pack/filebeat/module/google_workspace/admin/config/config.yml index 9abc6ee0004..2e86249c2ce 100644 --- a/x-pack/filebeat/module/google_workspace/admin/config/config.yml +++ b/x-pack/filebeat/module/google_workspace/admin/config/config.yml @@ -16,7 +16,7 @@ request.proxy_url: {{ .proxy_url }} request.transforms: - set: target: url.params.startTime - value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]' + value: '[[.cursor.last_execution_datetime]]' default: '[[formatDate (now (parseDuration "-{{.initial_interval}}"))]]' response.split: target: body.items @@ -24,13 +24,34 @@ response.split: target: body.events keep_parent: true response.pagination: + - set: + target: url.params.startTime + value: '[[.last_response.url.params.Get "startTime"]]' + fail_on_template_error: true - set: target: url.params.pageToken - value: "[[.last_response.body.nextPageToken]]" + value: >- + [[- if index .last_response.body "nextPageToken" -]] + [[- .last_response.body.nextPageToken -]] + [[- end -]] fail_on_template_error: true cursor: last_execution_datetime: - value: "[[formatDate now]]" + value: >- + [[- $time := .last_event.id.time -]] + [[- if not (parseDate $time "RFC3339").IsZero -]] + [[- $time -]] + [[- else if not (parseDate $time "2006-01-02T15:04:05").IsZero -]] + [[- formatDate (parseDate $time "2006-01-02T15:04:05") -]] + [[- else if not (parseDate $time "2006-01-02T15:04:05Z").IsZero -]] + [[- formatDate (parseDate $time "2006-01-02T15:04:05Z") -]] + [[- else if not (parseDate $time "2006-01-02T15:04:05.999Z").IsZero -]] + [[- formatDate (parseDate $time "2006-01-02T15:04:05.999Z") -]] + [[- else if not (parseDate $time "2006-01-02T15:04:05 MST").IsZero -]] + [[- formatDate (parseDate $time "2006-01-02T15:04:05 MST") -]] + [[- else -]] + [[- formatDate now -]] + [[- end -]] {{ else if eq .input "file" }} type: log diff --git a/x-pack/filebeat/module/google_workspace/config/common.js b/x-pack/filebeat/module/google_workspace/config/common.js index 76371112fc7..f0b4a8ea573 100644 --- a/x-pack/filebeat/module/google_workspace/config/common.js +++ b/x-pack/filebeat/module/google_workspace/config/common.js @@ -18,17 +18,19 @@ var googleWorkspace = (function () { ignore_missing: true, }); - var addID = new processor.Fingerprint({ - fields: [ - "json.id.time", - "json.id.uniqueQualifier", - "json.id.applicationName", - "json.id.customerId", - ], - target_field: "@metadata._id", - ignore_missing: true, - fail_on_error: false, - }); + var addID = function(evt) { + new processor.Fingerprint({ + fields: [ + "json.id.time", + "json.id.uniqueQualifier", + "json.id.applicationName", + "json.id.customerId", + ].concat(Object.keys(evt.Get("json.events"))), + target_field: "@metadata._id", + ignore_missing: true, + fail_on_error: false, + }).Run(evt); + }; var convertFields = new processor.Convert({ fields: [ diff --git a/x-pack/filebeat/module/google_workspace/drive/config/config.yml b/x-pack/filebeat/module/google_workspace/drive/config/config.yml index 86869fa2624..0d38af08cc3 100644 --- a/x-pack/filebeat/module/google_workspace/drive/config/config.yml +++ b/x-pack/filebeat/module/google_workspace/drive/config/config.yml @@ -16,7 +16,7 @@ request.proxy_url: {{ .proxy_url }} request.transforms: - set: target: url.params.startTime - value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]' + value: '[[.cursor.last_execution_datetime]]' default: '[[formatDate (now (parseDuration "-{{.initial_interval}}"))]]' response.split: target: body.items @@ -24,13 +24,34 @@ response.split: target: body.events keep_parent: true response.pagination: + - set: + target: url.params.startTime + value: '[[.last_response.url.params.Get "startTime"]]' + fail_on_template_error: true - set: target: url.params.pageToken - value: "[[.last_response.body.nextPageToken]]" + value: >- + [[- if index .last_response.body "nextPageToken" -]] + [[- .last_response.body.nextPageToken -]] + [[- end -]] fail_on_template_error: true cursor: last_execution_datetime: - value: "[[formatDate now]]" + value: >- + [[- $time := .last_event.id.time -]] + [[- if not (parseDate $time "RFC3339").IsZero -]] + [[- $time -]] + [[- else if not (parseDate $time "2006-01-02T15:04:05").IsZero -]] + [[- formatDate (parseDate $time "2006-01-02T15:04:05") -]] + [[- else if not (parseDate $time "2006-01-02T15:04:05Z").IsZero -]] + [[- formatDate (parseDate $time "2006-01-02T15:04:05Z") -]] + [[- else if not (parseDate $time "2006-01-02T15:04:05.999Z").IsZero -]] + [[- formatDate (parseDate $time "2006-01-02T15:04:05.999Z") -]] + [[- else if not (parseDate $time "2006-01-02T15:04:05 MST").IsZero -]] + [[- formatDate (parseDate $time "2006-01-02T15:04:05 MST") -]] + [[- else -]] + [[- formatDate now -]] + [[- end -]] {{ else if eq .input "file" }} type: log diff --git a/x-pack/filebeat/module/google_workspace/groups/config/config.yml b/x-pack/filebeat/module/google_workspace/groups/config/config.yml index 4438c21cc3a..b3391d75fba 100644 --- a/x-pack/filebeat/module/google_workspace/groups/config/config.yml +++ b/x-pack/filebeat/module/google_workspace/groups/config/config.yml @@ -16,7 +16,7 @@ request.proxy_url: {{ .proxy_url }} request.transforms: - set: target: url.params.startTime - value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]' + value: '[[.cursor.last_execution_datetime]]' default: '[[formatDate (now (parseDuration "-{{.initial_interval}}"))]]' response.split: target: body.items @@ -24,13 +24,34 @@ response.split: target: body.events keep_parent: true response.pagination: + - set: + target: url.params.startTime + value: '[[.last_response.url.params.Get "startTime"]]' + fail_on_template_error: true - set: target: url.params.pageToken - value: "[[.last_response.body.nextPageToken]]" + value: >- + [[- if index .last_response.body "nextPageToken" -]] + [[- .last_response.body.nextPageToken -]] + [[- end -]] fail_on_template_error: true cursor: last_execution_datetime: - value: "[[formatDate now]]" + value: >- + [[- $time := .last_event.id.time -]] + [[- if not (parseDate $time "RFC3339").IsZero -]] + [[- $time -]] + [[- else if not (parseDate $time "2006-01-02T15:04:05").IsZero -]] + [[- formatDate (parseDate $time "2006-01-02T15:04:05") -]] + [[- else if not (parseDate $time "2006-01-02T15:04:05Z").IsZero -]] + [[- formatDate (parseDate $time "2006-01-02T15:04:05Z") -]] + [[- else if not (parseDate $time "2006-01-02T15:04:05.999Z").IsZero -]] + [[- formatDate (parseDate $time "2006-01-02T15:04:05.999Z") -]] + [[- else if not (parseDate $time "2006-01-02T15:04:05 MST").IsZero -]] + [[- formatDate (parseDate $time "2006-01-02T15:04:05 MST") -]] + [[- else -]] + [[- formatDate now -]] + [[- end -]] {{ else if eq .input "file" }} type: log diff --git a/x-pack/filebeat/module/google_workspace/login/config/config.yml b/x-pack/filebeat/module/google_workspace/login/config/config.yml index 1f5052402fe..9da79ace7fc 100644 --- a/x-pack/filebeat/module/google_workspace/login/config/config.yml +++ b/x-pack/filebeat/module/google_workspace/login/config/config.yml @@ -16,7 +16,7 @@ request.proxy_url: {{ .proxy_url }} request.transforms: - set: target: url.params.startTime - value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]' + value: '[[.cursor.last_execution_datetime]]' default: '[[formatDate (now (parseDuration "-{{.initial_interval}}"))]]' response.split: target: body.items @@ -24,13 +24,34 @@ response.split: target: body.events keep_parent: true response.pagination: + - set: + target: url.params.startTime + value: '[[.last_response.url.params.Get "startTime"]]' + fail_on_template_error: true - set: target: url.params.pageToken - value: "[[.last_response.body.nextPageToken]]" + value: >- + [[- if index .last_response.body "nextPageToken" -]] + [[- .last_response.body.nextPageToken -]] + [[- end -]] fail_on_template_error: true cursor: last_execution_datetime: - value: "[[formatDate now]]" + value: >- + [[- $time := .last_event.id.time -]] + [[- if not (parseDate $time "RFC3339").IsZero -]] + [[- $time -]] + [[- else if not (parseDate $time "2006-01-02T15:04:05").IsZero -]] + [[- formatDate (parseDate $time "2006-01-02T15:04:05") -]] + [[- else if not (parseDate $time "2006-01-02T15:04:05Z").IsZero -]] + [[- formatDate (parseDate $time "2006-01-02T15:04:05Z") -]] + [[- else if not (parseDate $time "2006-01-02T15:04:05.999Z").IsZero -]] + [[- formatDate (parseDate $time "2006-01-02T15:04:05.999Z") -]] + [[- else if not (parseDate $time "2006-01-02T15:04:05 MST").IsZero -]] + [[- formatDate (parseDate $time "2006-01-02T15:04:05 MST") -]] + [[- else -]] + [[- formatDate now -]] + [[- end -]] {{ else if eq .input "file" }} type: log diff --git a/x-pack/filebeat/module/google_workspace/saml/config/config.yml b/x-pack/filebeat/module/google_workspace/saml/config/config.yml index faaefbbd1c6..85d03185ece 100644 --- a/x-pack/filebeat/module/google_workspace/saml/config/config.yml +++ b/x-pack/filebeat/module/google_workspace/saml/config/config.yml @@ -16,7 +16,7 @@ request.proxy_url: {{ .proxy_url }} request.transforms: - set: target: url.params.startTime - value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]' + value: '[[.cursor.last_execution_datetime]]' default: '[[formatDate (now (parseDuration "-{{.initial_interval}}"))]]' response.split: target: body.items @@ -24,13 +24,34 @@ response.split: target: body.events keep_parent: true response.pagination: + - set: + target: url.params.startTime + value: '[[.last_response.url.params.Get "startTime"]]' + fail_on_template_error: true - set: target: url.params.pageToken - value: "[[.last_response.body.nextPageToken]]" + value: >- + [[- if index .last_response.body "nextPageToken" -]] + [[- .last_response.body.nextPageToken -]] + [[- end -]] fail_on_template_error: true cursor: last_execution_datetime: - value: "[[formatDate now]]" + value: >- + [[- $time := .last_event.id.time -]] + [[- if not (parseDate $time "RFC3339").IsZero -]] + [[- $time -]] + [[- else if not (parseDate $time "2006-01-02T15:04:05").IsZero -]] + [[- formatDate (parseDate $time "2006-01-02T15:04:05") -]] + [[- else if not (parseDate $time "2006-01-02T15:04:05Z").IsZero -]] + [[- formatDate (parseDate $time "2006-01-02T15:04:05Z") -]] + [[- else if not (parseDate $time "2006-01-02T15:04:05.999Z").IsZero -]] + [[- formatDate (parseDate $time "2006-01-02T15:04:05.999Z") -]] + [[- else if not (parseDate $time "2006-01-02T15:04:05 MST").IsZero -]] + [[- formatDate (parseDate $time "2006-01-02T15:04:05 MST") -]] + [[- else -]] + [[- formatDate now -]] + [[- end -]] {{ else if eq .input "file" }} type: log diff --git a/x-pack/filebeat/module/google_workspace/user_accounts/config/config.yml b/x-pack/filebeat/module/google_workspace/user_accounts/config/config.yml index b3e41216108..cfb46fa33a4 100644 --- a/x-pack/filebeat/module/google_workspace/user_accounts/config/config.yml +++ b/x-pack/filebeat/module/google_workspace/user_accounts/config/config.yml @@ -16,7 +16,7 @@ request.proxy_url: {{ .proxy_url }} request.transforms: - set: target: url.params.startTime - value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]' + value: '[[.cursor.last_execution_datetime]]' default: '[[formatDate (now (parseDuration "-{{.initial_interval}}"))]]' response.split: target: body.items @@ -24,13 +24,34 @@ response.split: target: body.events keep_parent: true response.pagination: + - set: + target: url.params.startTime + value: '[[.last_response.url.params.Get "startTime"]]' + fail_on_template_error: true - set: target: url.params.pageToken - value: "[[.last_response.body.nextPageToken]]" + value: >- + [[- if index .last_response.body "nextPageToken" -]] + [[- .last_response.body.nextPageToken -]] + [[- end -]] fail_on_template_error: true cursor: last_execution_datetime: - value: "[[formatDate now]]" + value: >- + [[- $time := .last_event.id.time -]] + [[- if not (parseDate $time "RFC3339").IsZero -]] + [[- $time -]] + [[- else if not (parseDate $time "2006-01-02T15:04:05").IsZero -]] + [[- formatDate (parseDate $time "2006-01-02T15:04:05") -]] + [[- else if not (parseDate $time "2006-01-02T15:04:05Z").IsZero -]] + [[- formatDate (parseDate $time "2006-01-02T15:04:05Z") -]] + [[- else if not (parseDate $time "2006-01-02T15:04:05.999Z").IsZero -]] + [[- formatDate (parseDate $time "2006-01-02T15:04:05.999Z") -]] + [[- else if not (parseDate $time "2006-01-02T15:04:05 MST").IsZero -]] + [[- formatDate (parseDate $time "2006-01-02T15:04:05 MST") -]] + [[- else -]] + [[- formatDate now -]] + [[- end -]] {{ else if eq .input "file" }} type: log From 17834058bf45f957732817dc90daa01660076e82 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Mon, 14 Nov 2022 10:09:22 +0100 Subject: [PATCH 2/5] Add changelog entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 42685e384ec..d1e132fbc13 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -58,6 +58,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] - Fix handling of empty array in httpjson input. {pull}32001[32001] - Fix reporting of `filebeat.events.active` in log events such that the current value is always reported instead of the difference from the last value. {pull}33597[33597] - Fix splitting array of strings/arrays in httpjson input {issue}30345[30345] {pull}33609[33609] +- Fix Google workspace pagination and document ID generation. {pull}[] *Heartbeat* - Fix bug affecting let's encrypt and other users of cross-signed certs, where cert expiration was incorrectly calculated. {issue}33215[33215] From e08c24ea28f2cfaf1d75119d8c1528db360f3362 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Mon, 14 Nov 2022 10:25:38 +0100 Subject: [PATCH 3/5] Add processing for user_accounts event parameters --- .../user_accounts/config/pipeline.js | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/x-pack/filebeat/module/google_workspace/user_accounts/config/pipeline.js b/x-pack/filebeat/module/google_workspace/user_accounts/config/pipeline.js index 89b54fa72db..69235bf6cdc 100644 --- a/x-pack/filebeat/module/google_workspace/user_accounts/config/pipeline.js +++ b/x-pack/filebeat/module/google_workspace/user_accounts/config/pipeline.js @@ -10,8 +10,34 @@ var userAccounts = (function () { evt.Put("event.category", ["iam"]); }; + var getParamValue = function(param) { + if (param.value) { + return param.value; + } + if (param.multiValue) { + return param.multiValue; + } + if (param.intValue !== null) { + return param.intValue; + } + }; + + var flattenParams = function(evt) { + var params = evt.Get("json.events.parameters"); + if (!params || !Array.isArray(params)) { + return; + } + + params.forEach(function(p){ + evt.Put("google_workspace.user_accounts."+p.name, getParamValue(p)); + }); + + evt.Delete("json.events.parameters"); + }; + var pipeline = new processor.Chain() .Add(categorizeEvent) + .Add(flattenParams) .Build(); return { From 906d505b2d26f8ff22482357a4c3d8ddaacbf19b Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Mon, 14 Nov 2022 10:27:44 +0100 Subject: [PATCH 4/5] Add PR number --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d1e132fbc13..141906bd163 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -58,7 +58,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] - Fix handling of empty array in httpjson input. {pull}32001[32001] - Fix reporting of `filebeat.events.active` in log events such that the current value is always reported instead of the difference from the last value. {pull}33597[33597] - Fix splitting array of strings/arrays in httpjson input {issue}30345[30345] {pull}33609[33609] -- Fix Google workspace pagination and document ID generation. {pull}[] +- Fix Google workspace pagination and document ID generation. {pull}33666[33666] *Heartbeat* - Fix bug affecting let's encrypt and other users of cross-signed certs, where cert expiration was incorrectly calculated. {issue}33215[33215] From ca990f85cf5bfa5758ae226cfff3a7db4323e5e3 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Mon, 14 Nov 2022 11:31:03 +0100 Subject: [PATCH 5/5] Ensure only scalars are added to generate the id --- .../module/google_workspace/config/common.js | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/x-pack/filebeat/module/google_workspace/config/common.js b/x-pack/filebeat/module/google_workspace/config/common.js index f0b4a8ea573..ecaea138a93 100644 --- a/x-pack/filebeat/module/google_workspace/config/common.js +++ b/x-pack/filebeat/module/google_workspace/config/common.js @@ -19,13 +19,21 @@ var googleWorkspace = (function () { }); var addID = function(evt) { + var keys = [ + "json.id.time", + "json.id.uniqueQualifier", + "json.id.applicationName", + "json.id.customerId", + ]; + Object.keys(evt.Get("json.events")).forEach(function(evtsKey) { + var key = "json.events."+evtsKey; + var value = evt.Get(key); + if (!Array.isArray(value) && !(typeof value === "object")) { + keys.push(key); + } + }); new processor.Fingerprint({ - fields: [ - "json.id.time", - "json.id.uniqueQualifier", - "json.id.applicationName", - "json.id.customerId", - ].concat(Object.keys(evt.Get("json.events"))), + fields: keys, target_field: "@metadata._id", ignore_missing: true, fail_on_error: false,