Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Filebeat] Update crowdstrike module #20138

Merged
merged 18 commits into from
Jul 23, 2020
Merged

[Filebeat] Update crowdstrike module #20138

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
4c341fd
Update crowdstrike module
Jul 22, 2020
665a80b
Add in blank UserIP test
Jul 22, 2020
a6d288c
Fix up conversion error with proper fail_on_error usage
Jul 22, 2020
30cf26d
Add timestamp processing for all timestamps
Jul 22, 2020
2151eea
Add event ingested timestamp
Jul 22, 2020
8b4b57a
Add Changelog entry
Jul 22, 2020
e361568
Merge branch 'master' into crowdstrike-fixes
Jul 22, 2020
cc8f478
Revert accidentally touching suricata pipeline
Jul 22, 2020
0c625ee
Fix up integration tests
Jul 22, 2020
1838a90
Merge branch 'master' of github.com:elastic/beats into crowdstrike-fixes
Jul 22, 2020
13c46c0
Update with new crowdstrike samples
Jul 22, 2020
7dfae0a
Only normalize process when we have info
Jul 22, 2020
ed257ee
Remove empty arguments from string split
Jul 22, 2020
ad59746
update changelog entry
Jul 22, 2020
4d8c1cf
Refactor to use built-in processors
Jul 22, 2020
0606fc4
clean up pipeline code
Jul 22, 2020
1589719
Fix linter errors and add related fields
Jul 23, 2020
3ca4e46
Run mage update in OSS folder to pick up updated fields docs
Jul 23, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions x-pack/filebeat/module/crowdstrike/falcon/_meta/fields.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
Event data fields for each event and alert.
type: group
default_field: false
fields:
fields:
- name: ProcessStartTime
type: date
description: >
Expand Down Expand Up @@ -106,7 +106,7 @@
type: keyword
description: >
SHA256 sum of the executable associated with the detection.

- name: MD5String
type: keyword
description: >
Expand Down
62 changes: 47 additions & 15 deletions x-pack/filebeat/module/crowdstrike/falcon/config/pipeline.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,18 @@
// or more contributor license agreements. Licensed under the Elastic License;
// you may not use this file except in compliance with the Elastic License.

var crowdstrikeFalcon = (function() {
var crowdstrikeFalcon = (function () {
var processor = require("processor");

var convertUnderscore = function(text) {
return text.split(/(?=[A-Z])/).join('_').toLowerCase();
var convertUnderscore = function (text) {
return text.split(/(?=[A-Z])/).join('_').toLowerCase();
};

var convertToMSEpoch = function (evt, field) {
var timestamp = evt.Get(field);
if (timestamp && timestamp < 100000000000) { // check if we have a seconds timestamp, this is roughly 1973 in MS
evt.Put(field, timestamp * 1000);
}
};

var decodeJson = new processor.DecodeJSONFields({
Expand All @@ -16,22 +23,41 @@ var crowdstrikeFalcon = (function() {
max_depth: 8
});

var dropFields = function(evt) {
var dropIfEmpty = function (evt, field) {
var value = evt.Get(field);
if (!value || value === "") {
evt.Delete(field);
}
}

var dropFields = function (evt) {
evt.Delete("message");
evt.Delete("host.name");
dropIfEmpty(evt, "crowdstrike.event.UserIp");
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Don't think we need this. In the convert it should be "fail_on_error: false", not "ignore_failure: true", then we don't need the dropIfEmpty

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Made the change, but are we ok with passing off an empty string in an ip field? You can take a look at the difference in the approach here: a6d288c

cc: @tonymeehan

};

var setFields = function (evt) {
evt.Put("agent.name", "falcon");
};

var convertFields = new processor.Convert({
fields: [
// DetectionSummaryEvent
{ from: "crowdstrike.event.LocalIP", to: "source.ip", type: "ip" },
{ from: "crowdstrike.event.ProcessId", to: "process.pid" },
{
from: "crowdstrike.event.LocalIP",
to: "source.ip",
type: "ip"
},
{
from: "crowdstrike.event.ProcessId",
to: "process.pid"
},
// UserActivityAuditEvent and AuthActivityAuditEvent
{ from: "crowdstrike.event.UserIp", to: "source.ip", type: "ip" },
{
from: "crowdstrike.event.UserIp",
to: "source.ip",
type: "ip"
},
],
mode: "copy",
ignore_missing: true,
Expand All @@ -46,27 +72,31 @@ var crowdstrikeFalcon = (function() {
ignore_missing: false,
});

var processEvent = function(evt) {
var normalizeEpochMS = function (evt) {
convertToMSEpoch(evt, "crowdstrike.event.StartTimestamp")
convertToMSEpoch(evt, "crowdstrike.event.EndTimestamp")
convertToMSEpoch(evt, "crowdstrike.event.UTCTimestamp")
};

var processEvent = function (evt) {
var eventType = evt.Get("crowdstrike.metadata.eventType")
var outcome = evt.Get("crowdstrike.event.Success")

evt.Put("event.kind", "event")

if (outcome === true) {
evt.Put("event.outcome", "success")
}
else if (outcome === false) {
} else if (outcome === false) {
evt.Put("event.outcome", "failure")
}
else {
} else {
evt.Put("event.outcome", "unknown")
}

switch (eventType) {
case "DetectionSummaryEvent":
var tactic = evt.Get("crowdstrike.event.Tactic").toLowerCase()
var technique = evt.Get("crowdstrike.event.Technique").toLowerCase()
evt.Put("threat.technique.name", technique)
evt.Put("threat.technique.name", technique)
evt.Put("threat.tactic.name", tactic)

evt.Put("event.action", evt.Get("crowdstrike.event.PatternDispositionDescription"))
Expand Down Expand Up @@ -167,14 +197,16 @@ var crowdstrikeFalcon = (function() {
default:
break;
}
}
}

var pipeline = new processor.Chain()
.Add(decodeJson)
.Add(parseTimestamp)
.Add(normalizeEpochMS)
.Add(dropFields)
.Add(convertFields)
.Add(processEvent)
.Add(setFields)
.Build();

return {
Expand Down
10 changes: 5 additions & 5 deletions x-pack/filebeat/module/crowdstrike/falcon/test/falcon-audit-events.log-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"@timestamp": "2020-02-27T19:12:14.000Z",
"crowdstrike.event.HostnameField": "hostnameofmachine",
"crowdstrike.event.SessionId": "6020260b-0398-4d41-999d-5531b55522de",
"crowdstrike.event.StartTimestamp": 1582830734,
"crowdstrike.event.StartTimestamp": 1582830734000,
"crowdstrike.event.UserName": "[email protected]",
"crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b",
"crowdstrike.metadata.eventCreationTime": 1582830734000,
Expand Down Expand Up @@ -35,7 +35,7 @@
},
{
"@timestamp": "2020-02-27T19:12:52.000Z",
"crowdstrike.event.EndTimestamp": 1582830772,
"crowdstrike.event.EndTimestamp": 1582830772000,
"crowdstrike.event.HostnameField": "hostnameofmachine",
"crowdstrike.event.SessionId": "6020260b-0398-4d41-999d-5531b55522de",
"crowdstrike.event.UserName": "[email protected]",
Expand Down Expand Up @@ -94,7 +94,7 @@
"crowdstrike.event.OperationName": "streamStarted",
"crowdstrike.event.ServiceName": "Crowdstrike Streaming API",
"crowdstrike.event.Success": true,
"crowdstrike.event.UTCTimestamp": 1581542950,
"crowdstrike.event.UTCTimestamp": 1581542950000,
"crowdstrike.event.UserId": "api-client-id:1234567890abcdefghijklmnopqrstuvwxyz",
"crowdstrike.event.UserIp": "10.10.0.8",
"crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b",
Expand Down Expand Up @@ -219,7 +219,7 @@
],
"crowdstrike.event.OperationName": "update_group",
"crowdstrike.event.ServiceName": "groups",
"crowdstrike.event.UTCTimestamp": 1581546248,
"crowdstrike.event.UTCTimestamp": 1581546248000,
"crowdstrike.event.UserId": "[email protected]",
"crowdstrike.event.UserIp": "192.168.6.13",
"crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b",
Expand Down Expand Up @@ -521,7 +521,7 @@
],
"crowdstrike.event.OperationName": "detection_update",
"crowdstrike.event.ServiceName": "detections",
"crowdstrike.event.UTCTimestamp": 1581603262,
"crowdstrike.event.UTCTimestamp": 1581603262000,
"crowdstrike.event.UserId": "[email protected]",
"crowdstrike.event.UserIp": "192.168.6.8",
"crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b",
Expand Down
26 changes: 26 additions & 0 deletions x-pack/filebeat/module/crowdstrike/falcon/test/falcon-events.log
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,3 +66,29 @@
"FineScore": 1.2
}
}
{
"metadata": {
"customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b",
"offset": 22865,
"eventType": "UserActivityAuditEvent",
"eventCreationTime": 1593186952000,
"version": "1.0"
},
"event": {
"UserId": "Crowdstrike",
"UserIp": "",
"OperationName": "quarantined_file_update",
"ServiceName": "quarantined_files",
"AuditKeyValues": [
{
"Key": "quarantined_file_id",
"ValueString": "35b35a53da374816a6b471cf09e12019_a076d3121743755f2d4f8d4d5807f0bc013177f7847d09b48e76de88ace08c78"
},
{
"Key": "action_taken",
"ValueString": "quarantined"
}
],
"UTCTimestamp": 1593186952
}
}
45 changes: 45 additions & 0 deletions x-pack/filebeat/module/crowdstrike/falcon/test/falcon-events.log-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,5 +122,50 @@
"tags": [
"forwarded"
]
},
{
"@timestamp": "2020-06-26T15:55:52.000Z",
"crowdstrike.event.AuditKeyValues": [
{
"Key": "quarantined_file_id",
"ValueString": "35b35a53da374816a6b471cf09e12019_a076d3121743755f2d4f8d4d5807f0bc013177f7847d09b48e76de88ace08c78"
},
{
"Key": "action_taken",
"ValueString": "quarantined"
}
],
"crowdstrike.event.OperationName": "quarantined_file_update",
"crowdstrike.event.ServiceName": "quarantined_files",
"crowdstrike.event.UTCTimestamp": 1593186952000,
"crowdstrike.event.UserId": "Crowdstrike",
"crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b",
"crowdstrike.metadata.eventCreationTime": 1593186952000,
"crowdstrike.metadata.eventType": "UserActivityAuditEvent",
"crowdstrike.metadata.offset": 22865,
"crowdstrike.metadata.version": "1.0",
"event.action": "user_activity_audit_event",
"event.category": [
"iam"
],
"event.dataset": "crowdstrike.falcon_audit",
"event.kind": "event",
"event.module": "crowdstrike",
"event.outcome": "unknown",
"event.type": [
"change"
],
"fileset.name": "falcon",
"input.type": "log",
"log.flags": [
"multiline"
],
"log.offset": 2579,
"message": "quarantined_file_update",
"service.type": "crowdstrike",
"tags": [
"forwarded"
],
"user.name": "Crowdstrike"
}
]
2 changes: 1 addition & 1 deletion x-pack/filebeat/module/crowdstrike/fields.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.