x-pack/filebeat/module/sophos/xg: fix kv field separation and add support for timestamped log line

CHANGELOG.next.asciidoc

@@ -150,6 +150,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix handling of escaped newlines in the `decode_cef` processor. {issue}16995[16995] {pull}29268[29268]
 - Fix `panw` module ingest errors for GLOBALPROTECT logs {pull}29154[29154]
 - Fix handling of IPv6 addresses in netflow flow events. {issue}19210[19210] {pull}29383[29383]
+- Fix `sophos` KV splitting and syslog header handling {issue}24237[24237] {pull}29331[29331]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -143278,6 +143278,16 @@ type: keyword
 The related XSS caught by the WAF
 
 
+type: keyword
+
+--
+
+*`sophos.xg.ether_type`*::
++
+--
+The ethernet frame type
+
+
 type: keyword
 
 --

x-pack/filebeat/module/sophos/xg/_meta/fields.yml

@@ -943,3 +943,8 @@
       type: keyword
       description: >
         The related XSS caught by the WAF
+
+    - name: ether_type
+      type: keyword
+      description: >
+        The ethernet frame type

x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml

@@ -6,7 +6,7 @@ processors:
 - grok:
     field: message
     patterns:
-    - '%{SYSLOG5424PRI}%{GREEDYDATA:log.original}$'
+    - '%{SYSLOG5424PRI}(%{SYSLOGTIMESTAMP} %{NOTSPACE} )?%{GREEDYDATA:log.original}$'
 
 # optimize fields / strings in log.original for KV processor
 - gsub:
@@ -17,7 +17,7 @@ processors:
 # split Sophos-XG fields
 - kv:
     field: log.original
-    field_split: " (?=[a-zA-Z0-9\\_\\-]+=)"
+    field_split: " (?=[a-zA-Z0-9_]+=)"
     value_split: "="
     prefix: "sophos.xg."
     ignore_missing: true
@@ -248,6 +248,10 @@ processors:
     - sophos.xg.dir_disp
     - sophos.xg.srczone
     - sophos.xg.dstzone
+    - sophos.xg.log_occurrence
+    - sophos.xg.nat_rule_id
+    - sophos.xg.in_display_interface
+    - sophos.xg.out_display_interface
     - syslog5424_pri
     ignore_missing: true
 

x-pack/filebeat/module/sophos/xg/test/anti-spam.log-expected.json

@@ -91,7 +91,7 @@
         "host.name": "some_other_host.local",
         "input.type": "log",
         "log.level": "informational",
-        "log.offset": 748,
+        "log.offset": 747,
         "network.transport": "TCP",
         "observer.product": "XG",
         "observer.serial_number": "1234567890123457",
@@ -172,7 +172,7 @@
         "host.name": "my_fancy_host",
         "input.type": "log",
         "log.level": "warning",
-        "log.offset": 1541,
+        "log.offset": 1539,
         "network.transport": "TCP",
         "observer.product": "XG",
         "observer.serial_number": "1234567890123456",
@@ -249,7 +249,7 @@
         "host.name": "some_other_host.local",
         "input.type": "log",
         "log.level": "warning",
-        "log.offset": 2295,
+        "log.offset": 2292,
         "network.transport": "TCP",
         "observer.product": "XG",
         "observer.serial_number": "1234567890123457",
@@ -326,7 +326,7 @@
         "host.name": "firewall.localgroup.local",
         "input.type": "log",
         "log.level": "warning",
-        "log.offset": 3125,
+        "log.offset": 3121,
         "network.transport": "TCP",
         "observer.product": "XG",
         "observer.serial_number": "C44313350024-P29PUA",
@@ -397,7 +397,7 @@
         "host.name": "firewall.localgroup.local",
         "input.type": "log",
         "log.level": "warning",
-        "log.offset": 3854,
+        "log.offset": 3849,
         "network.transport": "TCP",
         "observer.product": "XG",
         "observer.serial_number": "S4000806149EE49",
@@ -468,7 +468,7 @@
         "host.name": "firewall.localgroup.local",
         "input.type": "log",
         "log.level": "informational",
-        "log.offset": 4629,
+        "log.offset": 4623,
         "network.transport": "TCP",
         "observer.product": "XG",
         "observer.serial_number": "S4000806149EE49",
@@ -537,7 +537,7 @@
         "host.name": "firewall.localgroup.local",
         "input.type": "log",
         "log.level": "informational",
-        "log.offset": 5391,
+        "log.offset": 5384,
         "network.transport": "TCP",
         "observer.product": "XG",
         "observer.serial_number": "S4000806149EE49",
@@ -607,7 +607,7 @@
         "host.name": "firewall.localgroup.local",
         "input.type": "log",
         "log.level": "warning",
-        "log.offset": 6145,
+        "log.offset": 6137,
         "network.transport": "TCP",
         "observer.product": "XG",
         "observer.serial_number": "S4000806149EE49",
@@ -674,7 +674,7 @@
         "host.name": "firewall.localgroup.local",
         "input.type": "log",
         "log.level": "warning",
-        "log.offset": 6742,
+        "log.offset": 6733,
         "network.transport": "TCP",
         "observer.product": "XG",
         "observer.serial_number": "S4000806149EE49",
@@ -744,7 +744,7 @@
         "host.name": "firewall.localgroup.local",
         "input.type": "log",
         "log.level": "warning",
-        "log.offset": 7447,
+        "log.offset": 7437,
         "network.transport": "TCP",
         "observer.product": "XG",
         "observer.serial_number": "C44313350024-P29PUA",

x-pack/filebeat/module/sophos/xg/test/anti-virus.log

@@ -1,9 +1,9 @@
-<30>device="SFW" date=2020-05-18 time=14:38:33 timezone="CEST" device_name="XG230" device_id=1234567890123457 log_id=030906208001 log_type="Anti-Virus" log_component="HTTP" log_subtype="Virus" status="" priority=Critical fw_rule_id=2 user_name="" iap=13 av_policy_name="" virus="Sandstorm" url="http://sophostest.com/Sandstorm/SBTestFile1.pdf" domainname="sophostest.com" src_ip=172.16.34.24 src_country_code=R1 dst_ip=13.226.155.93 dst_country_code=USA protocol="TCP" src_port=57695 dst_port=80 sent_bytes=550 recv_bytes=1616 user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36" status_code=403
-<30>device="SFW" date=2020-05-18 time=14:38:34 timezone="CEST" device_name="XG230" device_id=1234567890123456 log_id=030906208001 log_type="Anti-Virus" log_component="HTTP" log_subtype="Virus" status="" priority=Critical fw_rule_id=2 user_name="" iap=13 av_policy_name="" virus="EICAR-AV-Test" url="http://sophostest.com/eicar/index.html" domainname="sophostest.com" src_ip=172.16.34.24 src_country_code=R1 dst_ip=13.226.155.18 dst_country_code=USA protocol="TCP" src_port=57835 dst_port=80 sent_bytes=541 recv_bytes=553 user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36" status_code=403
-<30>device="SFW" date=2020-05-18 time=14:38:35 timezone="CEST" device_name="XG230" device_id=1234567890123457 log_id=031106210001 log_type="Anti-Virus" log_component="SMTP" log_subtype="Virus" status="" priority=Critical fw_rule_id=22 user_name="" av_policy_name="default-smtp-av" from_email_address="[email protected]" to_email_address="[email protected]" subject="ZAHLUNG (PROFORMA INVOICE)" mailid="<[email protected]" mailsize=2254721 virus="TR/AD.AgentTesla.eaz" filename="" quarantine="" src_domainname="farasamed.com" dst_domainname="" src_ip=1.128.3.4 src_country_code=DEU dst_ip=186.8.209.194 dst_country_code=DEU protocol="TCP" src_port=56336 dst_port=25 sent_bytes=0 recv_bytes=0 quarantine_reason="Infected"
-<30>device="SFW" date=2020-05-18 time=14:38:36 timezone="CEST" device_name="XG230" device_id=1234567890123456 log_id=031106210001 log_type="Anti-Virus" log_component="SMTP" log_subtype="Virus" status="" priority=Critical fw_rule_id=22 user_name="" av_policy_name="default-smtp-av" from_email_address="[email protected]" to_email_address="[email protected]" subject="Re: NEW PRO-FORMA INVOICE" mailid="<[email protected]>" mailsize=537457 virus="Mal/BredoZp-B" filename="" quarantine="" src_domainname="divella.it" dst_domainname="" src_ip=216.160.83.61 src_country_code=USA dst_ip=185.7.209.194 dst_country_code=DEU protocol="TCP" src_port=54693 dst_port=25 sent_bytes=0 recv_bytes=0 quarantine_reason="Infected"
-<30>device="SFW" date=2018-06-06 time=10:51:29 timezone="IST" device_name="SG430" device_id=S4000806149EE49 log_id=036106211001 log_type="Anti-Virus" log_component="POPS" log_subtype="Virus" status="" priority=Critical fw_rule_id=0 user_name="" av_policy_name="None" from_email_address="[email protected]" to_email_address="[email protected]" subject="EICAR" mailid="<[email protected]>" mailsize=0 virus="EICAR-AV-Test" filename="" quarantine="" src_domainname="postman.local" dst_domainname="" src_ip=10.198.16.121 src_country_code=R1 dst_ip=10.198.234.240 dst_country_code=R1 protocol="TCP" src_port=56653 dst_port=995 sent_bytes=0 recv_bytes=0 quarantine_reason="Other"
-<30>device="SFW" date=2018-06-06 time=10:58:29 timezone="IST" device_name="SG430" device_id=S4000806149EE49 log_id=036206212001 log_type="Anti-Virus" log_component="IMAPS" log_subtype="Virus" status="" priority=Critical fw_rule_id=0 user_name="" av_policy_name="None" from_email_address="[email protected]" to_email_address="[email protected]" subject="EICAR test email" mailid="<[email protected]>" mailsize=0 virus="EICAR-AV-Test" filename="" quarantine="" src_domainname="postman.local" dst_domainname="" src_ip=10.198.16.121 src_country_code=R1 dst_ip=10.198.234.240 dst_country_code=R1 protocol="TCP" src_port=56632 dst_port=993 sent_bytes=0 recv_bytes=0 quarantine_reason="Other"
-<30>device="SFW" date=2018-06-21 time=19:50:23 timezone="CEST" device_name="SF01V" device_id=SFDemo-2df0960 log_id=031006209001 log_type="Anti-Virus" log_component="FTP" log_subtype="Virus" status="" priority=Critical fw_rule_id=0 user_name="" virus="EICAR-AV-Test" FTP_url="/var/www//home/ftp-user/ta_test_file_1ta-cl1-46" FTP_direction="Upload" filename=" /home/ftp-user/ta_test_file_1ta-cl1-46" file_size=0 file_path="/var/www//home/ftp-user/ta_test_file_1ta-cl1-46" ftpcommand="STOR" src_ip=10.146.13.49 src_country_code=R1 dst_ip=10.8.142.181 dst_country_code=R1 protocol="TCP" src_port=39910 dst_port=21 dstdomain="" sent_bytes=0 recv_bytes=0
-<30>device="SFW" date=2018-06-21 time=19:50:48 timezone="CEST" device_name="SF01V" device_id=SFDemo-2df0960 log_id=031001609002 log_type="Anti-Virus" log_component="FTP" log_subtype="Allowed" status="" priority=Information fw_rule_id=0 user_name="" virus="" FTP_url="/var/www//home/ftp-user/ta_test_file_1ta-cl1-46" FTP_direction="Download" filename="/home/ftp-user /ta_test_file_1ta-cl1-46" file_size=19926248 file_path="/var/www//home/ftp-user/ta_test_file_1ta-cl1-46" ftpcommand="RETR" src_ip=10.146.13.49 src_country_code= dst_ip=10.8.142.181 dst_country_code= protocol="TCP" src_port=39936 dst_port=21 dstdomain="" sent_bytes=0 recv_bytes=19926248
-
+<30>device="SFW" date=2020-05-18 time=14:38:33 timezone="CEST" device_name="XG230" device_id=1234567890123457 log_id=030906208001 log_type="Anti-Virus" log_component="HTTP" log_subtype="Virus" status="" priority=Critical fw_rule_id=2 user_name="" iap=13 av_policy_name="" virus="Sandstorm" url="http://sophostest.com/Sandstorm/SBTestFile1.pdf" domainname="sophostest.com" src_ip=172.16.34.24 src_country_code=R1 dst_ip=13.226.155.93 dst_country_code=USA protocol="TCP" src_port=57695 dst_port=80 sent_bytes=550 recv_bytes=1616 user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36" status_code=403
+<30>device="SFW" date=2020-05-18 time=14:38:34 timezone="CEST" device_name="XG230" device_id=1234567890123456 log_id=030906208001 log_type="Anti-Virus" log_component="HTTP" log_subtype="Virus" status="" priority=Critical fw_rule_id=2 user_name="" iap=13 av_policy_name="" virus="EICAR-AV-Test" url="http://sophostest.com/eicar/index.html" domainname="sophostest.com" src_ip=172.16.34.24 src_country_code=R1 dst_ip=13.226.155.18 dst_country_code=USA protocol="TCP" src_port=57835 dst_port=80 sent_bytes=541 recv_bytes=553 user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36" status_code=403
+<30>device="SFW" date=2020-05-18 time=14:38:35 timezone="CEST" device_name="XG230" device_id=1234567890123457 log_id=031106210001 log_type="Anti-Virus" log_component="SMTP" log_subtype="Virus" status="" priority=Critical fw_rule_id=22 user_name="" av_policy_name="default-smtp-av" from_email_address="[email protected]" to_email_address="[email protected]" subject="ZAHLUNG (PROFORMA INVOICE)" mailid="<[email protected]" mailsize=2254721 virus="TR/AD.AgentTesla.eaz" filename="" quarantine="" src_domainname="farasamed.com" dst_domainname="" src_ip=1.128.3.4 src_country_code=DEU dst_ip=186.8.209.194 dst_country_code=DEU protocol="TCP" src_port=56336 dst_port=25 sent_bytes=0 recv_bytes=0 quarantine_reason="Infected"
+<30>device="SFW" date=2020-05-18 time=14:38:36 timezone="CEST" device_name="XG230" device_id=1234567890123456 log_id=031106210001 log_type="Anti-Virus" log_component="SMTP" log_subtype="Virus" status="" priority=Critical fw_rule_id=22 user_name="" av_policy_name="default-smtp-av" from_email_address="[email protected]" to_email_address="[email protected]" subject="Re: NEW PRO-FORMA INVOICE" mailid="<[email protected]>" mailsize=537457 virus="Mal/BredoZp-B" filename="" quarantine="" src_domainname="divella.it" dst_domainname="" src_ip=216.160.83.61 src_country_code=USA dst_ip=185.7.209.194 dst_country_code=DEU protocol="TCP" src_port=54693 dst_port=25 sent_bytes=0 recv_bytes=0 quarantine_reason="Infected"
+<30>device="SFW" date=2018-06-06 time=10:51:29 timezone="IST" device_name="SG430" device_id=S4000806149EE49 log_id=036106211001 log_type="Anti-Virus" log_component="POPS" log_subtype="Virus" status="" priority=Critical fw_rule_id=0 user_name="" av_policy_name="None" from_email_address="[email protected]" to_email_address="[email protected]" subject="EICAR" mailid="<[email protected]>" mailsize=0 virus="EICAR-AV-Test" filename="" quarantine="" src_domainname="postman.local" dst_domainname="" src_ip=10.198.16.121 src_country_code=R1 dst_ip=10.198.234.240 dst_country_code=R1 protocol="TCP" src_port=56653 dst_port=995 sent_bytes=0 recv_bytes=0 quarantine_reason="Other"
+<30>device="SFW" date=2018-06-06 time=10:58:29 timezone="IST" device_name="SG430" device_id=S4000806149EE49 log_id=036206212001 log_type="Anti-Virus" log_component="IMAPS" log_subtype="Virus" status="" priority=Critical fw_rule_id=0 user_name="" av_policy_name="None" from_email_address="[email protected]" to_email_address="[email protected]" subject="EICAR test email" mailid="<[email protected]>" mailsize=0 virus="EICAR-AV-Test" filename="" quarantine="" src_domainname="postman.local" dst_domainname="" src_ip=10.198.16.121 src_country_code=R1 dst_ip=10.198.234.240 dst_country_code=R1 protocol="TCP" src_port=56632 dst_port=993 sent_bytes=0 recv_bytes=0 quarantine_reason="Other"
+<30>device="SFW" date=2018-06-21 time=19:50:23 timezone="CEST" device_name="SF01V" device_id=SFDemo-2df0960 log_id=031006209001 log_type="Anti-Virus" log_component="FTP" log_subtype="Virus" status="" priority=Critical fw_rule_id=0 user_name="" virus="EICAR-AV-Test" FTP_url="/var/www//home/ftp-user/ta_test_file_1ta-cl1-46" FTP_direction="Upload" filename=" /home/ftp-user/ta_test_file_1ta-cl1-46" file_size=0 file_path="/var/www//home/ftp-user/ta_test_file_1ta-cl1-46" ftpcommand="STOR" src_ip=10.146.13.49 src_country_code=R1 dst_ip=10.8.142.181 dst_country_code=R1 protocol="TCP" src_port=39910 dst_port=21 dstdomain="" sent_bytes=0 recv_bytes=0
+<30>device="SFW" date=2018-06-21 time=19:50:48 timezone="CEST" device_name="SF01V" device_id=SFDemo-2df0960 log_id=031001609002 log_type="Anti-Virus" log_component="FTP" log_subtype="Allowed" status="" priority=Information fw_rule_id=0 user_name="" virus="" FTP_url="/var/www//home/ftp-user/ta_test_file_1ta-cl1-46" FTP_direction="Download" filename="/home/ftp-user /ta_test_file_1ta-cl1-46" file_size=19926248 file_path="/var/www//home/ftp-user/ta_test_file_1ta-cl1-46" ftpcommand="RETR" src_ip=10.146.13.49 src_country_code= dst_ip=10.8.142.181 dst_country_code= protocol="TCP" src_port=39936 dst_port=21 dstdomain="" sent_bytes=0 recv_bytes=19926248
+

x-pack/filebeat/module/sophos/xg/test/anti-virus.log-expected.json

@@ -104,7 +104,7 @@
         "http.response.status_code": 403,
         "input.type": "log",
         "log.level": "critical",
-        "log.offset": 673,
+        "log.offset": 672,
         "network.transport": "TCP",
         "observer.product": "XG",
         "observer.serial_number": "1234567890123456",
@@ -178,7 +178,7 @@
         "host.name": "some_other_host.local",
         "input.type": "log",
         "log.level": "critical",
-        "log.offset": 1340,
+        "log.offset": 1338,
         "network.transport": "TCP",
         "observer.product": "XG",
         "observer.serial_number": "1234567890123457",
@@ -254,7 +254,7 @@
         "host.name": "my_fancy_host",
         "input.type": "log",
         "log.level": "critical",
-        "log.offset": 2113,
+        "log.offset": 2110,
         "network.transport": "TCP",
         "observer.product": "XG",
         "observer.serial_number": "1234567890123456",
@@ -337,7 +337,7 @@
         "host.name": "firewall.localgroup.local",
         "input.type": "log",
         "log.level": "critical",
-        "log.offset": 2862,
+        "log.offset": 2858,
         "network.transport": "TCP",
         "observer.product": "XG",
         "observer.serial_number": "S4000806149EE49",
@@ -411,7 +411,7 @@
         "host.name": "firewall.localgroup.local",
         "input.type": "log",
         "log.level": "critical",
-        "log.offset": 3578,
+        "log.offset": 3573,
         "network.transport": "TCP",
         "observer.product": "XG",
         "observer.serial_number": "S4000806149EE49",
@@ -487,7 +487,7 @@
         "host.name": "firewall.localgroup.local",
         "input.type": "log",
         "log.level": "critical",
-        "log.offset": 4304,
+        "log.offset": 4298,
         "network.transport": "TCP",
         "observer.product": "XG",
         "observer.serial_number": "SFDemo-2df0960",
@@ -557,7 +557,7 @@
         "host.name": "firewall.localgroup.local",
         "input.type": "log",
         "log.level": "informational",
-        "log.offset": 4954,
+        "log.offset": 4947,
         "network.transport": "TCP",
         "observer.product": "XG",
         "observer.serial_number": "SFDemo-2df0960",