[Tenable IO] Add request tracing to vulnerability datastream and capture chain requests

packages/tenable_io/changelog.yml

@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "2.8.0"
+  changes:
+    - description: Add request tracing to vulnerability datastream.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8831
+    - description: Add chain request/response tracers to asset and vulnerability.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8831
 - version: "2.7.0"
   changes:
     - description: Limit request tracer log count to five.

packages/tenable_io/data_stream/asset/agent/stream/httpjson.yml.hbs

@@ -51,6 +51,11 @@ chain:
       {{/if}}
       replace: $.export_uuid
       until: '[[ eq .last_response.body.status "FINISHED" ]]'
+      {{#if enable_request_tracer}}
+      {{!-- The file name below is hardcoded but once https://github.com/elastic/beats/issues/37551 is implemented, it can be changed back to `"../../logs/httpjson/http-request-trace-*.ndjson"` --}}
+      request.tracer.filename: "../../logs/httpjson/http-request-trace-httpjson-tenable_io.asset-assets_export_status.ndjson"
+      request.tracer.maxbackups: 5
+      {{/if}}
       request.retry.max_attempts: {{retry_max_attempts}}
       request.retry.wait_min: {{retry_wait_min}}
       request.transforms:
@@ -78,6 +83,11 @@ chain:
       {{/if}}
       replace: $.chunks_available[:]
       replace_with: '$.export_uuid,.first_response.body.export_uuid'
+      {{#if enable_request_tracer}}
+      {{!-- The file name below is hardcoded but once https://github.com/elastic/beats/issues/37551 is implemented, it can be changed back to `"../../logs/httpjson/http-request-trace-*.ndjson"` --}}
+      request.tracer.filename: "../../logs/httpjson/http-request-trace-httpjson-tenable_io.asset-assets_export_chunks.ndjson"
+      request.tracer.maxbackups: 5
+      {{/if}}
       request.transforms:
         - delete:
             target: header.User-Agent

packages/tenable_io/data_stream/asset/sample_event.json

@@ -1,11 +1,11 @@
 {
     "@timestamp": "2018-12-31T22:27:58.599Z",
     "agent": {
-        "ephemeral_id": "0a36656c-ec16-48b9-9bec-807010cfc59d",
-        "id": "3c385f00-c1f1-40dd-b812-1cf0a8cc55cf",
+        "ephemeral_id": "b3014c52-baa0-4157-a405-a7c7aba233d9",
+        "id": "9fcad6fb-106f-48a9-a13e-3a4026b60340",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.7.1"
+        "version": "8.11.3"
     },
     "cloud": {
         "availability_zone": "12",
@@ -25,18 +25,18 @@
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "3c385f00-c1f1-40dd-b812-1cf0a8cc55cf",
+        "id": "9fcad6fb-106f-48a9-a13e-3a4026b60340",
         "snapshot": false,
-        "version": "8.7.1"
+        "version": "8.11.3"
     },
     "event": {
         "agent_id_status": "verified",
         "category": [
             "host"
         ],
-        "created": "2023-10-04T07:01:57.013Z",
+        "created": "2024-01-05T11:54:49.061Z",
         "dataset": "tenable_io.asset",
-        "ingested": "2023-10-04T07:02:00Z",
+        "ingested": "2024-01-05T11:54:52Z",
         "kind": "state",
         "original": "{\"acr_score\":\"3\",\"agent_names\":[],\"agent_uuid\":\"22\",\"aws_availability_zone\":null,\"aws_ec2_instance_ami_id\":\"12\",\"aws_ec2_instance_group_name\":null,\"aws_ec2_instance_id\":\"12\",\"aws_ec2_instance_state_name\":null,\"aws_ec2_instance_type\":null,\"aws_ec2_name\":null,\"aws_ec2_product_code\":null,\"aws_owner_id\":\"44\",\"aws_region\":null,\"aws_subnet_id\":null,\"aws_vpc_id\":null,\"azure_resource_id\":\"12\",\"azure_vm_id\":\"12\",\"bigfix_asset_id\":null,\"bios_uuid\":\"33\",\"created_at\":\"2017-12-31T20:40:44.535Z\",\"deleted_at\":\"2017-12-31T20:40:44.535Z\",\"deleted_by\":\"user\",\"exposure_score\":\"721\",\"first_scan_time\":\"2017-12-31T20:40:23.447Z\",\"first_seen\":\"2017-12-31T20:40:23.447Z\",\"fqdns\":[\"example.com\"],\"gcp_instance_id\":\"12\",\"gcp_project_id\":\"12\",\"gcp_zone\":\"12\",\"has_agent\":false,\"has_plugin_results\":true,\"hostnames\":[],\"id\":\"95c2725c-7298-4a44-8a1d-63131ca3f01f\",\"installed_software\":[\"cpe:/a:test:xyz:12.8\",\"cpe:/a:test:abc:7.7.3\",\"cpe:/a:test:pqr:6.9\",\"cpe:/a:test:xyz\"],\"ipv4s\":[\"89.160.20.112\"],\"ipv6s\":[],\"last_authenticated_scan_date\":\"2017-12-31T20:40:44.535Z\",\"last_licensed_scan_date\":\"2018-12-31T22:27:52.869Z\",\"last_scan_id\":\"00283024-afee-44ea-b467-db5a6ed9fd50ab8f7ecb158c480e\",\"last_scan_time\":\"2018-03-31T22:27:52.869Z\",\"last_schedule_id\":\"72284901-7c68-42b2-a0c4-c1e75568849df60557ee0e264228\",\"last_seen\":\"2018-12-31T22:27:52.869Z\",\"mac_addresses\":[],\"manufacturer_tpm_ids\":[],\"mcafee_epo_agent_guid\":null,\"mcafee_epo_guid\":null,\"netbios_names\":[],\"network_interfaces\":[{\"fqdns\":[\"example.com\"],\"ipv4s\":[\"89.160.20.112\",\"81.2.69.144\"],\"ipv6s\":[\"2a02:cf40::\"],\"mac_addresses\":[\"00-00-5E-00-53-00\",\"00-00-5E-00-53-FF\"],\"name\":\"test.0.1234\"}],\"operating_systems\":[],\"qualys_asset_ids\":[],\"qualys_host_ids\":[],\"servicenow_sysid\":null,\"sources\":[{\"first_seen\":\"2017-12-31T20:40:23.447Z\",\"last_seen\":\"2018-12-31T22:27:52.869Z\",\"name\":\"TEST_SCAN\"}],\"ssh_fingerprints\":[],\"symantec_ep_hardware_keys\":[],\"system_types\":[],\"tags\":[{\"added_at\":\"2018-12-31T14:53:13.817Z\",\"added_by\":\"ac2e7ef6-fac9-47bf-9170-617331322885\",\"key\":\"Geographic Area\",\"uuid\":\"47e7f5f6-1013-4401-a705-479bfadc7826\",\"value\":\"APAC\"}],\"terminated_at\":\"2017-12-31T20:40:44.535Z\",\"terminated_by\":\"user\",\"updated_at\":\"2018-12-31T22:27:58.599Z\"}",
         "type": [
@@ -163,4 +163,4 @@
             "updated_at": "2018-12-31T22:27:58.599Z"
         }
     }
-}
+}

packages/tenable_io/data_stream/vulnerability/_dev/test/system/test-default-config.yml

@@ -4,6 +4,7 @@ vars:
   hostname: http://{{Hostname}}:{{Port}}
   access_key: xxxx
   secret_key: xxxx
+  enable_request_tracer: true
 data_stream:
   vars:
     num_assets: 500

packages/tenable_io/data_stream/vulnerability/agent/stream/httpjson.yml.hbs

@@ -1,5 +1,9 @@
 config_version: 2
 interval: {{interval}}
+{{#if enable_request_tracer}}
+request.tracer.filename: "../../logs/httpjson/http-request-trace-*.ndjson"
+request.tracer.maxbackups: 5
+{{/if}}
 {{#if proxy_url}}
 request.proxy_url: {{proxy_url}}
 {{/if}}
@@ -51,6 +55,11 @@ chain:
       {{/if}}
       replace: $.export_uuid
       until: '[[ eq .last_response.body.status "FINISHED" ]]'
+      {{#if enable_request_tracer}}
+      {{!-- The file name below is hardcoded but once https://github.com/elastic/beats/issues/37551 is implemented, it can be changed back to `"../../logs/httpjson/http-request-trace-*.ndjson"` --}}
+      request.tracer.filename: "../../logs/httpjson/http-request-trace-httpjson-tenable_io.vulnerability-vulns_export_status.ndjson"
+      request.tracer.maxbackups: 5
+      {{/if}}
       request.retry.max_attempts: {{retry_max_attempts}}
       request.retry.wait_min: {{retry_wait_min}}
       request.transforms:
@@ -78,6 +87,11 @@ chain:
       {{/if}}
       replace: $.chunks_available[:]
       replace_with: '$.export_uuid,.first_response.body.export_uuid'
+      {{#if enable_request_tracer}}
+      {{!-- The file name below is hardcoded but once https://github.com/elastic/beats/issues/37551 is implemented, it can be changed back to `"../../logs/httpjson/http-request-trace-*.ndjson"` --}}
+      request.tracer.filename: "../../logs/httpjson/http-request-trace-httpjson-tenable_io.vulnerability-vulns_export_chunks.ndjson"
+      request.tracer.maxbackups: 5
+      {{/if}}
       request.transforms:
         - delete:
             target: header.User-Agent

packages/tenable_io/data_stream/vulnerability/sample_event.json

@@ -1,11 +1,11 @@
 {
     "@timestamp": "2018-12-31T20:59:47.000Z",
     "agent": {
-        "ephemeral_id": "6e3ece9c-b654-4877-8236-df8512f3db02",
-        "id": "3c385f00-c1f1-40dd-b812-1cf0a8cc55cf",
+        "ephemeral_id": "42734433-3701-4961-aa7e-b3e499d63cdf",
+        "id": "9fcad6fb-106f-48a9-a13e-3a4026b60340",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.7.1"
+        "version": "8.11.3"
     },
     "data_stream": {
         "dataset": "tenable_io.vulnerability",
@@ -16,18 +16,18 @@
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "3c385f00-c1f1-40dd-b812-1cf0a8cc55cf",
+        "id": "9fcad6fb-106f-48a9-a13e-3a4026b60340",
         "snapshot": false,
-        "version": "8.7.1"
+        "version": "8.11.3"
     },
     "event": {
         "agent_id_status": "verified",
         "category": [
             "vulnerability"
         ],
-        "created": "2023-10-04T07:04:18.191Z",
+        "created": "2024-01-05T11:53:21.505Z",
         "dataset": "tenable_io.vulnerability",
-        "ingested": "2023-10-04T07:04:22Z",
+        "ingested": "2024-01-05T11:53:24Z",
         "kind": "state",
         "original": "{\"asset\":{\"fqdn\":\"example.com\",\"hostname\":\"89.160.20.112\",\"ipv4\":\"81.2.69.142\",\"network_id\":\"00000000-0000-0000-0000-000000000000\",\"operating_system\":[\"Test Demo OS X 10.5.8\"],\"tracked\":true,\"uuid\":\"cf165808-6a31-48e1-9cf3-c6c3174df51d\"},\"first_found\":\"2018-12-31T20:59:47Z\",\"indexed\":\"2022-11-30T14:09:12.061Z\",\"last_found\":\"2018-12-31T20:59:47Z\",\"output\":\"The observed version of Test  is : \\n /21.0.1180.90\",\"plugin\":{\"cve\":[\"CVE-2016-1620\",\"CVE-2016-1614\",\"CVE-2016-1613\",\"CVE-2016-1612\",\"CVE-2016-1618\",\"CVE-2016-1617\",\"CVE-2016-1616\",\"CVE-2016-1615\",\"CVE-2016-1619\"],\"cvss_base_score\":9.3,\"cvss_temporal_score\":6.9,\"cvss_temporal_vector\":{\"exploitability\":\"Unproven\",\"raw\":\"E:U/RL:OF/RC:C\",\"remediation_level\":\"Official-fix\",\"report_confidence\":\"Confirmed\"},\"cvss_vector\":{\"access_complexity\":\"Medium\",\"access_vector\":\"Network\",\"authentication\":\"None required\",\"availability_impact\":\"Complete\",\"confidentiality_impact\":\"Complete\",\"integrity_impact\":\"Complete\",\"raw\":\"AV:N/AC:M/Au:N/C:C/I:C/A:C\"},\"description\":\"The version of Test  on the remote host is prior to 48.0.2564.82 and is affected by the following vulnerabilities: \\n\\n - An unspecified vulnerability exists in Test V8 when handling compatible receiver checks hidden behind receptors.  An attacker can exploit this to have an unspecified impact.  No other details are available. (CVE-2016-1612)\\n - A use-after-free error exists in `PDFium` due to improper invalidation of `IPWL_FocusHandler` and `IPWL_Provider` upon destruction.  An attacker can exploit this to dereference already freed memory, resulting in the execution of arbitrary code. (CVE-2016-1613)\\n - An unspecified vulnerability exists in `Blink` that is related to the handling of bitmaps.  An attacker can exploit this to access sensitive information.  No other details are available. (CVE-2016-1614)\\n - An unspecified vulnerability exists in `omnibox` that is related to origin confusion.  An attacker can exploit this to have an unspecified impact.  No other details are available. (CVE-2016-1615)\\n - An unspecified vulnerability exists that allows an attacker to spoof a displayed URL.  No other details are available. (CVE-2016-1616)\\n - An unspecified vulnerability exists that is related to history sniffing with HSTS and CSP. No other details are available. (CVE-2016-1617)\\n - A flaw exists in `Blink` due to the weak generation of random numbers by the ARC4-based random number generator.  An attacker can exploit this to gain access to sensitive information.  No other details are available. (CVE-2016-1618)\\n - An out-of-bounds read error exists in `PDFium` in file `fx_codec_jpx_opj.cpp` in the `sycc4{22,44}_to_rgb()` functions. An attacker can exploit this to cause a denial of service by crashing the application linked using the library. (CVE-2016-1619)\\n - Multiple vulnerabilities exist, the most serious of which allow an attacker to execute arbitrary code via a crafted web page. (CVE-2016-1620)\\n - A flaw in `objects.cc` is triggered when handling cleared `WeakCells`, which may allow a context-dependent attacker to have an unspecified impact. No further details have been provided. (CVE-2016-2051)\",\"family\":\"Web Clients\",\"family_id\":1000020,\"has_patch\":false,\"id\":9062,\"name\":\"Test  \\u0026lt; 48.0.2564.82 Multiple Vulnerabilities\",\"risk_factor\":\"HIGH\",\"see_also\":[\"http://testreleases.blogspot.com/2016/01/beta-channel-update_20.html\"],\"solution\":\"Update the  browser to 48.0.2564.82 or later.\",\"synopsis\":\"The remote host is utilizing a web browser that is affected by multiple vulnerabilities.\",\"vpr\":{\"drivers\":{\"age_of_vuln\":{\"lower_bound\":366,\"upper_bound\":730},\"cvss3_impact_score\":5.9,\"cvss_impact_score_predicted\":false,\"exploit_code_maturity\":\"UNPROVEN\",\"product_coverage\":\"LOW\",\"threat_intensity_last28\":\"VERY_LOW\",\"threat_sources_last28\":[\"No recorded events\"]},\"score\":5.9,\"updated\":\"2019-12-31T10:08:58Z\"}},\"port\":{\"port\":\"0\",\"protocol\":\"TCP\"},\"scan\":{\"completed_at\":\"2018-12-31T20:59:47Z\",\"schedule_uuid\":\"6f7db010-9cb6-4870-b745-70a2aea2f81ce1b6640fe8a2217b\",\"started_at\":\"2018-12-31T20:59:47Z\",\"uuid\":\"0e55ec5d-c7c7-4673-a618-438a84e9d1b78af3a9957a077904\"},\"severity\":\"low\",\"severity_default_id\":3,\"severity_id\":3,\"severity_modification_type\":\"NONE\",\"state\":\"OPEN\"}",
         "type": [
@@ -197,4 +197,4 @@
         },
         "severity": "low"
     }
-}
+}