Stix2-validator taking long time for validating input with network-traffic SCO

[santosh-java]

Dear Team,

Observing that the stix2-validator is taking long time to validate STIX input when it contains a network-traffic SCO in it.

Sample timings for the input are listed below:

INPUT ::::: TIME TAKEN
input1 ~10.45 seconds
input2 ~13.00 seconds

NOTE: When we donot have networ-traffic SCO in the input, the validator framework is taking long time. If network-traffic SCO is not there in the input, then the timings are magnitudes less. For example,

INPUT ::::: TIME TAKEN
input3 ~3.17 seconds

NOTE2: I have few custom schemas added for validation using the following code:
custom_schemas = os.path.abspath(os.path.dirname(file) + "/XDR-customSchemas")
options = ValidationOptions(strict=False, version="2.1", schema_dir=custom_schemas)
results = validate_instance(stix_instance, options)

======input1=====
{
"type": "bundle",
"id": "bundle--1affa123-0a4f-4fc5-81c6-fc1b058f9e44",
"objects": [
{
"type": "identity",
"spec_version": "2.1",
"id": "identity--f3c5def5-9a49-49bd-8e58-e317b754fe6f",
"created": "2017-01-14T13:07:49.812Z",
"modified": "2017-01-14T13:07:49.812Z",
"name": "f3c5def5-9a49-49bd-8e58-e317b754fe6f"
},
{
"type": "x-signal",
"spec_version": "2.1",
"id": "x-signal--4527e5de-8572-446a-a57a-706f15467461",
"created": "2020-01-16T18:52:24.277Z",
"modified": "2020-01-16T18:52:24.277Z",
"x_datasource": "WEBSAAS",
"x_custom_properties": {
"x_category": "Social Networking",
"x_reputation": "Minimal Risk",
"x_block_reason": "TLS handshake failed"
}
},
{
"type": "sighting",
"spec_version": "2.1",
"id": "sighting--779c4ae8-e134-4180-baa4-03141095d971",
"created_by_ref": "identity--f3c5def5-9a49-49bd-8e58-e317b754fe6f",
"created": "2017-02-28T19:37:11.213Z",
"modified": "2017-02-28T19:37:11.213Z",
"first_seen": "2017-02-28T19:07:24.856Z",
"last_seen": "2017-02-28T19:07:24.857Z",
"count": 1,
"sighting_of_ref": "x-signal--4527e5de-8572-446a-a57a-706f15467461",
"observed_data_refs": [
"observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf"
],
"where_sighted_refs": [
"identity--f3c5def5-9a49-49bd-8e58-e317b754fe6f"
]
},
{
"id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf",
"type": "observed-data",
"spec_version": "2.1",
"created": "2020-09-16T12:06:40.000Z",
"modified": "2020-09-16T12:06:40.000Z",
"first_observed": "2020-09-16T12:06:40.000Z",
"last_observed": "2020-09-16T12:06:40.000Z",
"number_observed": 1,
"object_refs": [
"network-traffic--630d7bb1-0bbc-53a6-a6d4-f3c2d35c2734"
]
},
{
"type": "ipv4-addr",
"spec_version": "2.1",
"id": "ipv4-addr--e42c19c8-f9fe-5ae9-9fc8-22c398f78fb7",
"value": "103.245.47.23"
},
{
"type": "ipv4-addr",
"spec_version": "2.1",
"id": "ipv4-addr--03b708d9-7761-5523-ab75-5ea096294a68",
"value": "157.240.7.35"
},
{
"type": "network-traffic",
"spec_version": "2.1",
"id": "network-traffic--630d7bb1-0bbc-53a6-a6d4-f3c2d35c2734",
"src_ref": "ipv4-addr--e42c19c8-f9fe-5ae9-9fc8-22c398f78fb7",
"src_byte_count": 0,
"dst_ref": "ipv4-addr--03b708d9-7761-5523-ab75-5ea096294a68",
"dst_port": 443,
"dst_byte_count": 10140,
"start": "2020-09-16T12:06:40.000Z",
"protocols": [
"https"
],
"extensions": {
"http-request-ext": {
"request_method": "CONNECT",
"request_value": "/",
"request_version": "http/1.1",
"request_header": {
"Host": "www.facebook.com",
"Content-Type": "application/x-empty"
}
}
}
}
]
}
======input2=====
{
"type": "bundle",
"id": "bundle--1affa123-0a4f-4fc5-81c6-fc1b058f9e44",
"objects": [
{
"type": "identity",
"spec_version": "2.1",
"id": "identity--f3c5def5-9a49-49bd-8e58-e317b754fe6f",
"created": "2017-01-14T13:07:49.812Z",
"modified": "2017-01-14T13:07:49.812Z",
"name": "f3c5def5-9a49-49bd-8e58-e317b754fe6f"
},
{
"type": "x-signal",
"spec_version": "2.1",
"id": "x-signal--4527e5de-8572-446a-a57a-706f15467461",
"created": "2020-01-16T18:52:24.277Z",
"modified": "2020-01-16T18:52:24.277Z",
"x_datasource": "WEBSAAS",
"x_custom_properties": {
"x_category": "Social Networking",
"x_reputation": "Minimal Risk",
"x_block_reason": "TLS handshake failed",
"x_result": ""
}
},
{
"type": "relationship",
"spec_version": "2.1",
"id": "relationship--54298a74-ad52-4f0c-87a3-1824e67d7abc",
"created": "2016-04-06T20:07:10.000Z",
"modified": "2016-04-06T20:07:10.000Z",
"relationship_type": "related-to",
"source_ref": "x-device--2D754FC6-56C0-4708-990B-73636CEFA236",
"target_ref": "ipv4-addr--e42c19c8-f9fe-5ae9-9fc8-22c398f78fb7"
},
{
"type": "sighting",
"spec_version": "2.1",
"id": "sighting--779c4ae8-e134-4180-baa4-03141095d971",
"created_by_ref": "identity--f3c5def5-9a49-49bd-8e58-e317b754fe6f",
"created": "2017-02-28T19:37:11.213Z",
"modified": "2017-02-28T19:37:11.213Z",
"first_seen": "2017-02-28T19:07:24.856Z",
"last_seen": "2017-02-28T19:07:24.857Z",
"count": 1,
"sighting_of_ref": "x-signal--4527e5de-8572-446a-a57a-706f15467461",
"observed_data_refs": [
"observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf"
],
"where_sighted_refs": [
"identity--f3c5def5-9a49-49bd-8e58-e317b754fe6f"
]
},
{
"id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf",
"type": "observed-data",
"spec_version": "2.1",
"created": "2020-09-16T12:06:40.000Z",
"modified": "2020-09-16T12:06:40.000Z",
"first_observed": "2020-09-16T12:06:40.000Z",
"last_observed": "2020-09-16T12:06:40.000Z",
"number_observed": 1,
"object_refs": [
"network-traffic--630d7bb1-0bbc-53a6-a6d4-f3c2d35c2734"
]
},
{
"type": "ipv4-addr",
"spec_version": "2.1",
"id": "ipv4-addr--e42c19c8-f9fe-5ae9-9fc8-22c398f78fb7",
"value": "103.245.47.23"
},
{
"type": "ipv4-addr",
"spec_version": "2.1",
"id": "ipv4-addr--03b708d9-7761-5523-ab75-5ea096294a68",
"value": "157.240.7.35"
},
{
"type": "network-traffic",
"spec_version": "2.1",
"id": "network-traffic--630d7bb1-0bbc-53a6-a6d4-f3c2d35c2734",
"src_ref": "ipv4-addr--e42c19c8-f9fe-5ae9-9fc8-22c398f78fb7",
"src_byte_count": 0,
"dst_ref": "ipv4-addr--03b708d9-7761-5523-ab75-5ea096294a68",
"dst_port": 443,
"dst_byte_count": 10140,
"start": "2020-09-16T12:06:40.000Z",
"protocols": [
"https"
],
"extensions": {
"http-request-ext": {
"request_method": "CONNECT",
"request_value": "/",
"request_version": "http/1.1",
"request_header": {
"Host": "www.facebook.com",
"Content-Type": "application/x-empty"
}
}
}
}
]
}

======input3======
{
"type": "bundle",
"id": "bundle--1affa123-0a4f-4fc5-81c6-fc1b058f9e44",
"objects": [
{
"type": "identity",
"spec_version": "2.1",
"id": "identity--f3c5def5-9a49-49bd-8e58-e317b754fe6f",
"created": "2017-01-14T13:07:49.812Z",
"modified": "2017-01-14T13:07:49.812Z",
"name": "f3c5def5-9a49-49bd-8e58-e317b754fe6f"
},
{
"type": "x-signal",
"spec_version": "2.1",
"id": "x-signal--4527e5de-8572-446a-a57a-706f15467461",
"created": "2020-01-16T18:52:24.277Z",
"modified": "2020-01-16T18:52:24.277Z",
"x_severity": "s1",
"x_score": 41,
"x_tactic": [
"@ATE.T1134",
"@ATE.T1059",
"@ATE.T1572"
],
"x_technique": [
"@ATA.CommandAndControl",
"@ATA.PrivilegeEscalation",
"@ATA.Execution"
],
"x_rule_id": "",
"x_datasource": "EDR",
"x_custom_properties": {
"x_hash_id": "7/s41rEyld2DtUrL5m2SRQ==",
"x_event_type": "Threat Detection Summary",
"x_detection_tags": [
"@MSI._process_cmd_meterpreter_other",
"@MSI._file_deletecommon",
"@MSI._file_ep0057_newpe",
"@MSI._reg_ep0170_winservices",
"@MSI._file_ep0119_dropintodata",
"@MSI._process_long_cmdline"
],
"x_related_detections": [
"+nrk/xeytW/hnQTYYZQWFA==",
"/VcEKlD7K7/6o0vh5aqSHw==",
"7S0r+2jo9+H5I58p72uZug==",
"9A7969hjbaJ/Bnk75ZP/Gw==",
"Q24Ky3YiKO4Y1yrdf7cUaw==",
"RX6cm3GYDTWau293sKd9/Q==",
"gD2zZ8u1rvWY6nJWqx//lA==",
"gqR9Ch4AdvMakYbsD6PajA==",
"h88ns9J+bRR3TVrjuGSAhQ=="
]
}
},
{
"type": "sighting",
"spec_version": "2.1",
"id": "sighting--779c4ae8-e134-4180-baa4-03141095d971",
"created_by_ref": "identity--f3c5def5-9a49-49bd-8e58-e317b754fe6f",
"created": "2017-02-28T19:37:11.213Z",
"modified": "2017-02-28T19:37:11.213Z",
"first_seen": "2017-02-28T19:07:24.856Z",
"last_seen": "2017-02-28T19:07:24.857Z",
"count": 1,
"sighting_of_ref": "x-signal--4527e5de-8572-446a-a57a-706f15467461",
"observed_data_refs": [
"observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf"
],
"where_sighted_refs": [
"identity--f3c5def5-9a49-49bd-8e58-e317b754fe6f"
]
},
{
"id": "observed-data--b67d30ff-02ac-498a-92f9-32f845f448cf",
"type": "observed-data",
"spec_version": "2.1",
"created": "2020-09-16T12:06:40.000Z",
"modified": "2020-09-16T12:06:40.000Z",
"first_observed": "2020-09-16T12:06:40.000Z",
"last_observed": "2020-09-16T12:06:40.000Z",
"number_observed": 1,
"object_refs": [
"file--1190f2c9-166f-55f1-9706-eea3971d8082",
"process--f52a906a-0dfc-40bd-92f1-e7778ead38a9",
"x-device--2D754FC6-56C0-4708-990B-73636CEFA236"
]
},
{
"type": "x-device",
"id": "x-device--2D754FC6-56C0-4708-990B-73636CEFA236",
"spec_version": "2.1",
"created": "2016-08-01T00:00:00.000Z",
"modified": "2016-08-01T00:00:00.000Z",
"labels": [
"Workstation"
],
"x_hostname": "",
"x_agent_guid": "2D754FC6-56C0-4708-990B-73636CEFA236",
"x_operating_system_type": "",
"x_operating_system": ""
},
{
"type": "file",
"id": "file--1190f2c9-166f-55f1-9706-eea3971d8082",
"spec_version": "2.1",
"hashes": {
"MD5": "911905B05C4057E0B90B6A445B6D3B4D",
"SHA-1": "46A9244320AE9A38001A7B9EE525EAA20ABA01AD",
"SHA-256": "839348F23263D440CAD107A4F098DB4341231B835593528E521E3E96F9849C01"
},
"name": "cmd.exe",
"parent_directory_ref": "directory--255cb0e4-8bdb-5d63-bb32-9c6f0b733ab2"
},
{
"type": "directory",
"id": "directory--255cb0e4-8bdb-5d63-bb32-9c6f0b733ab2",
"spec_version": "2.1",
"path": "C:\Windows\System32\cmd.exe"
},
{
"type": "process",
"spec_version": "2.1",
"id": "process--f52a906a-0dfc-40bd-92f1-e7778ead38a9",
"pid": 5600,
"created": "2016-01-20T14:11:25.55Z",
"command_line": "cmd.exe",
"image-ref": "file--e04f22d1-be2c-59de-add8-10f61d15fe20"
},
{
"type": "file",
"id": "file--e04f22d1-be2c-59de-add8-10f61d15fe20",
"spec_version": "2.1",
"name": "cmd.exe",
"hashes": {
"MD5": "4E2ACF4F8A396486AB4268C94A6A245F",
"SHA-1": "3CE71813199ABAE99348F61F0CAA34E2574F831C",
"SHA-256": "9A7C58BD98D70631AA1473F7B57B426DB367D72429A5455B433A05EE251F3236"
}
}
]
}