[ML] Fixes errors in JSON of SIEM module job configurations (#48508) (#…

…48543) * [ML] Fixes errors in JSON of SIEM module job configurations * [ML] Fixes queries in custom URLs for two SIEM jobs
elastic · Oct 18, 2019 · b06d553 · b06d553
1 parent 5672882
commit b06d553
Show file tree

Hide file tree

Showing 9 changed files with 180 additions and 292 deletions.
diff --git a/...els/data_recognizer/modules/siem_auditbeat/ml/datafeed_suspicious_login_activity_ecs.json b/...els/data_recognizer/modules/siem_auditbeat/ml/datafeed_suspicious_login_activity_ecs.json
diff --git a/...odels/data_recognizer/modules/siem_auditbeat/ml/linux_anomalous_network_activity_ecs.json b/...odels/data_recognizer/modules/siem_auditbeat/ml/linux_anomalous_network_activity_ecs.json
@@ -15,52 +15,39 @@
         "by_field_name": "process.name"
       }
     ],
-    "analysis_config": {
-      "bucket_span": "15m",
-      "detectors": [
-        {
-          "detector_description": "rare by \"process.name\"",
-          "function": "rare",
-          "by_field_name": "process.name"
-        }
-      ],
-      "influencers": [
-        "host.name",
-        "process.name",
-        "user.name",
-        "destination.ip"
-      ]
-    },
-    "allow_lazy_open": true,
-    "analysis_limits": {
-      "model_memory_limit": "64mb"
-    },
-    "data_description": {
-      "time_field": "@timestamp"
-    },
-    "custom_settings": {
-      "created_by": "ml-module-siem-auditbeat",
-      "custom_urls": [
-        {
-          "url_name": "Host Details by process name",
-          "url_value": "siem#/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
-        },
-        {
-          "url_name": "Host Details by user name",
-          "url_value": "siem#/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
-        },
-        {
-          "url_name": "Hosts Overview by process name",
-          "url_value": "siem#/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
-        },
-        {
-          "url_name": "Hosts Overview by user name",
-          "url_value": "siem#/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
-        }
-      ]
-    }
-  ]
-}
-]
+    "influencers": [
+      "host.name",
+      "process.name",
+      "user.name",
+      "destination.ip"
+    ]
+  },
+  "allow_lazy_open": true,
+  "analysis_limits": {
+    "model_memory_limit": "64mb"
+  },
+  "data_description": {
+    "time_field": "@timestamp"
+  },
+  "custom_settings": {
+    "created_by": "ml-module-siem-auditbeat",
+    "custom_urls": [
+      {
+        "url_name": "Host Details by process name",
+        "url_value": "siem#/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
+      },
+      {
+        "url_name": "Host Details by user name",
+        "url_value": "siem#/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
+      },
+      {
+        "url_name": "Hosts Overview by process name",
+        "url_value": "siem#/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
+      },
+      {
+        "url_name": "Hosts Overview by user name",
+        "url_value": "siem#/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
+      }
+    ]
+  }
 }
-}
diff --git a/.../data_recognizer/modules/siem_auditbeat/ml/linux_anomalous_network_port_activity_ecs.json b/.../data_recognizer/modules/siem_auditbeat/ml/linux_anomalous_network_port_activity_ecs.json
@@ -34,20 +34,20 @@
     "custom_urls": [
       {
         "url_name": "Host Details by process name",
-        "url_value": "siem#/ml-hosts/$host.name$?_g=()&query=(expression:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
+        "url_value": "siem#/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
       },
       {
         "url_name": "Host Details by user name",
-        "url_value": "siem#/ml-hosts/$host.name$?_g=()&query=(expression:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
+        "url_value": "siem#/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
       },
       {
         "url_name": "Hosts Overview by process name",
-        "url_value": "siem#/ml-hosts?_g=()&query=(expression:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
+        "url_value": "siem#/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
       },
       {
         "url_name": "Hosts Overview by user name",
-        "url_value": "siem#/ml-hosts?_g=()&query=(expression:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
+        "url_value": "siem#/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
       }
     ]
   }
-}
+}
diff --git a/...erver/models/data_recognizer/modules/siem_auditbeat/ml/linux_anomalous_user_name_ecs.json b/...erver/models/data_recognizer/modules/siem_auditbeat/ml/linux_anomalous_user_name_ecs.json
@@ -15,50 +15,38 @@
         "by_field_name": "user.name"
       }
     ],
-    "description": "SIEM Auditbeat: Rare and unusual users that are not normally active may indicate unauthorized changes or activity by an unauthorized user which may be credentialed access or lateral movement (beta)",
-    "analysis_config": {
-      "bucket_span": "15m",
-      "detectors": [
-        {
-          "detector_description": "rare by \"user.name\"",
-          "function": "rare",
-          "by_field_name": "user.name"
-        }
-      ],
-      "influencers": [
-        "host.name",
-        "process.name",
-        "user.name"
-      ]
-    },
-    "allow_lazy_open": true,
-    "analysis_limits": {
-      "model_memory_limit": "32mb"
-    },
-    "data_description": {
-      "time_field": "@timestamp"
-    },
-    "custom_settings": {
-      "created_by": "ml-module-siem-auditbeat",
-      "custom_urls": [
-        {
-          "url_name": "Host Details by process name",
-          "url_value": "siem#/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
-        },
-        {
-          "url_name": "Host Details by user name",
-          "url_value": "siem#/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
-        },
-        {
-          "url_name": "Hosts Overview by process name",
-          "url_value": "siem#/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
-        },
-        {
-          "url_name": "Hosts Overview by user name",
-          "url_value": "siem#/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
-        }
-      ]
-    }
+    "influencers": [
+      "host.name",
+      "process.name",
+      "user.name"
+    ]
+  },
+  "allow_lazy_open": true,
+  "analysis_limits": {
+    "model_memory_limit": "32mb"
+  },
+  "data_description": {
+    "time_field": "@timestamp"
+  },
+  "custom_settings": {
+    "created_by": "ml-module-siem-auditbeat",
+    "custom_urls": [
+      {
+        "url_name": "Host Details by process name",
+        "url_value": "siem#/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
+      },
+      {
+        "url_name": "Host Details by user name",
+        "url_value": "siem#/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
+      },
+      {
+        "url_name": "Hosts Overview by process name",
+        "url_value": "siem#/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
+      },
+      {
+        "url_name": "Hosts Overview by user name",
+        "url_value": "siem#/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
+      }
+    ]
   }
 }
-}
diff --git a/...erver/models/data_recognizer/modules/siem_auditbeat/ml/suspicious_login_activity_ecs.json b/...erver/models/data_recognizer/modules/siem_auditbeat/ml/suspicious_login_activity_ecs.json
diff --git a/...ls/data_recognizer/modules/siem_winlogbeat/ml/windows_anomalous_network_activity_ecs.json b/...ls/data_recognizer/modules/siem_winlogbeat/ml/windows_anomalous_network_activity_ecs.json
@@ -15,52 +15,39 @@
         "by_field_name": "process.name"
       }
     ],
-    "analysis_config": {
-      "bucket_span": "15m",
-      "detectors": [
-        {
-          "detector_description": "rare by \"process.name\"",
-          "function": "rare",
-          "by_field_name": "process.name"
-        }
-      ],
-      "influencers": [
-        "host.name",
-        "process.name",
-        "user.name",
-        "destination.ip"
-      ]
-    },
-    "allow_lazy_open": true,
-    "analysis_limits": {
-      "model_memory_limit": "64mb"
-    },
-    "data_description": {
-      "time_field": "@timestamp"
-    },
-    "custom_settings": {
-      "created_by": "ml-module-siem-winlogbeat",
-      "custom_urls": [
-        {
-          "url_name": "Host Details by process name",
-          "url_value": "siem#/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
-        },
-        {
-          "url_name": "Host Details by user name",
-          "url_value": "siem#/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
-        },
-        {
-          "url_name": "Hosts Overview by process name",
-          "url_value": "siem#/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
-        },
-        {
-          "url_name": "Hosts Overview by user name",
-          "url_value": "siem#/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
-        }
-      ]
-    }
-  ]
-}
-]
+    "influencers": [
+      "host.name",
+      "process.name",
+      "user.name",
+      "destination.ip"
+    ]
+  },
+  "allow_lazy_open": true,
+  "analysis_limits": {
+    "model_memory_limit": "64mb"
+  },
+  "data_description": {
+    "time_field": "@timestamp"
+  },
+  "custom_settings": {
+    "created_by": "ml-module-siem-winlogbeat",
+    "custom_urls": [
+      {
+        "url_name": "Host Details by process name",
+        "url_value": "siem#/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
+      },
+      {
+        "url_name": "Host Details by user name",
+        "url_value": "siem#/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
+      },
+      {
+        "url_name": "Hosts Overview by process name",
+        "url_value": "siem#/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
+      },
+      {
+        "url_name": "Hosts Overview by user name",
+        "url_value": "siem#/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
+      }
+    ]
+  }
 }
-}