Skip to content

Commit

Permalink
[ML] Fixing module datafeed overrides (#78925) (#78999)
Browse files Browse the repository at this point in the history
  • Loading branch information
jgowdyelastic authored Oct 1, 2020
1 parent 292af55 commit ea018b3
Show file tree
Hide file tree
Showing 27 changed files with 756 additions and 589 deletions.
4 changes: 2 additions & 2 deletions x-pack/plugins/ml/common/types/modules.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ export interface ModuleJob {
config: Omit<Job, 'job_id'>;
}

export interface ModuleDataFeed {
export interface ModuleDatafeed {
id: string;
config: Omit<Datafeed, 'datafeed_id'>;
}
Expand Down Expand Up @@ -49,7 +49,7 @@ export interface Module {
defaultIndexPattern: string;
query: any;
jobs: ModuleJob[];
datafeeds: ModuleDataFeed[];
datafeeds: ModuleDatafeed[];
kibana: KibanaObjects;
}

Expand Down
33 changes: 27 additions & 6 deletions x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import { MlInfoResponse } from '../../../common/types/ml_server_info';
import {
KibanaObjects,
KibanaObjectConfig,
ModuleDataFeed,
ModuleDatafeed,
ModuleJob,
Module,
JobOverride,
Expand Down Expand Up @@ -283,7 +283,7 @@ export class DataRecognizer {
}

const jobs: ModuleJob[] = [];
const datafeeds: ModuleDataFeed[] = [];
const datafeeds: ModuleDatafeed[] = [];
const kibana: KibanaObjects = {};
// load all of the job configs
await Promise.all(
Expand Down Expand Up @@ -710,7 +710,7 @@ export class DataRecognizer {
// save the datafeeds.
// if any fail (e.g. it already exists), catch the error and mark the result
// as success: false
async saveDatafeeds(datafeeds: ModuleDataFeed[]) {
async saveDatafeeds(datafeeds: ModuleDatafeed[]) {
return await Promise.all(
datafeeds.map(async (datafeed) => {
try {
Expand All @@ -723,7 +723,7 @@ export class DataRecognizer {
);
}

async saveDatafeed(datafeed: ModuleDataFeed) {
async saveDatafeed(datafeed: ModuleDatafeed) {
return this._asInternalUser.ml.putDatafeed(
{
datafeed_id: datafeed.id,
Expand All @@ -734,7 +734,7 @@ export class DataRecognizer {
}

async startDatafeeds(
datafeeds: ModuleDataFeed[],
datafeeds: ModuleDatafeed[],
start?: number,
end?: number
): Promise<{ [key: string]: DatafeedResponse }> {
Expand All @@ -746,7 +746,7 @@ export class DataRecognizer {
}

async startDatafeed(
datafeed: ModuleDataFeed,
datafeed: ModuleDatafeed,
start: number | undefined,
end: number | undefined
): Promise<DatafeedResponse> {
Expand Down Expand Up @@ -1229,6 +1229,25 @@ export class DataRecognizer {
const overrides = Array.isArray(datafeedOverrides) ? datafeedOverrides : [datafeedOverrides];
const { datafeeds } = moduleConfig;

// for some items in the datafeed, we should not merge.
// we should instead use the whole override object
function overwriteObjects(source: ModuleDatafeed['config'], update: DatafeedOverride) {
Object.entries(update).forEach(([key, val]) => {
if (typeof val === 'object') {
switch (key) {
case 'query':
case 'aggregations':
case 'aggs':
case 'script_fields':
source[key] = val as any;
break;
default:
break;
}
}
});
}

// separate all the overrides.
// the overrides which don't contain a datafeed id or a job id will be applied to all jobs in the module
const generalOverrides: GeneralDatafeedsOverride[] = [];
Expand All @@ -1244,6 +1263,7 @@ export class DataRecognizer {
generalOverrides.forEach((o) => {
datafeeds.forEach(({ config }) => {
merge(config, o);
overwriteObjects(config, o);
});
});

Expand All @@ -1259,6 +1279,7 @@ export class DataRecognizer {
delete o.job_id;
delete o.datafeed_id;
merge(datafeed.config, o);
overwriteObjects(datafeed.config, o);
}
});
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
{
"job_type": "anomaly_detector",
"description": "Logs UI: Detects anomalies in count of log entries by category",
"groups": ["logs-ui"],
"groups": [
"logs-ui"
],
"analysis_config": {
"bucket_span": "15m",
"categorization_field_name": "message",
Expand All @@ -14,7 +16,10 @@
"use_null": true
}
],
"influencers": ["event.dataset", "mlcategory"],
"influencers": [
"event.dataset",
"mlcategory"
],
"per_partition_categorization": {
"enabled": true,
"stop_on_warn": false
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
{
"icon": "metricbeatApp"
"icon": "metricbeatApp"
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,12 @@
"query": {
"bool": {
"filter": {
"terms" : { "event.dataset" : ["system.cpu", "system.filesystem"]}
"terms": {
"event.dataset": [
"system.cpu",
"system.filesystem"
]
}
}
}
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,14 @@
"query": {
"bool": {
"filter": {
"term": { "event.dataset": "system.cpu" }
"term": {
"event.dataset": "system.cpu"
}
},
"must": {
"exists": { "field": "system.cpu.iowait.pct" }
"exists": {
"field": "system.cpu.iowait.pct"
}
}
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,16 +1,20 @@
{
"job_id": "JOB_ID",
"indices": [
"INDEX_PATTERN_NAME"
],
"query": {
"bool": {
"filter": {
"term": { "event.dataset": "system.filesystem" }
},
"must": {
"exists": { "field": "system.filesystem.used.pct" }
"job_id": "JOB_ID",
"indices": [
"INDEX_PATTERN_NAME"
],
"query": {
"bool": {
"filter": {
"term": {
"event.dataset": "system.filesystem"
}
},
"must": {
"exists": {
"field": "system.filesystem.used.pct"
}
}
}
}
}
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
{
"job_id": "JOB_ID",
"indices": [
"INDEX_PATTERN_NAME"
],
"query": {
"bool": {
"must": {
"exists": { "field": "event.dataset" }
"job_id": "JOB_ID",
"indices": [
"INDEX_PATTERN_NAME"
],
"query": {
"bool": {
"must": {
"exists": {
"field": "event.dataset"
}
}
}
}
}
Original file line number Diff line number Diff line change
@@ -1,54 +1,56 @@
{
"job_type": "anomaly_detector",
"description": "Metricbeat CPU: Detect unusual increases in cpu time spent in iowait (ECS)",
"groups": ["metricbeat"],
"analysis_config": {
"bucket_span": "10m",
"detectors": [
{
"detector_description": "high mean system.cpu.iowait.pct",
"function": "high_mean",
"field_name": "system.cpu.iowait.pct",
"partition_field_name": "host.name",
"custom_rules": [
{
"actions": [
"skip_result"
],
"conditions": [
{
"applies_to": "actual",
"operator": "lt",
"value": 0.25
}
]
}
]
}
],
"influencers": [
"host.name"
]
},
"analysis_limits": {
"model_memory_limit": "25mb"
},
"data_description": {
"time_field": "@timestamp",
"time_format": "epoch_ms"
},
"custom_settings": {
"created_by": "ml-module-metricbeat-system",
"custom_urls": [
{
"url_name": "Host overview",
"time_range": "3h",
"url_value": "dashboards#/view/79ffd6e0-faa0-11e6-947f-177f697178b8-ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(filters:!(),query:(language:kuery,query:\u0027host.name:\u0022$host.name$\u0022\u0027))"
},
{
"url_name": "Raw data",
"url_value": "discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:'event.dataset:\u0022system.cpu\u0022'),sort:!('@timestamp',desc))"
}
]
}
"job_type": "anomaly_detector",
"description": "Metricbeat CPU: Detect unusual increases in cpu time spent in iowait (ECS)",
"groups": [
"metricbeat"
],
"analysis_config": {
"bucket_span": "10m",
"detectors": [
{
"detector_description": "high mean system.cpu.iowait.pct",
"function": "high_mean",
"field_name": "system.cpu.iowait.pct",
"partition_field_name": "host.name",
"custom_rules": [
{
"actions": [
"skip_result"
],
"conditions": [
{
"applies_to": "actual",
"operator": "lt",
"value": 0.25
}
]
}
]
}
],
"influencers": [
"host.name"
]
},
"analysis_limits": {
"model_memory_limit": "25mb"
},
"data_description": {
"time_field": "@timestamp",
"time_format": "epoch_ms"
},
"custom_settings": {
"created_by": "ml-module-metricbeat-system",
"custom_urls": [
{
"url_name": "Host overview",
"time_range": "3h",
"url_value": "dashboards#/view/79ffd6e0-faa0-11e6-947f-177f697178b8-ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(filters:!(),query:(language:kuery,query:\u0027host.name:\u0022$host.name$\u0022\u0027))"
},
{
"url_name": "Raw data",
"url_value": "discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:'event.dataset:\u0022system.cpu\u0022'),sort:!('@timestamp',desc))"
}
]
}
}
Loading

0 comments on commit ea018b3

Please sign in to comment.