From 366679cebf7bb3379009222a100aed4ec4cdbd50 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 27 Sep 2024 22:08:26 +1000 Subject: [PATCH] [8.x] [ML] Single Metric Viewer: Enable cross-filtering for 'by', 'over' and 'partition' field values (#193255) (#194280) # Backport This will backport the following commits from `main` to `8.x`: - [[ML] Single Metric Viewer: Enable cross-filtering for 'by', 'over' and 'partition' field values (#193255)](https://github.com/elastic/kibana/pull/193255) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Robert Jaszczurek <92210485+rbrtj@users.noreply.github.com> --- x-pack/plugins/ml/common/types/storage.ts | 1 + .../entity_control/entity_control.tsx | 2 +- .../series_controls/series_controls.tsx | 58 ++++++---- .../get_partition_fields_values.ts | 17 ++- .../routes/schemas/results_service_schema.ts | 1 + .../ml/results/get_partition_fields_values.ts | 100 +++++++++++++++++- 6 files changed, 154 insertions(+), 25 deletions(-) diff --git a/x-pack/plugins/ml/common/types/storage.ts b/x-pack/plugins/ml/common/types/storage.ts index 7213fa134c1a56..145be087fcfdd1 100644 --- a/x-pack/plugins/ml/common/types/storage.ts +++ b/x-pack/plugins/ml/common/types/storage.ts @@ -35,6 +35,7 @@ export type PartitionFieldConfig = by: 'anomaly_score' | 'name'; order: 'asc' | 'desc'; }; + value: string; } | undefined; diff --git a/x-pack/plugins/ml/public/application/timeseriesexplorer/components/entity_control/entity_control.tsx b/x-pack/plugins/ml/public/application/timeseriesexplorer/components/entity_control/entity_control.tsx index 04f8944376fe5e..9a877d8c52fc7f 100644 --- a/x-pack/plugins/ml/public/application/timeseriesexplorer/components/entity_control/entity_control.tsx +++ b/x-pack/plugins/ml/public/application/timeseriesexplorer/components/entity_control/entity_control.tsx @@ -188,7 +188,7 @@ export class EntityControl extends Component; * Provides default fields configuration. */ const getDefaultFieldConfig = ( - fieldTypes: MlEntityFieldType[], + entities: Entity[], isAnomalousOnly: boolean, applyTimeRange: boolean ): UiPartitionFieldsConfig => { - return fieldTypes.reduce((acc, f) => { - acc[f] = { + return entities.reduce((acc, f) => { + acc[f.fieldType] = { applyTimeRange, anomalousOnly: isAnomalousOnly, sort: { by: 'anomaly_score', order: 'desc' }, + ...(f.fieldValue && { value: f.fieldValue }), }; return acc; }, {} as UiPartitionFieldsConfig); @@ -141,18 +142,28 @@ export const SeriesControls: FC> = ({ // Merge the default config with the one from the local storage const resultFieldsConfig = useMemo(() => { - return { - ...getDefaultFieldConfig( - entityControls.map((v) => v.fieldType), - !storageFieldsConfig - ? true - : Object.values(storageFieldsConfig).some((v) => !!v?.anomalousOnly), - !storageFieldsConfig - ? true - : Object.values(storageFieldsConfig).some((v) => !!v?.applyTimeRange) - ), - ...(!storageFieldsConfig ? {} : storageFieldsConfig), - }; + const resultFieldConfig = getDefaultFieldConfig( + entityControls, + !storageFieldsConfig + ? true + : Object.values(storageFieldsConfig).some((v) => !!v?.anomalousOnly), + !storageFieldsConfig + ? true + : Object.values(storageFieldsConfig).some((v) => !!v?.applyTimeRange) + ); + + // Early return to prevent unnecessary looping through the default config + if (!storageFieldsConfig) return resultFieldConfig; + + // Override only the fields properties stored in the local storage + for (const key of Object.keys(resultFieldConfig) as MlEntityFieldType[]) { + resultFieldConfig[key] = { + ...resultFieldConfig[key], + ...storageFieldsConfig[key], + } as UiPartitionFieldConfig; + } + + return resultFieldConfig; }, [entityControls, storageFieldsConfig]); /** @@ -286,9 +297,20 @@ export const SeriesControls: FC> = ({ } } + // Remove the value from the field config to avoid storing it in the local storage + const { value, ...updatedFieldConfigWithoutValue } = updatedFieldConfig; + + // Remove the value from the result config to avoid storing it in the local storage + const updatedResultConfigWithoutValues = Object.fromEntries( + Object.entries(updatedResultConfig).map(([key, fieldValue]) => { + const { value: _, ...rest } = fieldValue; + return [key, rest]; + }) + ); + setStorageFieldsConfig({ - ...updatedResultConfig, - [fieldType]: updatedFieldConfig, + ...updatedResultConfigWithoutValues, + [fieldType]: updatedFieldConfigWithoutValue, }); }, [resultFieldsConfig, setStorageFieldsConfig] diff --git a/x-pack/plugins/ml/server/models/results_service/get_partition_fields_values.ts b/x-pack/plugins/ml/server/models/results_service/get_partition_fields_values.ts index c709c2754953ed..5e7d01b4bf9fb0 100644 --- a/x-pack/plugins/ml/server/models/results_service/get_partition_fields_values.ts +++ b/x-pack/plugins/ml/server/models/results_service/get_partition_fields_values.ts @@ -35,16 +35,24 @@ function getFieldAgg( fieldType: MlPartitionFieldsType, isModelPlotSearch: boolean, query?: string, - fieldConfig?: FieldConfig + fieldsConfig?: FieldsConfig ) { const AGG_SIZE = 100; + const fieldConfig = fieldsConfig?.[fieldType]; const fieldNameKey = `${fieldType}_name`; const fieldValueKey = `${fieldType}_value`; const sortByField = fieldConfig?.sort?.by === 'name' || isModelPlotSearch ? '_key' : 'maxRecordScore'; + const splitFieldFilterValues = Object.entries(fieldsConfig ?? {}) + .filter(([key, field]) => key !== fieldType && field.value) + .map(([key, field]) => ({ + fieldValueKey: `${key}_value`, + fieldValue: field.value, + })); + return { [fieldNameKey]: { terms: { @@ -77,6 +85,11 @@ function getFieldAgg( }, ] : []), + ...splitFieldFilterValues.map((filterValue) => ({ + term: { + [filterValue.fieldValueKey]: filterValue.fieldValue, + }, + })), ], }, }, @@ -233,7 +246,7 @@ export const getPartitionFieldsValuesFactory = (mlClient: MlClient) => ...ML_PARTITION_FIELDS.reduce((acc, key) => { return Object.assign( acc, - getFieldAgg(key, isModelPlotSearch, searchTerm[key], fieldsConfig[key]) + getFieldAgg(key, isModelPlotSearch, searchTerm[key], fieldsConfig) ); }, {}), }, diff --git a/x-pack/plugins/ml/server/routes/schemas/results_service_schema.ts b/x-pack/plugins/ml/server/routes/schemas/results_service_schema.ts index 5b3d268c2fffd9..43a3516b9d6c69 100644 --- a/x-pack/plugins/ml/server/routes/schemas/results_service_schema.ts +++ b/x-pack/plugins/ml/server/routes/schemas/results_service_schema.ts @@ -61,6 +61,7 @@ const fieldConfig = schema.maybe( by: schema.string(), order: schema.maybe(schema.string()), }), + value: schema.maybe(schema.string()), }) ); diff --git a/x-pack/test/api_integration/apis/ml/results/get_partition_fields_values.ts b/x-pack/test/api_integration/apis/ml/results/get_partition_fields_values.ts index cdb6b1df28c9ef..b1124bc5b4f441 100644 --- a/x-pack/test/api_integration/apis/ml/results/get_partition_fields_values.ts +++ b/x-pack/test/api_integration/apis/ml/results/get_partition_fields_values.ts @@ -38,10 +38,38 @@ export default ({ getService }: FtrProviderContext) => { } as Job; } - function getDatafeedConfig(jobId: string) { + function getJobConfigWithByField(jobId: string) { + return { + job_id: jobId, + description: + 'count by geoip.city_name partition=day_of_week on ecommerce dataset with 1h bucket span', + analysis_config: { + bucket_span: '1h', + influencers: ['geoip.city_name', 'day_of_week'], + detectors: [ + { + function: 'count', + by_field_name: 'geoip.city_name', + partition_field_name: 'day_of_week', + }, + ], + }, + data_description: { + time_field: 'order_date', + time_format: 'epoch_ms', + }, + analysis_limits: { + model_memory_limit: '11mb', + categorization_examples_limit: 4, + }, + model_plot_config: { enabled: false }, + } as Job; + } + + function getDatafeedConfig(jobId: string, indices: string[]) { return { datafeed_id: `datafeed-${jobId}`, - indices: ['ft_farequote'], + indices, job_id: jobId, query: { bool: { must: [{ match_all: {} }] } }, } as Datafeed; @@ -50,12 +78,17 @@ export default ({ getService }: FtrProviderContext) => { async function createMockJobs() { await ml.api.createAndRunAnomalyDetectionLookbackJob( getJobConfig('fq_multi_1_ae'), - getDatafeedConfig('fq_multi_1_ae') + getDatafeedConfig('fq_multi_1_ae', ['ft_farequote']) ); await ml.api.createAndRunAnomalyDetectionLookbackJob( getJobConfig('fq_multi_2_ae', false), - getDatafeedConfig('fq_multi_2_ae') + getDatafeedConfig('fq_multi_2_ae', ['ft_farequote']) + ); + + await ml.api.createAndRunAnomalyDetectionLookbackJob( + getJobConfigWithByField('ecommerce_advanced_1'), + getDatafeedConfig('ecommerce_advanced_1', ['ft_ecommerce']) ); } @@ -72,6 +105,7 @@ export default ({ getService }: FtrProviderContext) => { describe('PartitionFieldsValues', function () { before(async () => { await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); + await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/ecommerce'); await ml.testResources.setKibanaTimeZoneToUTC(); await createMockJobs(); }); @@ -229,5 +263,63 @@ export default ({ getService }: FtrProviderContext) => { expect(body.partition_field.values.length).to.eql(19); }); }); + + describe('cross filtering', () => { + it('should return filtered values for by_field when partition_field is set', async () => { + const requestBody = { + jobId: 'ecommerce_advanced_1', + criteriaFields: [{ fieldName: 'detector_index', fieldValue: 0 }], + earliestMs: 1687968000000, // June 28, 2023 16:00:00 GMT + latestMs: 1688140800000, // June 30, 2023 16:00:00 GMT + searchTerm: {}, + fieldsConfig: { + by_field: { + anomalousOnly: true, + applyTimeRange: true, + sort: { order: 'desc', by: 'anomaly_score' }, + }, + partition_field: { + anomalousOnly: true, + applyTimeRange: true, + sort: { order: 'desc', by: 'anomaly_score' }, + value: 'Saturday', + }, + }, + }; + const body = await runRequest(requestBody); + + expect(body.by_field.values.length).to.eql(1); + expect(body.by_field.values[0].value).to.eql('Abu Dhabi'); + }); + + it('should return filtered values for partition_field when by_field is set', async () => { + const requestBody = { + jobId: 'ecommerce_advanced_1', + criteriaFields: [{ fieldName: 'detector_index', fieldValue: 0 }], + earliestMs: 1687968000000, // June 28, 2023 16:00:00 GMT + latestMs: 1688140800000, // June 30, 2023 16:00:00 GMT + searchTerm: {}, + fieldsConfig: { + by_field: { + anomalousOnly: true, + applyTimeRange: true, + sort: { order: 'desc', by: 'anomaly_score' }, + value: 'Abu Dhabi', + }, + partition_field: { + anomalousOnly: true, + applyTimeRange: true, + sort: { order: 'desc', by: 'anomaly_score' }, + }, + }, + }; + + const body = await runRequest(requestBody); + + expect(body.partition_field.values.length).to.eql(2); + expect(body.partition_field.values[0].value).to.eql('Saturday'); + expect(body.partition_field.values[1].value).to.eql('Monday'); + }); + }); }); };